npm - axiom - Versions diffs - 0.44.0 → 0.45.0 - Mend

axiom 0.44.0 → 0.45.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (40) hide show

package/dist/bin.cjs +6 -6
package/dist/bin.cjs.map +1 -1
package/dist/bin.js +5 -5
package/dist/chunk-3Q36GUCI.js +23 -0
package/dist/chunk-3Q36GUCI.js.map +1 -0
package/dist/{chunk-MCKVQ2IB.js → chunk-JSEK2D3F.js} +5 -19
package/dist/chunk-JSEK2D3F.js.map +1 -0
package/dist/{chunk-HW6E4M7S.js → chunk-N3RHEQMQ.js} +2 -2
package/dist/{chunk-HW6E4M7S.js.map → chunk-N3RHEQMQ.js.map} +1 -1
package/dist/{chunk-U3JDCQ3Y.js → chunk-QAIGRB75.js} +7 -7
package/dist/{chunk-7NTVK3F4.js → chunk-R4CARCOX.js} +2 -2
package/dist/{chunk-56V2A6IW.js → chunk-RSHQRY3F.js} +3 -3
package/dist/{chunk-AAYRMTT6.js → chunk-SGAIMXRV.js} +24 -9
package/dist/chunk-SGAIMXRV.js.map +1 -0
package/dist/evals/online.cjs +31 -8
package/dist/evals/online.cjs.map +1 -1
package/dist/evals/online.d.cts +9 -4
package/dist/evals/online.d.ts +9 -4
package/dist/evals/online.js +3 -2
package/dist/evals/online.js.map +1 -1
package/dist/evals/scorers.cjs +13 -16
package/dist/evals/scorers.cjs.map +1 -1
package/dist/evals/scorers.js +2 -1
package/dist/evals/scorers.js.map +1 -1
package/dist/evals.cjs +22 -23
package/dist/evals.cjs.map +1 -1
package/dist/evals.js +8 -8
package/dist/evals.js.map +1 -1
package/dist/index.cjs +36 -24
package/dist/index.cjs.map +1 -1
package/dist/index.js +5 -4
package/dist/index.js.map +1 -1
package/dist/{run-vitest-4OEEDEHV.js → run-vitest-SYYO32TD.js} +5 -5
package/package.json +1 -1
package/dist/chunk-AAYRMTT6.js.map +0 -1
package/dist/chunk-MCKVQ2IB.js.map +0 -1
/package/dist/{chunk-U3JDCQ3Y.js.map → chunk-QAIGRB75.js.map} +0 -0
/package/dist/{chunk-7NTVK3F4.js.map → chunk-R4CARCOX.js.map} +0 -0
/package/dist/{chunk-56V2A6IW.js.map → chunk-RSHQRY3F.js.map} +0 -0
/package/dist/{run-vitest-4OEEDEHV.js.map → run-vitest-SYYO32TD.js.map} +0 -0

package/dist/evals/online.cjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"sources":["../../src/evals/online.ts","../../src/online-evals/onlineEval.ts","../../src/otel/initAxiomAI.ts","../../package.json","../../src/otel/utils/redaction.ts","../../src/online-evals/executor.ts","../../src/otel/semconv/attributes.ts","../../src/otel/semconv/eval_proposal.ts"],"sourcesContent":["export { onlineEval } from '../online-evals/onlineEval';\nexport type { OnlineEvalMeta, OnlineEvalOptions } from '../online-evals/onlineEval';\nexport type {\n OnlineEvalScorerEntry,\n OnlineEvalScorerInput,\n SampledOnlineEvalScorer,\n ScorerSampling,\n} from '../online-evals/types';\n","import { context, trace, SpanStatusCode, type SpanContext } from '@opentelemetry/api';\nimport { getGlobalTracer } from '../otel/initAxiomAI';\nimport type {\n OnlineEvalScorerEntry,\n OnlineEvalScorerInput,\n SampledOnlineEvalScorer,\n Scorer,\n ScorerResult,\n ScorerSampling,\n} from './types';\nimport { executeScorer } from './executor';\nimport { Attr } from '../otel/semconv/attributes';\n\ntype ScorerEntry<TInput, TOutput> = OnlineEvalScorerEntry<TInput, TOutput, any>;\ntype ScorerInput<TInput, TOutput> = OnlineEvalScorerInput<TInput, TOutput, any>;\n\ntype InferScorerMetadata<TScorerEntry> =\n TScorerEntry extends SampledOnlineEvalScorer<any, any, any>\n ? InferScorerMetadata<TScorerEntry['scorer']>\n : TScorerEntry extends Scorer<any, any, infer TMetadata>\n ? TMetadata\n : TScorerEntry extends ScorerResult<infer TMetadata>\n ? TMetadata\n : never;\n\ntype InferScorerName<TScorerEntry> =\n TScorerEntry extends SampledOnlineEvalScorer<any, any, any>\n ? InferScorerName<TScorerEntry['scorer']>\n : TScorerEntry extends ScorerResult<any>\n ? TScorerEntry['name']\n : TScorerEntry extends Scorer<any, any, any>\n ? string\n : never;\n\ntype IsBroadString<T extends string> = string extends T ? true : false;\n\ntype DuplicateScorerNames<\n TEntries extends readonly unknown[],\n Seen extends string = never,\n Duplicates extends string = never,\n> = TEntries extends readonly [infer Head, ...infer Tail]\n ? InferScorerName<Head> extends infer ScorerName\n ? ScorerName extends string\n ? IsBroadString<ScorerName> extends true\n ? DuplicateScorerNames<Tail extends readonly unknown[] ? Tail : never, Seen, Duplicates>\n : ScorerName extends Seen\n ? DuplicateScorerNames<\n Tail extends readonly unknown[] ? Tail : never,\n Seen,\n Duplicates \| ScorerName\n >\n : DuplicateScorerNames<\n Tail extends readonly unknown[] ? Tail : never,\n Seen \| ScorerName,\n Duplicates\n >\n : DuplicateScorerNames<Tail extends readonly unknown[] ? Tail : never, Seen, Duplicates>\n : never\n : Duplicates;\n\ntype EnsureUniqueScorerNames<TEntries extends readonly unknown[]> = [\n DuplicateScorerNames<TEntries>,\n] extends [never]\n ? TEntries\n : never & {\n __axiomDuplicateScorerNames__: DuplicateScorerNames<TEntries>;\n };\n\ntype InferOnlineEvalResult<TScorers extends readonly unknown[]> = ScorerResult<\n InferScorerMetadata<TScorers[number]>\n>;\n\ntype InferOnlineEvalResultRecord<TScorers extends readonly unknown[]> = Partial<\n Record<Extract<InferScorerName<TScorers[number]>, string>, InferOnlineEvalResult<TScorers>>\n>;\n\ntype NormalizedScorerEntry<TInput, TOutput> = {\n name: string;\n scorer: ScorerInput<TInput, TOutput>;\n sampling?: ScorerSampling<TInput, TOutput>;\n};\n\n/*\n Metadata for categorizing online evaluations.\n /\nexport type OnlineEvalMeta = {\n /* High-level capability being evaluated (e.g., 'qa', 'summarization') /\n capability: string;\n /* Specific step within the capability (e.g., 'answer', 'extract') /\n step?: string;\n /\n Explicit SpanContext to link the eval span to the originating generation span.\n * When omitted, the active span's context is used automatically.\n * Use this for deferred evaluation when onlineEval is called after the\n * originating span has completed.\n /\n link?: SpanContext;\n};\n\n/\n Options for online evaluation.\n /\nexport type OnlineEvalOptions<\n TInput,\n TOutput,\n TScorers extends readonly ScorerEntry<TInput, TOutput>[] = readonly ScorerEntry<\n TInput,\n TOutput\n >[],\n> = {\n /* Input to pass to scorers (optional - only needed for input+output scorers) /\n input?: TInput;\n /* Output to evaluate /\n output: TOutput;\n /* Scorers or precomputed scores. Supports optional per-scorer sampling. /\n scorers: EnsureUniqueScorerNames<TScorers>;\n};\n\n/\n Determines if an individual scorer should run based on sampling configuration.\n /\nasync function shouldSample<TInput, TOutput>(\n sampling: ScorerSampling<TInput, TOutput> \| undefined,\n args: { input?: TInput; output: TOutput },\n): Promise<boolean> {\n if (sampling === undefined) return true;\n if (typeof sampling === 'number') {\n if (sampling >= 1) return true;\n if (sampling <= 0) return false;\n return Math.random() < sampling;\n }\n return Boolean(await sampling(args));\n}\n\nfunction isSampledScorerEntry<TInput, TOutput>(\n entry: ScorerEntry<TInput, TOutput>,\n): entry is SampledOnlineEvalScorer<TInput, TOutput, any> {\n return typeof entry === 'object' && entry !== null && 'scorer' in entry;\n}\n\nfunction resolveScorerName<TInput, TOutput>(scorer: ScorerInput<TInput, TOutput>): string {\n if (typeof scorer === 'function') {\n return scorer.name \|\| 'unknown';\n }\n return scorer.name;\n}\n\nfunction normalizeScorerEntry<TInput, TOutput>(\n entry: ScorerEntry<TInput, TOutput>,\n): NormalizedScorerEntry<TInput, TOutput> {\n if (isSampledScorerEntry(entry)) {\n return {\n name: resolveScorerName(entry.scorer),\n scorer: entry.scorer,\n sampling: entry.sampling,\n };\n }\n\n return {\n name: resolveScorerName(entry),\n scorer: entry,\n };\n}\n\nfunction getDuplicateScorerNames<TInput, TOutput>(\n entries: readonly NormalizedScorerEntry<TInput, TOutput>[],\n): string[] {\n const nameCounts = new Map<string, number>();\n\n for (const entry of entries) {\n nameCounts.set(entry.name, (nameCounts.get(entry.name) ?? 0) + 1);\n }\n\n return [...nameCounts.entries()]\n .filter(([, count]) => count > 1)\n .map(([name]) => name)\n .sort();\n}\n\n/\n Run online evaluation scorers against production outputs.\n \n Returns a promise that resolves to scorer results. Use `void onlineEval(...)`\n * for fire-and-forget, or `await onlineEval(...)` when you need to wait for\n * completion (e.g., before flushing telemetry in short-lived processes).\n \n Each eval span links back to the originating generation span via an\n * OpenTelemetry span link. Parent/child hierarchy follows natural context\n * propagation — inside `withSpan` the eval is a child, outside it depends\n * on the active context.\n \n ## Usage Patterns\n \n Inside withSpan (recommended):\n * Active span is automatically detected and linked.\n * ```ts\n * await withSpan({ capability: 'qa', step: 'answer' }, async () => {\n * const response = await generateText({ ... });\n * void onlineEval(\n * { capability: 'qa', step: 'answer' },\n * { output: response.text, scorers: [formatScorer] }\n * );\n * return response.text;\n * });\n * ```\n \n Deferred evaluation with explicit link:\n * Pass the originating span's context for linking when evaluating later.\n * ```ts\n * let spanCtx: SpanContext;\n * const result = await withSpan({ ... }, async (span) => {\n * spanCtx = span.spanContext();\n * return await generateText({ ... });\n * });\n * void onlineEval({ ..., link: spanCtx }, { output: result, scorers });\n * ```\n \n Awaiting for flush (short-lived processes):\n * ```ts\n * await onlineEval({ ... }, { output, scorers });\n * await flushTelemetry();\n * ```\n \n @param meta - Evaluation metadata for categorization\n * @param meta.capability - High-level capability being evaluated\n * @param meta.step - Optional step within the capability\n * @param meta.link - Optional SpanContext to link to (auto-detected if omitted)\n * @param options - Evaluation configuration\n * @param options.input - Input to pass to scorers\n * @param options.output - Output to evaluate\n * @param options.scorers - Scorer entries with optional per-scorer sampling\n * @returns Promise resolving to scorer results keyed by scorer name\n /\nexport function onlineEval<\n TInput,\n TOutput,\n const TScorers extends readonly ScorerEntry<TInput, TOutput>[],\n>(\n meta: OnlineEvalMeta,\n options: OnlineEvalOptions<TInput, TOutput, TScorers>,\n): Promise<InferOnlineEvalResultRecord<TScorers>> {\n if (options.scorers.length === 0) {\n return Promise.resolve({});\n }\n\n const linkSpanContext = meta.link ?? trace.getSpan(context.active())?.spanContext();\n\n return executeOnlineEvalInternal(meta, options, linkSpanContext);\n}\n\nasync function executeOnlineEvalInternal<\n TInput,\n TOutput,\n const TScorers extends readonly ScorerEntry<TInput, TOutput>[],\n>(\n meta: OnlineEvalMeta,\n options: OnlineEvalOptions<TInput, TOutput, TScorers>,\n linkSpanContext: SpanContext \| undefined,\n): Promise<InferOnlineEvalResultRecord<TScorers>> {\n const tracer = getGlobalTracer();\n\n const spanName = meta.step ? `eval ${meta.capability}/${meta.step}` : `eval ${meta.capability}`;\n\n const evalSpan = tracer.startSpan(\n spanName,\n linkSpanContext ? { links: [{ context: linkSpanContext }] } : {},\n );\n\n const evalAttrs: Record<string, string> = {\n [Attr.GenAI.Operation.Name]: 'eval',\n [Attr.Eval.Capability.Name]: meta.capability,\n [Attr.Eval.Tags]: JSON.stringify(['online']),\n };\n if (meta.step) {\n evalAttrs[Attr.Eval.Step.Name] = meta.step;\n }\n evalSpan.setAttributes(evalAttrs);\n\n try {\n const normalizedScorers = options.scorers.map((entry) => normalizeScorerEntry(entry));\n const duplicateScorerNames = getDuplicateScorerNames(normalizedScorers);\n if (duplicateScorerNames.length > 0) {\n throw new Error(\n `Duplicate scorer names are not allowed: ${duplicateScorerNames\n .map((name) => `\"${name}\"`)\n .join(', ')}`,\n );\n }\n\n const outcomes = await Promise.all(\n normalizedScorers.map(async (entry) => {\n try {\n const sampledIn = await shouldSample(entry.sampling, {\n input: options.input,\n output: options.output,\n });\n\n if (!sampledIn) {\n return { sampledOut: true as const };\n }\n\n return {\n sampledOut: false as const,\n result: await executeScorer(\n entry.scorer,\n options.input,\n options.output,\n evalSpan,\n meta,\n ),\n };\n } catch (err) {\n const error = err instanceof Error ? err : new Error(String(err));\n return {\n sampledOut: false as const,\n result: await executeScorer(\n {\n name: entry.name,\n score: null,\n error: error.message,\n },\n options.input,\n options.output,\n evalSpan,\n meta,\n ),\n };\n }\n }),\n );\n\n const results: Record<string, InferOnlineEvalResult<TScorers>> = {};\n let sampledOutCount = 0;\n\n for (const outcome of outcomes) {\n if (outcome.sampledOut) {\n sampledOutCount += 1;\n continue;\n }\n\n results[outcome.result.name] = outcome.result as InferOnlineEvalResult<TScorers>;\n }\n\n const failedCount = Object.values(results).filter((result) => result.error).length;\n const ranCount = Object.keys(results).length;\n\n evalSpan.setAttributes({\n 'axiom.eval.online.scorers.total': normalizedScorers.length,\n 'axiom.eval.online.scorers.ran': ranCount,\n 'axiom.eval.online.scorers.sampled_out': sampledOutCount,\n 'axiom.eval.online.scorers.failed': failedCount,\n });\n\n if (failedCount > 0) {\n evalSpan.setStatus({\n code: SpanStatusCode.ERROR,\n message: 'One or more scorers failed',\n });\n } else {\n evalSpan.setStatus({ code: SpanStatusCode.OK });\n }\n\n return results as InferOnlineEvalResultRecord<TScorers>;\n } catch (err) {\n const error = err instanceof Error ? err : new Error(String(err));\n evalSpan.recordException(error);\n evalSpan.setStatus({\n code: SpanStatusCode.ERROR,\n message: error.message,\n });\n return {};\n } finally {\n evalSpan.end();\n }\n}\n","import type { Tracer } from '@opentelemetry/api';\nimport { trace } from '@opentelemetry/api';\nimport packageJson from '../../package.json';\nimport { AXIOM_AI_REDACTION_KEY, type AxiomAIRedactionPolicy } from './utils/redaction';\n\n// Global key to store tracer scope information across all execution contexts\nconst AXIOM_AI_SCOPE_KEY = Symbol.for('__axiom_ai_scope__');\n\ninterface TracerScope {\n name: string;\n version?: string;\n}\n\n/\n Extract scope information from a tracer, with fallback to package.json\n /\nfunction extractTracerScope(tracer: Tracer): TracerScope {\n const tracerAny = tracer as any;\n\n // Guard access to private fields with optional chaining\n // Note: These are internal OTEL fields that may change in future versions\n // _instrumentationScope is modern, instrumentationLibrary is legacy (<1.16)\n const name =\n tracerAny._instrumentationScope?.name \|\|\n tracerAny.instrumentationLibrary?.name \|\|\n packageJson.name;\n\n const version =\n tracerAny._instrumentationScope?.version \|\|\n tracerAny.instrumentationLibrary?.version \|\|\n packageJson.version;\n\n return { name, version };\n}\n\n/\n Register this in your `instrumentation.ts` to set up axiom.\n * This function stores the tracer's scope information globally to enable Context Propagation\n * and Instrumentation Scope. The tracer will be available across all execution contexts including Next.js.\n \n This function is idempotent - calling it multiple times with the same scope has no additional cost.\n \n @param config\n * @param config.tracer - The tracer that you are using in your application.\n * @param config.redactionPolicy - Optional redaction policy to control what data is captured in spans.\n /\nexport function initAxiomAI(config: { tracer: Tracer; redactionPolicy?: AxiomAIRedactionPolicy }) {\n const newScope = extractTracerScope(config.tracer);\n const existingScope = (globalThis as any)[AXIOM_AI_SCOPE_KEY] as TracerScope \| undefined;\n\n // Check if already initialized with same scope (idempotent behavior)\n if (\n existingScope &&\n existingScope.name === newScope.name &&\n existingScope.version === newScope.version\n ) {\n return;\n }\n\n // Warn about double initialization with different scopes\n if (existingScope) {\n console.warn(\n '[AxiomAI] initAxiomAI() called multiple times with different scopes. ' +\n `Previous: ${existingScope.name}@${existingScope.version}, ` +\n `New: ${newScope.name}@${newScope.version}`,\n );\n }\n\n // Store scope information globally (works across Next.js contexts)\n (globalThis as any)[AXIOM_AI_SCOPE_KEY] = newScope;\n\n // Store redaction policy globally if provided\n if (config.redactionPolicy) {\n (globalThis as any)[AXIOM_AI_REDACTION_KEY] = config.redactionPolicy;\n }\n}\n\n/\n Get a tracer using the globally stored scope information\n * Fall back to package.json defaults if not set\n /\nexport function getGlobalTracer(): Tracer {\n // Get stored scope information or fall back to package defaults\n const scope = (globalThis as any)[AXIOM_AI_SCOPE_KEY] as TracerScope \| undefined;\n\n // Warn if initAxiomAI was never called\n if (!scope) {\n const isDebug = process.env.AXIOM_DEBUG === 'true';\n if (!isDebug) {\n console.warn(\n '[AxiomAI] AXIOM_AI_SCOPE_KEY is undefined. This probably means that ' +\n 'initAxiomAI() was never called. ' +\n 'Make sure to call initAxiomAI({ tracer }) in your instrumentation setup.',\n );\n }\n }\n\n let { name, version } = scope \|\| { name: packageJson.name, version: packageJson.version };\n\n if (!name \|\| !version) {\n name = packageJson.name;\n version = packageJson.version;\n if (!name \|\| !version) {\n name = 'axiom';\n version = 'unknown';\n }\n }\n\n // Use OpenTelemetry's standard global provider mechanism\n return trace.getTracer(name, version);\n}\n\n/\n Reset AxiomAI configuration (useful for testing)\n /\nexport function resetAxiomAI() {\n (globalThis as any)[AXIOM_AI_SCOPE_KEY] = undefined;\n (globalThis as any)[AXIOM_AI_REDACTION_KEY] = undefined;\n}\n","{\n \"name\": \"axiom\",\n \"version\": \"0.44.0\",\n \"type\": \"module\",\n \"author\": \"Axiom, Inc.\",\n \"contributors\": [\n \"Islam Shehata <islam@axiom.co>\",\n \"Chris Ehrlich <chris@axiom.co>\",\n \"Gabriel de Andrade <gabriel@axiom.co>\"\n ],\n \"scripts\": {\n \"dev\": \"tsup --watch\",\n \"build\": \"tsup && chmod +x dist/bin.js && pnpm check:vitest-entrypoints\",\n \"check:vitest-entrypoints\": \"node ./scripts/check-vitest-entrypoints.mjs\",\n \"format\": \"prettier --write .\",\n \"format:check\": \"prettier --check .\",\n \"lint\": \"eslint './/.{js,ts}'\",\n \"typecheck\": \"tsc --noEmit\",\n \"test\": \"vitest run\",\n \"test:watch\": \"vitest --watch\",\n \"publint\": \"npx publint\"\n },\n \"types\": \"./dist/index.d.ts\",\n \"main\": \"./dist/index.cjs\",\n \"module\": \"./dist/index.js\",\n \"bin\": {\n \"axiom\": \"./dist/bin.js\"\n },\n \"exports\": {\n \"./ai\": {\n \"import\": {\n \"types\": \"./dist/index.d.ts\",\n \"default\": \"./dist/index.js\"\n },\n \"require\": {\n \"types\": \"./dist/index.d.cts\",\n \"default\": \"./dist/index.cjs\"\n }\n },\n \"./ai/evals\": {\n \"import\": {\n \"types\": \"./dist/evals.d.ts\",\n \"default\": \"./dist/evals.js\"\n },\n \"require\": {\n \"types\": \"./dist/evals.d.cts\",\n \"default\": \"./dist/evals.cjs\"\n }\n },\n \"./ai/evals/aggregations\": {\n \"import\": {\n \"types\": \"./dist/evals/aggregations.d.ts\",\n \"default\": \"./dist/evals/aggregations.js\"\n },\n \"require\": {\n \"types\": \"./dist/evals/aggregations.d.cts\",\n \"default\": \"./dist/evals/aggregations.cjs\"\n }\n },\n \"./ai/evals/scorers\": {\n \"import\": {\n \"types\": \"./dist/evals/scorers.d.ts\",\n \"default\": \"./dist/evals/scorers.js\"\n },\n \"require\": {\n \"types\": \"./dist/evals/scorers.d.cts\",\n \"default\": \"./dist/evals/scorers.cjs\"\n }\n },\n \"./ai/evals/online\": {\n \"import\": {\n \"types\": \"./dist/evals/online.d.ts\",\n \"default\": \"./dist/evals/online.js\"\n },\n \"require\": {\n \"types\": \"./dist/evals/online.d.cts\",\n \"default\": \"./dist/evals/online.cjs\"\n }\n },\n \"./ai/config\": {\n \"import\": {\n \"types\": \"./dist/config.d.ts\",\n \"default\": \"./dist/config.js\"\n },\n \"require\": {\n \"types\": \"./dist/config.d.cts\",\n \"default\": \"./dist/config.cjs\"\n }\n },\n \"./ai/feedback\": {\n \"import\": {\n \"types\": \"./dist/feedback.d.ts\",\n \"default\": \"./dist/feedback.js\"\n },\n \"require\": {\n \"types\": \"./dist/feedback.d.cts\",\n \"default\": \"./dist/feedback.cjs\"\n }\n }\n },\n \"keywords\": [\n \"axiom\",\n \"logging\",\n \"ai\",\n \"otel\",\n \"opentelemetry\"\n ],\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/axiomhq/ai.git\",\n \"directory\": \"packages/ai\"\n },\n \"license\": \"MIT\",\n \"dependencies\": {\n \"@next/env\": \"^15.4.2\",\n \"@opentelemetry/auto-instrumentations-node\": \"^0.60.1\",\n \"@opentelemetry/context-async-hooks\": \"^2.0.1\",\n \"@opentelemetry/exporter-trace-otlp-http\": \"^0.202.0\",\n \"@opentelemetry/resources\": \"^2.0.1\",\n \"@opentelemetry/sdk-trace-node\": \"^2.0.1\",\n \"@opentelemetry/semantic-conventions\": \"^1.38.0\",\n \"@sinclair/typebox\": \"^0.34.37\",\n \"c12\": \"^3.3.3\",\n \"commander\": \"^14.0.0\",\n \"defu\": \"^6.1.4\",\n \"handlebars\": \"^4.7.8\",\n \"nanoid\": \"^5.1.5\",\n \"open\": \"^10.1.0\",\n \"vite-tsconfig-paths\": \"^5.1.4\",\n \"vitest\": \"^4.0.0\"\n },\n \"peerDependencies\": {\n \"@opentelemetry/api\": \"^1.9.0\",\n \"zod\": \"^3.25.0 \|\| ^4.0.0\"\n },\n \"devDependencies\": {\n \"@ai-sdk/anthropicv1\": \"npm:@ai-sdk/anthropic@^1.2.12\",\n \"@ai-sdk/anthropicv2\": \"npm:@ai-sdk/anthropic@^2.0.57\",\n \"@ai-sdk/anthropicv3\": \"npm:@ai-sdk/anthropic@^3.0.9\",\n \"@ai-sdk/openaiv1\": \"npm:@ai-sdk/openai@^1.3.24\",\n \"@ai-sdk/openaiv2\": \"npm:@ai-sdk/openai@^2.0.88\",\n \"@ai-sdk/openaiv3\": \"npm:@ai-sdk/openai@^3.0.7\",\n \"@ai-sdk/providerv1\": \"npm:@ai-sdk/provider@^1.1.3\",\n \"@ai-sdk/providerv2\": \"npm:@ai-sdk/provider@^2.0.1\",\n \"@ai-sdk/providerv3\": \"npm:@ai-sdk/provider@^3.0.2\",\n \"@opentelemetry/api\": \"^1.9.0\",\n \"@opentelemetry/core\": \"^2.0.1\",\n \"@opentelemetry/sdk-trace-base\": \"^2.0.1\",\n \"@opentelemetry/sdk-trace-node\": \"^2.0.1\",\n \"@repo/eslint-config\": \"workspace:\",\n \"@types/node\": \"^22.15.29\",\n \"@vitest/coverage-v8\": \"^4.0.0\",\n \"aiv4\": \"npm:ai@^4.3.19\",\n \"aiv5\": \"npm:ai@^5.0.118\",\n \"aiv6\": \"npm:ai@^6.0.6\",\n \"esbuild\": \"^0.25.8\",\n \"eslint\": \"catalog:\",\n \"msw\": \"^2.12.2\",\n \"prettier\": \"catalog:\",\n \"tinyrainbow\": \"^2.0.0\",\n \"tsup\": \"catalog:\",\n \"typescript\": \"catalog:\",\n \"vitest\": \"catalog:\",\n \"zod\": \"catalog:\"\n },\n \"files\": [\n \"dist\"\n ],\n \"packageManager\": \"pnpm@10.16.1\"\n}\n","import { context, propagation, type Baggage, type Span } from '@opentelemetry/api';\nimport { WITHSPAN_REDACTION_POLICY_KEY } from '../withSpanBaggageKey';\n\ntype CaptureMessageContent = 'full' \| 'off';\n\nexport type AxiomAIRedactionPolicy = {\n captureMessageContent?: CaptureMessageContent;\n mirrorToolPayloadOnToolSpan?: boolean;\n};\n\nexport const RedactionPolicy = {\n /\n Includes message content on chat spans, and mirrors tool\n * payload on tool spans for more convenient querying.\n /\n AxiomDefault: {\n captureMessageContent: 'full',\n mirrorToolPayloadOnToolSpan: true,\n },\n /\n Redacts message content on chat spans, and does not put\n * tool payload on tool spans.\n /\n OpenTelemetryDefault: {\n captureMessageContent: 'off',\n mirrorToolPayloadOnToolSpan: false,\n },\n} as const satisfies Record<string, AxiomAIRedactionPolicy>;\n\n// Global key to store redaction policy across all execution contexts\nexport const AXIOM_AI_REDACTION_KEY = Symbol.for('__axiom_ai_redaction__');\n\n/\n Get the globally stored redaction policy\n * @returns The global redaction policy or undefined if not set\n /\nfunction getGlobalRedactionPolicy(): AxiomAIRedactionPolicy \| undefined {\n return (globalThis as any)[AXIOM_AI_REDACTION_KEY];\n}\n\n/\n Resolves the effective redaction policy by merging global and per-call policies.\n * Per-call policies take precedence over global policies.\n \n @param globalPolicy - The global redaction policy from initAxiomAI\n * @param localPolicy - The per-call redaction policy from withSpan or middleware\n * @returns The effective redaction policy with all fields resolved\n /\nfunction getEffectiveRedactionPolicy(\n globalPolicy?: AxiomAIRedactionPolicy,\n localPolicy?: AxiomAIRedactionPolicy,\n): AxiomAIRedactionPolicy {\n // Per-call policy overrides global policy, with defaults\n return {\n captureMessageContent:\n localPolicy?.captureMessageContent ??\n globalPolicy?.captureMessageContent ??\n RedactionPolicy.AxiomDefault.captureMessageContent,\n mirrorToolPayloadOnToolSpan:\n localPolicy?.mirrorToolPayloadOnToolSpan ??\n globalPolicy?.mirrorToolPayloadOnToolSpan ??\n RedactionPolicy.AxiomDefault.mirrorToolPayloadOnToolSpan,\n };\n}\n\n/\n Get the active redaction policy\n /\nexport function getRedactionPolicy() {\n return getEffectiveRedactionPolicy(getGlobalRedactionPolicy(), getPerCallRedactionPolicy());\n}\n\n/\n Conditionally sets a span attribute based on the capture message content policy.\n * When policy is 'off', the attribute is not set at all.\n \n @param span - The OpenTelemetry span to set the attribute on\n * @param attribute - The attribute name/key\n * @param value - The attribute value\n * @param captureMessageContent - The capture policy ('full' sets attribute, 'off' skips it)\n /\nexport function handleMaybeRedactedAttribute(\n span: Span,\n attribute: string,\n value: any,\n captureMessageContent?: CaptureMessageContent,\n): void {\n if (captureMessageContent === 'full') {\n span.setAttribute(attribute, value);\n }\n\n // For 'off', don't set the attribute at all\n // Future: Could add callback-based redaction here\n}\n\n/\n Extracts the per-call redaction policy from OpenTelemetry baggage context.\n * This allows middleware and tools to access redaction policies passed via withSpan.\n \n @returns The per-call redaction policy or undefined if not set\n /\nfunction getPerCallRedactionPolicy(): AxiomAIRedactionPolicy \| undefined {\n const baggage: Baggage = propagation.getBaggage(context.active()) \|\| propagation.createBaggage();\n const serializedPolicy = baggage.getEntry(WITHSPAN_REDACTION_POLICY_KEY)?.value;\n\n if (!serializedPolicy) {\n return undefined;\n }\n\n try {\n return JSON.parse(serializedPolicy) as AxiomAIRedactionPolicy;\n } catch (error) {\n console.warn('[AxiomAI] Failed to parse redaction policy from baggage:', error);\n return undefined;\n }\n}\n","import { context, trace, SpanStatusCode, type Span } from '@opentelemetry/api';\nimport { getGlobalTracer } from '../otel/initAxiomAI';\nimport type { Scorer, ScorerResult } from './types';\nimport type { OnlineEvalMeta } from './onlineEval';\nimport { Attr } from '../otel/semconv/attributes';\n\ntype OnlineEvalScorerInput<TInput, TOutput> = Scorer<TInput, TOutput, any> \| ScorerResult<any>;\n\nfunction setScorerSpanAttrs(\n scorerSpan: Span,\n scorerName: string,\n result: Pick<ScorerResult<any>, 'score' \| 'metadata'>,\n meta?: OnlineEvalMeta,\n): void {\n const attrs: Record<string, string \| number \| boolean \| undefined> = {\n [Attr.GenAI.Operation.Name]: 'eval.score',\n [Attr.Eval.Score.Name]: scorerName,\n [Attr.Eval.Tags]: JSON.stringify(['online']),\n [Attr.Eval.Score.Value]: result.score ?? undefined,\n [Attr.Eval.Capability.Name]: meta?.capability,\n [Attr.Eval.Step.Name]: meta?.step,\n };\n\n if (result.metadata && Object.keys(result.metadata).length > 0) {\n attrs[Attr.Eval.Score.Metadata] = JSON.stringify(result.metadata);\n }\n\n scorerSpan.setAttributes(attrs);\n}\n\n/\n Executes a single scorer or emits a precomputed scorer result.\n /\nexport async function executeScorer<TInput, TOutput>(\n scorer: OnlineEvalScorerInput<TInput, TOutput>,\n input: TInput \| undefined,\n output: TOutput,\n parentSpan: Span,\n meta?: OnlineEvalMeta,\n): Promise<ScorerResult<any>> {\n const tracer = getGlobalTracer();\n const parentContext = trace.setSpan(context.active(), parentSpan);\n\n return context.with(parentContext, async () => {\n const scorerName =\n typeof scorer === 'function'\n ? // undefined/unknown case shouldn't happen, but better safe than sorry\n scorer.name \|\| 'unknown'\n : scorer.name;\n const scorerSpan = tracer.startSpan(`score ${scorerName}`);\n\n try {\n const result =\n typeof scorer === 'function'\n ? ({\n ...(await scorer({\n input,\n output,\n })),\n name: scorerName,\n } satisfies ScorerResult)\n : scorer;\n\n setScorerSpanAttrs(scorerSpan, scorerName, result, meta);\n if (result.error) {\n const error = new Error(result.error);\n scorerSpan.recordException(error);\n scorerSpan.setAttributes({\n [Attr.Error.Message]: error.message,\n [Attr.Error.Type]: error.name,\n });\n scorerSpan.setStatus({\n code: SpanStatusCode.ERROR,\n message: error.message,\n });\n } else {\n scorerSpan.setStatus({ code: SpanStatusCode.OK });\n }\n\n return result;\n } catch (err) {\n const error = err instanceof Error ? err : new Error(String(err));\n const failedResult: ScorerResult = {\n name: scorerName,\n score: null,\n error: error.message,\n };\n\n setScorerSpanAttrs(scorerSpan, scorerName, failedResult, meta);\n\n scorerSpan.recordException(error);\n scorerSpan.setAttributes({\n [Attr.Error.Message]: error.message,\n [Attr.Error.Type]: error.name,\n });\n scorerSpan.setStatus({\n code: SpanStatusCode.ERROR,\n message: error.message,\n });\n\n return failedResult;\n } finally {\n scorerSpan.end();\n }\n });\n}\n","import {\n ATTR_ERROR_TYPE,\n ATTR_HTTP_RESPONSE_STATUS_CODE,\n} from '@opentelemetry/semantic-conventions';\n\nimport {\n ATTR_EVAL_CASE_INDEX,\n ATTR_EVAL_CASE_EXPECTED,\n ATTR_EVAL_CASE_INPUT,\n ATTR_EVAL_CASE_OUTPUT,\n ATTR_EVAL_CASE_METADATA,\n ATTR_EVAL_CASE_SCORES,\n ATTR_EVAL_CASE_TRIALS,\n ATTR_EVAL_SCORE_PASSED,\n ATTR_EVAL_SCORE_IS_BOOLEAN,\n ATTR_EVAL_SCORE_VALUE,\n ATTR_EVAL_SCORE_NAME,\n ATTR_EVAL_SCORE_THRESHOLD,\n ATTR_EVAL_SCORE_METADATA,\n ATTR_EVAL_SCORE_AGGREGATION,\n ATTR_EVAL_SCORE_TRIALS,\n ATTR_EVAL_TASK_TYPE,\n ATTR_EVAL_TASK_NAME,\n ATTR_EVAL_TASK_OUTPUT,\n ATTR_EVAL_COLLECTION_NAME,\n ATTR_EVAL_COLLECTION_SIZE,\n ATTR_EVAL_ID,\n ATTR_EVAL_BASELINE_ID,\n ATTR_EVAL_BASELINE_NAME,\n ATTR_EVAL_BASELINE_VERSION,\n ATTR_EVAL_NAME,\n ATTR_EVAL_TAGS,\n ATTR_EVAL_TYPE,\n ATTR_EVAL_TRIALS,\n ATTR_EVAL_TRIAL_INDEX,\n ATTR_EVAL_TRIAL_ERROR,\n ATTR_EVAL_COLLECTION_ID,\n ATTR_EVAL_USER_NAME,\n ATTR_EVAL_USER_EMAIL,\n ATTR_EVAL_VERSION,\n ATTR_EVAL_RUN_ID,\n ATTR_EVAL_METADATA,\n ATTR_EVAL_CONFIG_FLAGS,\n ATTR_EVAL_CAPABILITY_NAME,\n ATTR_EVAL_STEP_NAME,\n} from './eval_proposal';\n\nimport {\n ATTR_ERROR_MESSAGE,\n ATTR_GEN_AI_AGENT_DESCRIPTION,\n ATTR_GEN_AI_AGENT_ID,\n ATTR_GEN_AI_AGENT_NAME,\n ATTR_GEN_AI_CONVERSATION_ID,\n ATTR_GEN_AI_INPUT_MESSAGES,\n ATTR_GEN_AI_OPERATION_NAME,\n ATTR_GEN_AI_OUTPUT_MESSAGES,\n ATTR_GEN_AI_OUTPUT_TYPE,\n ATTR_GEN_AI_PROVIDER_NAME,\n ATTR_GEN_AI_REQUEST_CHOICE_COUNT,\n ATTR_GEN_AI_REQUEST_ENCODING_FORMATS,\n ATTR_GEN_AI_REQUEST_FREQUENCY_PENALTY,\n ATTR_GEN_AI_REQUEST_MAX_TOKENS,\n ATTR_GEN_AI_REQUEST_MODEL,\n ATTR_GEN_AI_REQUEST_PRESENCE_PENALTY,\n ATTR_GEN_AI_REQUEST_SEED,\n ATTR_GEN_AI_REQUEST_STOP_SEQUENCES,\n ATTR_GEN_AI_REQUEST_TEMPERATURE,\n ATTR_GEN_AI_REQUEST_TOP_K,\n ATTR_GEN_AI_REQUEST_TOP_P,\n ATTR_GEN_AI_RESPONSE_FINISH_REASONS,\n ATTR_GEN_AI_RESPONSE_ID,\n ATTR_GEN_AI_RESPONSE_MODEL,\n ATTR_GEN_AI_TOOL_CALL_ID,\n ATTR_GEN_AI_TOOL_DESCRIPTION,\n ATTR_GEN_AI_TOOL_NAME,\n ATTR_GEN_AI_TOOL_TYPE,\n ATTR_GEN_AI_USAGE_INPUT_TOKENS,\n ATTR_GEN_AI_USAGE_OUTPUT_TOKENS,\n GEN_AI_OPERATION_NAME_VALUE_CHAT,\n GEN_AI_OPERATION_NAME_VALUE_CREATE_AGENT,\n GEN_AI_OPERATION_NAME_VALUE_EMBEDDINGS,\n GEN_AI_OPERATION_NAME_VALUE_EXECUTE_TOOL,\n GEN_AI_OPERATION_NAME_VALUE_GENERATE_CONTENT,\n GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT,\n GEN_AI_OUTPUT_TYPE_VALUE_IMAGE,\n GEN_AI_OUTPUT_TYPE_VALUE_JSON,\n GEN_AI_OUTPUT_TYPE_VALUE_SPEECH,\n GEN_AI_OUTPUT_TYPE_VALUE_TEXT,\n GEN_AI_PROVIDER_NAME_VALUE_ANTHROPIC,\n GEN_AI_PROVIDER_NAME_VALUE_AWS_BEDROCK,\n GEN_AI_PROVIDER_NAME_VALUE_AZURE_AI_INFERENCE,\n GEN_AI_PROVIDER_NAME_VALUE_AZURE_AI_OPENAI,\n GEN_AI_PROVIDER_NAME_VALUE_COHERE,\n GEN_AI_PROVIDER_NAME_VALUE_DEEPSEEK,\n GEN_AI_PROVIDER_NAME_VALUE_GCP_GEMINI,\n GEN_AI_PROVIDER_NAME_VALUE_GCP_GEN_AI,\n GEN_AI_PROVIDER_NAME_VALUE_GCP_VERTEX_AI,\n GEN_AI_PROVIDER_NAME_VALUE_GROQ,\n GEN_AI_PROVIDER_NAME_VALUE_IBM_WATSONX_AI,\n GEN_AI_PROVIDER_NAME_VALUE_MISTRAL_AI,\n GEN_AI_PROVIDER_NAME_VALUE_OPENAI,\n GEN_AI_PROVIDER_NAME_VALUE_PERPLEXITY,\n GEN_AI_PROVIDER_NAME_VALUE_X_AI,\n} from '@opentelemetry/semantic-conventions/incubating';\n\n/\n PROPRIETARY ATTRIBUTES (o11y)\n \n @see: https://axiom.co/docs/ai-engineering/semantic-conventions\n /\n\nconst ATTR_AXIOM_GEN_AI_SCHEMA_URL = 'axiom.gen_ai.schema_url';\nconst ATTR_AXIOM_GEN_AI_SDK_NAME = 'axiom.gen_ai.sdk.name';\nconst ATTR_AXIOM_GEN_AI_SDK_VERSION = 'axiom.gen_ai.sdk.version';\nconst ATTR_GEN_AI_CAPABILITY_NAME = 'gen_ai.capability.name';\nconst ATTR_GEN_AI_STEP_NAME = 'gen_ai.step.name';\nconst ATTR_GEN_AI_TOOL_ARGUMENTS = 'gen_ai.tool.arguments'; // deprecated by OTel\nconst ATTR_GEN_AI_TOOL_MESSAGE = 'gen_ai.tool.message'; // deprecated by OTel\n\nconst GEN_AI_PROVIDER_NAME_VALUE_ASSEMBLYAI = 'assemblyai';\nconst GEN_AI_PROVIDER_NAME_VALUE_CEREBRAS = 'cerebras';\nconst GEN_AI_PROVIDER_NAME_VALUE_DEEPGRAM = 'deepgram';\nconst GEN_AI_PROVIDER_NAME_VALUE_DEEPINFRA = 'deepinfra';\nconst GEN_AI_PROVIDER_NAME_VALUE_ELEVENLABS = 'elevenlabs';\nconst GEN_AI_PROVIDER_NAME_VALUE_FAL = 'fal';\nconst GEN_AI_PROVIDER_NAME_VALUE_FIREWORKS = 'fireworks';\nconst GEN_AI_PROVIDER_NAME_VALUE_GLADIA = 'gladia';\nconst GEN_AI_PROVIDER_NAME_VALUE_HUME = 'hume';\nconst GEN_AI_PROVIDER_NAME_VALUE_LMNT = 'lmnt';\nconst GEN_AI_PROVIDER_NAME_VALUE_LUMA = 'luma';\nconst GEN_AI_PROVIDER_NAME_VALUE_REPLICATE = 'replicate';\nconst GEN_AI_PROVIDER_NAME_VALUE_REVAI = 'revai';\nconst GEN_AI_PROVIDER_NAME_VALUE_TOGETHERAI = 'togetherai';\nconst GEN_AI_PROVIDER_NAME_VALUE_VERCEL = 'vercel';\n\n/\n When adding something new here, please:\n * 1. Make sure it doesn't already exist as part of OTel Semantic Conventions (use that instead)\n * 2. Make sure to use standard naming schema, ie snake_case\n * 3. If a specific feature has an attribute you would like to use, extract it to the shared section\n \n Also Experimental Attributes should always be imported here and then used from the CustomAttributes object\n * because they are unstable.\n \n @see: https://github.com/open-telemetry/opentelemetry-js/tree/c89cb38d0fec39d54cf3fcb35c429a8129e9c909/semantic-conventions#unstable-semconv\n /\nexport const Attr = {\n __EXPERIMENTAL_Flag: (flagName: string) => `flag.${flagName}`,\n __EXPERIMENTAL_Fact: (factName: string) => `fact.${factName}`,\n Axiom: {\n GenAI: {\n SchemaURL: ATTR_AXIOM_GEN_AI_SCHEMA_URL,\n SDK: {\n Name: ATTR_AXIOM_GEN_AI_SDK_NAME,\n Version: ATTR_AXIOM_GEN_AI_SDK_VERSION,\n },\n },\n },\n GenAI: {\n PromptMetadata: {\n ID: 'axiom.gen_ai.prompt.id',\n Name: 'axiom.gen_ai.prompt.name',\n Slug: 'axiom.gen_ai.prompt.slug',\n Version: 'axiom.gen_ai.prompt.version',\n },\n /\n These two are used to identify the span\n /\n Capability: {\n Name: ATTR_GEN_AI_CAPABILITY_NAME,\n },\n Step: {\n Name: ATTR_GEN_AI_STEP_NAME,\n },\n Provider: {\n Name: ATTR_GEN_AI_PROVIDER_NAME,\n Name_Values: {\n Anthropic: GEN_AI_PROVIDER_NAME_VALUE_ANTHROPIC,\n AssemblyAI: GEN_AI_PROVIDER_NAME_VALUE_ASSEMBLYAI,\n AWSBedrock: GEN_AI_PROVIDER_NAME_VALUE_AWS_BEDROCK,\n AzureAIInference: GEN_AI_PROVIDER_NAME_VALUE_AZURE_AI_INFERENCE,\n AzureAIOpenAI: GEN_AI_PROVIDER_NAME_VALUE_AZURE_AI_OPENAI,\n Cerebras: GEN_AI_PROVIDER_NAME_VALUE_CEREBRAS,\n Cohere: GEN_AI_PROVIDER_NAME_VALUE_COHERE,\n Deepgram: GEN_AI_PROVIDER_NAME_VALUE_DEEPGRAM,\n DeepInfra: GEN_AI_PROVIDER_NAME_VALUE_DEEPINFRA,\n Deepseek: GEN_AI_PROVIDER_NAME_VALUE_DEEPSEEK,\n ElevenLabs: GEN_AI_PROVIDER_NAME_VALUE_ELEVENLABS,\n Fal: GEN_AI_PROVIDER_NAME_VALUE_FAL,\n Fireworks: GEN_AI_PROVIDER_NAME_VALUE_FIREWORKS,\n GCPGemini: GEN_AI_PROVIDER_NAME_VALUE_GCP_GEMINI,\n GCPGenAI: GEN_AI_PROVIDER_NAME_VALUE_GCP_GEN_AI,\n GCPVertexAI: GEN_AI_PROVIDER_NAME_VALUE_GCP_VERTEX_AI,\n Gladia: GEN_AI_PROVIDER_NAME_VALUE_GLADIA,\n Groq: GEN_AI_PROVIDER_NAME_VALUE_GROQ,\n Hume: GEN_AI_PROVIDER_NAME_VALUE_HUME,\n IBMWatsonxAI: GEN_AI_PROVIDER_NAME_VALUE_IBM_WATSONX_AI,\n Lmnt: GEN_AI_PROVIDER_NAME_VALUE_LMNT,\n Luma: GEN_AI_PROVIDER_NAME_VALUE_LUMA,\n MistralAI: GEN_AI_PROVIDER_NAME_VALUE_MISTRAL_AI,\n OpenAI: GEN_AI_PROVIDER_NAME_VALUE_OPENAI,\n Perplexity: GEN_AI_PROVIDER_NAME_VALUE_PERPLEXITY,\n Replicate: GEN_AI_PROVIDER_NAME_VALUE_REPLICATE,\n RevAI: GEN_AI_PROVIDER_NAME_VALUE_REVAI,\n TogetherAI: GEN_AI_PROVIDER_NAME_VALUE_TOGETHERAI,\n Vercel: GEN_AI_PROVIDER_NAME_VALUE_VERCEL,\n XAI: GEN_AI_PROVIDER_NAME_VALUE_X_AI,\n },\n },\n /\n Regular attributes\n /\n Agent: {\n Description: ATTR_GEN_AI_AGENT_DESCRIPTION, // not yet used by axiom-ai\n ID: ATTR_GEN_AI_AGENT_ID, // not yet used by axiom-ai\n Name: ATTR_GEN_AI_AGENT_NAME, // not yet used by axiom-ai\n },\n Conversation: {\n ID: ATTR_GEN_AI_CONVERSATION_ID, // not yet used by axiom-ai, anyway probably needs to be provided by user\n },\n Input: {\n Messages: ATTR_GEN_AI_INPUT_MESSAGES,\n },\n Operation: {\n Name: ATTR_GEN_AI_OPERATION_NAME,\n Name_Values: {\n /\n Note that \"text_completion\" is deprecated in favor of \"chat\" for both OpenAI and Anthropic\n /\n Chat: GEN_AI_OPERATION_NAME_VALUE_CHAT,\n CreateAgent: GEN_AI_OPERATION_NAME_VALUE_CREATE_AGENT,\n Embeddings: GEN_AI_OPERATION_NAME_VALUE_EMBEDDINGS,\n ExecuteTool: GEN_AI_OPERATION_NAME_VALUE_EXECUTE_TOOL,\n GenerateContent: GEN_AI_OPERATION_NAME_VALUE_GENERATE_CONTENT,\n InvokeAgent: GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT,\n },\n },\n Output: {\n Messages: ATTR_GEN_AI_OUTPUT_MESSAGES,\n Type: ATTR_GEN_AI_OUTPUT_TYPE,\n Type_Values: {\n Text: GEN_AI_OUTPUT_TYPE_VALUE_TEXT,\n Json: GEN_AI_OUTPUT_TYPE_VALUE_JSON,\n Image: GEN_AI_OUTPUT_TYPE_VALUE_IMAGE,\n Speech: GEN_AI_OUTPUT_TYPE_VALUE_SPEECH,\n },\n },\n /\n The provider that is hosting the model, eg AWS Bedrock\n * There doesn't seem to be a semconv for this\n /\n Request: {\n ChoiceCount: ATTR_GEN_AI_REQUEST_CHOICE_COUNT, // not yet used by axiom-ai\n EncodingFormats: ATTR_GEN_AI_REQUEST_ENCODING_FORMATS, // not yet used by axiom-ai\n FrequencyPenalty: ATTR_GEN_AI_REQUEST_FREQUENCY_PENALTY,\n MaxTokens: ATTR_GEN_AI_REQUEST_MAX_TOKENS,\n /\n The model you asked for\n /\n Model: ATTR_GEN_AI_REQUEST_MODEL,\n PresencePenalty: ATTR_GEN_AI_REQUEST_PRESENCE_PENALTY,\n Seed: ATTR_GEN_AI_REQUEST_SEED,\n StopSequences: ATTR_GEN_AI_REQUEST_STOP_SEQUENCES,\n Temperature: ATTR_GEN_AI_REQUEST_TEMPERATURE,\n TopK: ATTR_GEN_AI_REQUEST_TOP_K,\n TopP: ATTR_GEN_AI_REQUEST_TOP_P,\n },\n Response: {\n FinishReasons: ATTR_GEN_AI_RESPONSE_FINISH_REASONS,\n ID: ATTR_GEN_AI_RESPONSE_ID,\n /\n The model that was actually used (might be different bc routing) - only ever get this from the response, otherwise omit\n /\n Model: ATTR_GEN_AI_RESPONSE_MODEL, // somehow not landing on the span for google models? check up on this...\n },\n Tool: {\n CallID: ATTR_GEN_AI_TOOL_CALL_ID,\n Description: ATTR_GEN_AI_TOOL_DESCRIPTION,\n Name: ATTR_GEN_AI_TOOL_NAME,\n Type: ATTR_GEN_AI_TOOL_TYPE,\n /\n Note, OTel Semantic Convention suggest only putting tool inputs/outputs on the parent chat span\n * But we at least want to give users THE OPTION to put them on the tool spans themselves as well\n * Because it enables a lot of things with querying\n * @see https://github.com/open-telemetry/semantic-conventions/releases/tag/v1.37.0\n /\n Arguments: ATTR_GEN_AI_TOOL_ARGUMENTS,\n /\n Note, OTel Semantic Convention suggest only putting tool inputs/outputs on the parent chat span\n * But we at least want to give users THE OPTION to put them on the tool spans themselves as well\n * Because it enables a lot of things with querying\n * @see https://github.com/open-telemetry/semantic-conventions/releases/tag/v1.37.0\n */\n Message: ATTR_GEN_AI_TOOL_MESSAGE,\n },\n Usage: {\n InputTokens: ATTR_GEN_AI_USAGE_INPUT_TOKENS,\n OutputTokens: ATTR_GEN_AI_USAGE_OUTPUT_TOKENS,\n },\n },\n Eval: {\n ID: ATTR_EVAL_ID,\n Name: ATTR_EVAL_NAME,\n Version: ATTR_EVAL_VERSION,\n Type: ATTR_EVAL_TYPE,\n Trials: ATTR_EVAL_TRIALS,\n Baseline: {\n ID: ATTR_EVAL_BASELINE_ID,\n Name: ATTR_EVAL_BASELINE_NAME,\n Version: ATTR_EVAL_BASELINE_VERSION,\n },\n Capability: {\n Name: ATTR_EVAL_CAPABILITY_NAME,\n },\n Step: {\n Name: ATTR_EVAL_STEP_NAME,\n },\n Tags: ATTR_EVAL_TAGS,\n Metadata: ATTR_EVAL_METADATA,\n Collection: {\n ID: ATTR_EVAL_COLLECTION_ID,\n Name: ATTR_EVAL_COLLECTION_NAME,\n Size: ATTR_EVAL_COLLECTION_SIZE,\n },\n Config: {\n Flags: ATTR_EVAL_CONFIG_FLAGS,\n },\n Run: {\n ID: ATTR_EVAL_RUN_ID,\n },\n Trial: {\n Index: ATTR_EVAL_TRIAL_INDEX,\n Error: ATTR_EVAL_TRIAL_ERROR,\n },\n Case: {\n Index: ATTR_EVAL_CASE_INDEX,\n Input: ATTR_EVAL_CASE_INPUT,\n Output: ATTR_EVAL_CASE_OUTPUT,\n Expected: ATTR_EVAL_CASE_EXPECTED,\n Scores: ATTR_EVAL_CASE_SCORES,\n Metadata: ATTR_EVAL_CASE_METADATA,\n Trials: ATTR_EVAL_CASE_TRIALS,\n },\n Task: {\n Output: ATTR_EVAL_TASK_OUTPUT,\n Name: ATTR_EVAL_TASK_NAME,\n Type: ATTR_EVAL_TASK_TYPE,\n },\n Score: {\n Name: ATTR_EVAL_SCORE_NAME,\n Value: ATTR_EVAL_SCORE_VALUE,\n Threshold: ATTR_EVAL_SCORE_THRESHOLD,\n Passed: ATTR_EVAL_SCORE_PASSED,\n IsBoolean: ATTR_EVAL_SCORE_IS_BOOLEAN,\n Metadata: ATTR_EVAL_SCORE_METADATA,\n Aggregation: ATTR_EVAL_SCORE_AGGREGATION,\n Trials: ATTR_EVAL_SCORE_TRIALS,\n },\n User: {\n Name: ATTR_EVAL_USER_NAME,\n Email: ATTR_EVAL_USER_EMAIL,\n },\n },\n Error: {\n Type: ATTR_ERROR_TYPE,\n Message: ATTR_ERROR_MESSAGE,\n },\n HTTP: {\n Response: {\n StatusCode: ATTR_HTTP_RESPONSE_STATUS_CODE,\n },\n },\n} as const;\n","// experiment\nexport const ATTR_EVAL_ID = 'eval.id' as const;\nexport const ATTR_EVAL_NAME = 'eval.name' as const;\nexport const ATTR_EVAL_VERSION = 'eval.version' as const;\nexport const ATTR_EVAL_TYPE = 'eval.type' as const;\nexport const ATTR_EVAL_TAGS = 'eval.tags' as const;\nexport const ATTR_EVAL_BASELINE_ID = 'eval.baseline.id' as const;\nexport const ATTR_EVAL_BASELINE_NAME = 'eval.baseline.name' as const;\nexport const ATTR_EVAL_BASELINE_VERSION = 'eval.baseline.version' as const;\nexport const ATTR_EVAL_METADATA = 'eval.metadata' as const;\nexport const ATTR_EVAL_TRIALS = 'eval.trials' as const;\n// capability\nexport const ATTR_EVAL_CAPABILITY_NAME = 'eval.capability.name' as const;\nexport const ATTR_EVAL_STEP_NAME = 'eval.step.name' as const;\n// collection\nexport const ATTR_EVAL_COLLECTION_ID = 'eval.collection.id' as const;\nexport const ATTR_EVAL_COLLECTION_SIZE = 'eval.collection.size' as const;\nexport const ATTR_EVAL_COLLECTION_NAME = 'eval.collection.name' as const;\n// config\nexport const ATTR_EVAL_CONFIG_FLAGS = 'eval.config.flags' as const;\n// case\nexport const ATTR_EVAL_CASE_INDEX = 'eval.case.index' as const;\nexport const ATTR_EVAL_CASE_INPUT = 'eval.case.input' as const;\nexport const ATTR_EVAL_CASE_OUTPUT = 'eval.case.output' as const;\nexport const ATTR_EVAL_CASE_EXPECTED = 'eval.case.expected' as const;\nexport const ATTR_EVAL_CASE_SCORES = 'eval.case.scores' as const;\nexport const ATTR_EVAL_CASE_METADATA = 'eval.case.metadata' as const;\nexport const ATTR_EVAL_CASE_TRIALS = 'eval.case.trials' as const;\n// trial\nexport const ATTR_EVAL_TRIAL_INDEX = 'eval.trial.index' as const;\nexport const ATTR_EVAL_TRIAL_ERROR = 'eval.trial.error' as const;\n// task\nexport const ATTR_EVAL_TASK_OUTPUT = 'eval.task.output' as const;\nexport const ATTR_EVAL_TASK_NAME = 'eval.task.name' as const;\nexport const ATTR_EVAL_TASK_TYPE = 'eval.task.type' as const;\n// run\nexport const ATTR_EVAL_RUN_ID = 'eval.run.id' as const;\n// score\nexport const ATTR_EVAL_SCORE_NAME = 'eval.score.name' as const;\nexport const ATTR_EVAL_SCORE_VALUE = 'eval.score.value' as const;\nexport const ATTR_EVAL_SCORE_THRESHOLD = 'eval.score.threshold' as const;\nexport const ATTR_EVAL_SCORE_PASSED = 'eval.score.passed' as const;\nexport const ATTR_EVAL_SCORE_IS_BOOLEAN = 'eval.score.is_boolean' as const;\nexport const ATTR_EVAL_SCORE_SCORER = 'eval.score.scorer' as const;\nexport const ATTR_EVAL_SCORE_METADATA = 'eval.score.metadata' as const;\nexport const ATTR_EVAL_SCORE_AGGREGATION = 'eval.score.aggregation' as const;\nexport const ATTR_EVAL_SCORE_TRIALS = 'eval.score.trials' as const;\n// user\nexport const ATTR_EVAL_USER_NAME = 'eval.user.name';\nexport const ATTR_EVAL_USER_EMAIL = 'eval.user.email';\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,IAAAA,cAAiE;;;ACCjE,IAAAC,cAAsB;;;ACDtB;AAAA,EACE,MAAQ;AAAA,EACR,SAAW;AAAA,EACX,MAAQ;AAAA,EACR,QAAU;AAAA,EACV,cAAgB;AAAA,IACd;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAAA,EACA,SAAW;AAAA,IACT,KAAO;AAAA,IACP,OAAS;AAAA,IACT,4BAA4B;AAAA,IAC5B,QAAU;AAAA,IACV,gBAAgB;AAAA,IAChB,MAAQ;AAAA,IACR,WAAa;AAAA,IACb,MAAQ;AAAA,IACR,cAAc;AAAA,IACd,SAAW;AAAA,EACb;AAAA,EACA,OAAS;AAAA,EACT,MAAQ;AAAA,EACR,QAAU;AAAA,EACV,KAAO;AAAA,IACL,OAAS;AAAA,EACX;AAAA,EACA,SAAW;AAAA,IACT,QAAQ;AAAA,MACN,QAAU;AAAA,QACR,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,MACA,SAAW;AAAA,QACT,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,IACF;AAAA,IACA,cAAc;AAAA,MACZ,QAAU;AAAA,QACR,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,MACA,SAAW;AAAA,QACT,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,IACF;AAAA,IACA,2BAA2B;AAAA,MACzB,QAAU;AAAA,QACR,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,MACA,SAAW;AAAA,QACT,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,IACF;AAAA,IACA,sBAAsB;AAAA,MACpB,QAAU;AAAA,QACR,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,MACA,SAAW;AAAA,QACT,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,IACF;AAAA,IACA,qBAAqB;AAAA,MACnB,QAAU;AAAA,QACR,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,MACA,SAAW;AAAA,QACT,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,IACF;AAAA,IACA,eAAe;AAAA,MACb,QAAU;AAAA,QACR,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,MACA,SAAW;AAAA,QACT,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,IACF;AAAA,IACA,iBAAiB;AAAA,MACf,QAAU;AAAA,QACR,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,MACA,SAAW;AAAA,QACT,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,IACF;AAAA,EACF;AAAA,EACA,UAAY;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAAA,EACA,YAAc;AAAA,IACZ,MAAQ;AAAA,IACR,KAAO;AAAA,IACP,WAAa;AAAA,EACf;AAAA,EACA,SAAW;AAAA,EACX,cAAgB;AAAA,IACd,aAAa;AAAA,IACb,6CAA6C;AAAA,IAC7C,sCAAsC;AAAA,IACtC,2CAA2C;AAAA,IAC3C,4BAA4B;AAAA,IAC5B,iCAAiC;AAAA,IACjC,uCAAuC;AAAA,IACvC,qBAAqB;AAAA,IACrB,KAAO;AAAA,IACP,WAAa;AAAA,IACb,MAAQ;AAAA,IACR,YAAc;AAAA,IACd,QAAU;AAAA,IACV,MAAQ;AAAA,IACR,uBAAuB;AAAA,IACvB,QAAU;AAAA,EACZ;AAAA,EACA,kBAAoB;AAAA,IAClB,sBAAsB;AAAA,IACtB,KAAO;AAAA,EACT;AAAA,EACA,iBAAmB;AAAA,IACjB,uBAAuB;AAAA,IACvB,uBAAuB;AAAA,IACvB,uBAAuB;AAAA,IACvB,oBAAoB;AAAA,IACpB,oBAAoB;AAAA,IACpB,oBAAoB;AAAA,IACpB,sBAAsB;AAAA,IACtB,sBAAsB;AAAA,IACtB,sBAAsB;AAAA,IACtB,sBAAsB;AAAA,IACtB,uBAAuB;AAAA,IACvB,iCAAiC;AAAA,IACjC,iCAAiC;AAAA,IACjC,uBAAuB;AAAA,IACvB,eAAe;AAAA,IACf,uBAAuB;AAAA,IACvB,MAAQ;AAAA,IACR,MAAQ;AAAA,IACR,MAAQ;AAAA,IACR,SAAW;AAAA,IACX,QAAU;AAAA,IACV,KAAO;AAAA,IACP,UAAY;AAAA,IACZ,aAAe;AAAA,IACf,MAAQ;AAAA,IACR,YAAc;AAAA,IACd,QAAU;AAAA,IACV,KAAO;AAAA,EACT;AAAA,EACA,OAAS;AAAA,IACP;AAAA,EACF;AAAA,EACA,gBAAkB;AACpB;;;ACzKA,iBAA8D;AA8BvD,IAAM,yBAAyB,OAAO,IAAI,wBAAwB;;;AFxBzE,IAAM,qBAAqB,OAAO,IAAI,oBAAoB;AA2EnD,SAAS,kBAA0B;AAExC,QAAM,QAAS,WAAmB,kBAAkB;AAGpD,MAAI,CAAC,OAAO;AACV,UAAM,UAAU,QAAQ,IAAI,gBAAgB;AAC5C,QAAI,CAAC,SAAS;AACZ,cAAQ;AAAA,QACN;AAAA,MAGF;AAAA,IACF;AAAA,EACF;AAEA,MAAI,EAAE,MAAM,QAAQ,IAAI,SAAS,EAAE,MAAM,gBAAY,MAAM,SAAS,gBAAY,QAAQ;AAExF,MAAI,CAAC,QAAQ,CAAC,SAAS;AACrB,WAAO,gBAAY;AACnB,cAAU,gBAAY;AACtB,QAAI,CAAC,QAAQ,CAAC,SAAS;AACrB,aAAO;AACP,gBAAU;AAAA,IACZ;AAAA,EACF;AAGA,SAAO,kBAAM,UAAU,MAAM,OAAO;AACtC;;;AG9GA,IAAAC,cAA0D;;;ACA1D,kCAGO;;;ACFA,IAAM,eAAe;AACrB,IAAM,iBAAiB;AACvB,IAAM,oBAAoB;AAC1B,IAAM,iBAAiB;AACvB,IAAM,iBAAiB;AACvB,IAAM,wBAAwB;AAC9B,IAAM,0BAA0B;AAChC,IAAM,6BAA6B;AACnC,IAAM,qBAAqB;AAC3B,IAAM,mBAAmB;AAEzB,IAAM,4BAA4B;AAClC,IAAM,sBAAsB;AAE5B,IAAM,0BAA0B;AAChC,IAAM,4BAA4B;AAClC,IAAM,4BAA4B;AAElC,IAAM,yBAAyB;AAE/B,IAAM,uBAAuB;AAC7B,IAAM,uBAAuB;AAC7B,IAAM,wBAAwB;AAC9B,IAAM,0BAA0B;AAChC,IAAM,wBAAwB;AAC9B,IAAM,0BAA0B;AAChC,IAAM,wBAAwB;AAE9B,IAAM,wBAAwB;AAC9B,IAAM,wBAAwB;AAE9B,IAAM,wBAAwB;AAC9B,IAAM,sBAAsB;AAC5B,IAAM,sBAAsB;AAE5B,IAAM,mBAAmB;AAEzB,IAAM,uBAAuB;AAC7B,IAAM,wBAAwB;AAC9B,IAAM,4BAA4B;AAClC,IAAM,yBAAyB;AAC/B,IAAM,6BAA6B;AAEnC,IAAM,2BAA2B;AACjC,IAAM,8BAA8B;AACpC,IAAM,yBAAyB;AAE/B,IAAM,sBAAsB;AAC5B,IAAM,uBAAuB;;;ADFpC,wBAwDO;AAQP,IAAM,+BAA+B;AACrC,IAAM,6BAA6B;AACnC,IAAM,gCAAgC;AACtC,IAAM,8BAA8B;AACpC,IAAM,wBAAwB;AAC9B,IAAM,6BAA6B;AACnC,IAAM,2BAA2B;AAEjC,IAAM,wCAAwC;AAC9C,IAAM,sCAAsC;AAC5C,IAAM,sCAAsC;AAC5C,IAAM,uCAAuC;AAC7C,IAAM,wCAAwC;AAC9C,IAAM,iCAAiC;AACvC,IAAM,uCAAuC;AAC7C,IAAM,oCAAoC;AAC1C,IAAM,kCAAkC;AACxC,IAAM,kCAAkC;AACxC,IAAM,kCAAkC;AACxC,IAAM,uCAAuC;AAC7C,IAAM,mCAAmC;AACzC,IAAM,wCAAwC;AAC9C,IAAM,oCAAoC;AAanC,IAAM,OAAO;AAAA,EAClB,qBAAqB,CAAC,aAAqB,QAAQ,QAAQ;AAAA,EAC3D,qBAAqB,CAAC,aAAqB,QAAQ,QAAQ;AAAA,EAC3D,OAAO;AAAA,IACL,OAAO;AAAA,MACL,WAAW;AAAA,MACX,KAAK;AAAA,QACH,MAAM;AAAA,QACN,SAAS;AAAA,MACX;AAAA,IACF;AAAA,EACF;AAAA,EACA,OAAO;AAAA,IACL,gBAAgB;AAAA,MACd,IAAI;AAAA,MACJ,MAAM;AAAA,MACN,MAAM;AAAA,MACN,SAAS;AAAA,IACX;AAAA;AAAA;AAAA;AAAA,IAIA,YAAY;AAAA,MACV,MAAM;AAAA,IACR;AAAA,IACA,MAAM;AAAA,MACJ,MAAM;AAAA,IACR;AAAA,IACA,UAAU;AAAA,MACR,MAAM;AAAA,MACN,aAAa;AAAA,QACX,WAAW;AAAA,QACX,YAAY;AAAA,QACZ,YAAY;AAAA,QACZ,kBAAkB;AAAA,QAClB,eAAe;AAAA,QACf,UAAU;AAAA,QACV,QAAQ;AAAA,QACR,UAAU;AAAA,QACV,WAAW;AAAA,QACX,UAAU;AAAA,QACV,YAAY;AAAA,QACZ,KAAK;AAAA,QACL,WAAW;AAAA,QACX,WAAW;AAAA,QACX,UAAU;AAAA,QACV,aAAa;AAAA,QACb,QAAQ;AAAA,QACR,MAAM;AAAA,QACN,MAAM;AAAA,QACN,cAAc;AAAA,QACd,MAAM;AAAA,QACN,MAAM;AAAA,QACN,WAAW;AAAA,QACX,QAAQ;AAAA,QACR,YAAY;AAAA,QACZ,WAAW;AAAA,QACX,OAAO;AAAA,QACP,YAAY;AAAA,QACZ,QAAQ;AAAA,QACR,KAAK;AAAA,MACP;AAAA,IACF;AAAA;AAAA;AAAA;AAAA,IAIA,OAAO;AAAA,MACL,aAAa;AAAA;AAAA,MACb,IAAI;AAAA;AAAA,MACJ,MAAM;AAAA;AAAA,IACR;AAAA,IACA,cAAc;AAAA,MACZ,IAAI;AAAA;AAAA,IACN;AAAA,IACA,OAAO;AAAA,MACL,UAAU;AAAA,IACZ;AAAA,IACA,WAAW;AAAA,MACT,MAAM;AAAA,MACN,aAAa;AAAA;AAAA;AAAA;AAAA,QAIX,MAAM;AAAA,QACN,aAAa;AAAA,QACb,YAAY;AAAA,QACZ,aAAa;AAAA,QACb,iBAAiB;AAAA,QACjB,aAAa;AAAA,MACf;AAAA,IACF;AAAA,IACA,QAAQ;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,aAAa;AAAA,QACX,MAAM;AAAA,QACN,MAAM;AAAA,QACN,OAAO;AAAA,QACP,QAAQ;AAAA,MACV;AAAA,IACF;AAAA;AAAA;AAAA;AAAA;AAAA,IAKA,SAAS;AAAA,MACP,aAAa;AAAA;AAAA,MACb,iBAAiB;AAAA;AAAA,MACjB,kBAAkB;AAAA,MAClB,WAAW;AAAA;AAAA;AAAA;AAAA,MAIX,OAAO;AAAA,MACP,iBAAiB;AAAA,MACjB,MAAM;AAAA,MACN,eAAe;AAAA,MACf,aAAa;AAAA,MACb,MAAM;AAAA,MACN,MAAM;AAAA,IACR;AAAA,IACA,UAAU;AAAA,MACR,eAAe;AAAA,MACf,IAAI;AAAA;AAAA;AAAA;AAAA,MAIJ,OAAO;AAAA;AAAA,IACT;AAAA,IACA,MAAM;AAAA,MACJ,QAAQ;AAAA,MACR,aAAa;AAAA,MACb,MAAM;AAAA,MACN,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAON,WAAW;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAOX,SAAS;AAAA,IACX;AAAA,IACA,OAAO;AAAA,MACL,aAAa;AAAA,MACb,cAAc;AAAA,IAChB;AAAA,EACF;AAAA,EACA,MAAM;AAAA,IACJ,IAAI;AAAA,IACJ,MAAM;AAAA,IACN,SAAS;AAAA,IACT,MAAM;AAAA,IACN,QAAQ;AAAA,IACR,UAAU;AAAA,MACR,IAAI;AAAA,MACJ,MAAM;AAAA,MACN,SAAS;AAAA,IACX;AAAA,IACA,YAAY;AAAA,MACV,MAAM;AAAA,IACR;AAAA,IACA,MAAM;AAAA,MACJ,MAAM;AAAA,IACR;AAAA,IACA,MAAM;AAAA,IACN,UAAU;AAAA,IACV,YAAY;AAAA,MACV,IAAI;AAAA,MACJ,MAAM;AAAA,MACN,MAAM;AAAA,IACR;AAAA,IACA,QAAQ;AAAA,MACN,OAAO;AAAA,IACT;AAAA,IACA,KAAK;AAAA,MACH,IAAI;AAAA,IACN;AAAA,IACA,OAAO;AAAA,MACL,OAAO;AAAA,MACP,OAAO;AAAA,IACT;AAAA,IACA,MAAM;AAAA,MACJ,OAAO;AAAA,MACP,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,UAAU;AAAA,MACV,QAAQ;AAAA,MACR,UAAU;AAAA,MACV,QAAQ;AAAA,IACV;AAAA,IACA,MAAM;AAAA,MACJ,QAAQ;AAAA,MACR,MAAM;AAAA,MACN,MAAM;AAAA,IACR;AAAA,IACA,OAAO;AAAA,MACL,MAAM;AAAA,MACN,OAAO;AAAA,MACP,WAAW;AAAA,MACX,QAAQ;AAAA,MACR,WAAW;AAAA,MACX,UAAU;AAAA,MACV,aAAa;AAAA,MACb,QAAQ;AAAA,IACV;AAAA,IACA,MAAM;AAAA,MACJ,MAAM;AAAA,MACN,OAAO;AAAA,IACT;AAAA,EACF;AAAA,EACA,OAAO;AAAA,IACL,MAAM;AAAA,IACN,SAAS;AAAA,EACX;AAAA,EACA,MAAM;AAAA,IACJ,UAAU;AAAA,MACR,YAAY;AAAA,IACd;AAAA,EACF;AACF;;;AD5WA,SAAS,mBACP,YACA,YACA,QACA,MACM;AACN,QAAM,QAA+D;AAAA,IACnE,CAAC,KAAK,MAAM,UAAU,IAAI,GAAG;AAAA,IAC7B,CAAC,KAAK,KAAK,MAAM,IAAI,GAAG;AAAA,IACxB,CAAC,KAAK,KAAK,IAAI,GAAG,KAAK,UAAU,CAAC,QAAQ,CAAC;AAAA,IAC3C,CAAC,KAAK,KAAK,MAAM,KAAK,GAAG,OAAO,SAAS;AAAA,IACzC,CAAC,KAAK,KAAK,WAAW,IAAI,GAAG,MAAM;AAAA,IACnC,CAAC,KAAK,KAAK,KAAK,IAAI,GAAG,MAAM;AAAA,EAC/B;AAEA,MAAI,OAAO,YAAY,OAAO,KAAK,OAAO,QAAQ,EAAE,SAAS,GAAG;AAC9D,UAAM,KAAK,KAAK,MAAM,QAAQ,IAAI,KAAK,UAAU,OAAO,QAAQ;AAAA,EAClE;AAEA,aAAW,cAAc,KAAK;AAChC;AAKA,eAAsB,cACpB,QACA,OACA,QACA,YACA,MAC4B;AAC5B,QAAM,SAAS,gBAAgB;AAC/B,QAAM,gBAAgB,kBAAM,QAAQ,oBAAQ,OAAO,GAAG,UAAU;AAEhE,SAAO,oBAAQ,KAAK,eAAe,YAAY;AAC7C,UAAM,aACJ,OAAO,WAAW;AAAA;AAAA,MAEd,OAAO,QAAQ;AAAA,QACf,OAAO;AACb,UAAM,aAAa,OAAO,UAAU,SAAS,UAAU,EAAE;AAEzD,QAAI;AACF,YAAM,SACJ,OAAO,WAAW,aACb;AAAA,QACC,GAAI,MAAM,OAAO;AAAA,UACf;AAAA,UACA;AAAA,QACF,CAAC;AAAA,QACD,MAAM;AAAA,MACR,IACA;AAEN,yBAAmB,YAAY,YAAY,QAAQ,IAAI;AACvD,UAAI,OAAO,OAAO;AAChB,cAAM,QAAQ,IAAI,MAAM,OAAO,KAAK;AACpC,mBAAW,gBAAgB,KAAK;AAChC,mBAAW,cAAc;AAAA,UACvB,CAAC,KAAK,MAAM,OAAO,GAAG,MAAM;AAAA,UAC5B,CAAC,KAAK,MAAM,IAAI,GAAG,MAAM;AAAA,QAC3B,CAAC;AACD,mBAAW,UAAU;AAAA,UACnB,MAAM,2BAAe;AAAA,UACrB,SAAS,MAAM;AAAA,QACjB,CAAC;AAAA,MACH,OAAO;AACL,mBAAW,UAAU,EAAE,MAAM,2BAAe,GAAG,CAAC;AAAA,MAClD;AAEA,aAAO;AAAA,IACT,SAAS,KAAK;AACZ,YAAM,QAAQ,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC;AAChE,YAAM,eAA6B;AAAA,QACjC,MAAM;AAAA,QACN,OAAO;AAAA,QACP,OAAO,MAAM;AAAA,MACf;AAEA,yBAAmB,YAAY,YAAY,cAAc,IAAI;AAE7D,iBAAW,gBAAgB,KAAK;AAChC,iBAAW,cAAc;AAAA,QACvB,CAAC,KAAK,MAAM,OAAO,GAAG,MAAM;AAAA,QAC5B,CAAC,KAAK,MAAM,IAAI,GAAG,MAAM;AAAA,MAC3B,CAAC;AACD,iBAAW,UAAU;AAAA,QACnB,MAAM,2BAAe;AAAA,QACrB,SAAS,MAAM;AAAA,MACjB,CAAC;AAED,aAAO;AAAA,IACT,UAAE;AACA,iBAAW,IAAI;AAAA,IACjB;AAAA,EACF,CAAC;AACH;;;AJgBA,eAAe,aACb,UACA,MACkB;AAClB,MAAI,aAAa,OAAW,QAAO;AACnC,MAAI,OAAO,aAAa,UAAU;AAChC,QAAI,YAAY,EAAG,QAAO;AAC1B,QAAI,YAAY,EAAG,QAAO;AAC1B,WAAO,KAAK,OAAO,IAAI;AAAA,EACzB;AACA,SAAO,QAAQ,MAAM,SAAS,IAAI,CAAC;AACrC;AAEA,SAAS,qBACP,OACwD;AACxD,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,YAAY;AACpE;AAEA,SAAS,kBAAmC,QAA8C;AACxF,MAAI,OAAO,WAAW,YAAY;AAChC,WAAO,OAAO,QAAQ;AAAA,EACxB;AACA,SAAO,OAAO;AAChB;AAEA,SAAS,qBACP,OACwC;AACxC,MAAI,qBAAqB,KAAK,GAAG;AAC/B,WAAO;AAAA,MACL,MAAM,kBAAkB,MAAM,MAAM;AAAA,MACpC,QAAQ,MAAM;AAAA,MACd,UAAU,MAAM;AAAA,IAClB;AAAA,EACF;AAEA,SAAO;AAAA,IACL,MAAM,kBAAkB,KAAK;AAAA,IAC7B,QAAQ;AAAA,EACV;AACF;AAEA,SAAS,wBACP,SACU;AACV,QAAM,aAAa,oBAAI,IAAoB;AAE3C,aAAW,SAAS,SAAS;AAC3B,eAAW,IAAI,MAAM,OAAO,WAAW,IAAI,MAAM,IAAI,KAAK,KAAK,CAAC;AAAA,EAClE;AAEA,SAAO,CAAC,GAAG,WAAW,QAAQ,CAAC,EAC5B,OAAO,CAAC,CAAC,EAAE,KAAK,MAAM,QAAQ,CAAC,EAC/B,IAAI,CAAC,CAAC,IAAI,MAAM,IAAI,EACpB,KAAK;AACV;AAwDO,SAAS,WAKd,MACA,SACgD;AAChD,MAAI,QAAQ,QAAQ,WAAW,GAAG;AAChC,WAAO,QAAQ,QAAQ,CAAC,CAAC;AAAA,EAC3B;AAEA,QAAM,kBAAkB,KAAK,QAAQ,kBAAM,QAAQ,oBAAQ,OAAO,CAAC,GAAG,YAAY;AAElF,SAAO,0BAA0B,MAAM,SAAS,eAAe;AACjE;AAEA,eAAe,0BAKb,MACA,SACA,iBACgD;AAChD,QAAM,SAAS,gBAAgB;AAE/B,QAAM,WAAW,KAAK,OAAO,QAAQ,KAAK,UAAU,IAAI,KAAK,IAAI,KAAK,QAAQ,KAAK,UAAU;AAE7F,QAAM,WAAW,OAAO;AAAA,IACtB;AAAA,IACA,kBAAkB,EAAE,OAAO,CAAC,EAAE,SAAS,gBAAgB,CAAC,EAAE,IAAI,CAAC;AAAA,EACjE;AAEA,QAAM,YAAoC;AAAA,IACxC,CAAC,KAAK,MAAM,UAAU,IAAI,GAAG;AAAA,IAC7B,CAAC,KAAK,KAAK,WAAW,IAAI,GAAG,KAAK;AAAA,IAClC,CAAC,KAAK,KAAK,IAAI,GAAG,KAAK,UAAU,CAAC,QAAQ,CAAC;AAAA,EAC7C;AACA,MAAI,KAAK,MAAM;AACb,cAAU,KAAK,KAAK,KAAK,IAAI,IAAI,KAAK;AAAA,EACxC;AACA,WAAS,cAAc,SAAS;AAEhC,MAAI;AACF,UAAM,oBAAoB,QAAQ,QAAQ,IAAI,CAAC,UAAU,qBAAqB,KAAK,CAAC;AACpF,UAAM,uBAAuB,wBAAwB,iBAAiB;AACtE,QAAI,qBAAqB,SAAS,GAAG;AACnC,YAAM,IAAI;AAAA,QACR,2CAA2C,qBACxC,IAAI,CAAC,SAAS,IAAI,IAAI,GAAG,EACzB,KAAK,IAAI,CAAC;AAAA,MACf;AAAA,IACF;AAEA,UAAM,WAAW,MAAM,QAAQ;AAAA,MAC7B,kBAAkB,IAAI,OAAO,UAAU;AACrC,YAAI;AACF,gBAAM,YAAY,MAAM,aAAa,MAAM,UAAU;AAAA,YACnD,OAAO,QAAQ;AAAA,YACf,QAAQ,QAAQ;AAAA,UAClB,CAAC;AAED,cAAI,CAAC,WAAW;AACd,mBAAO,EAAE,YAAY,KAAc;AAAA,UACrC;AAEA,iBAAO;AAAA,YACL,YAAY;AAAA,YACZ,QAAQ,MAAM;AAAA,cACZ,MAAM;AAAA,cACN,QAAQ;AAAA,cACR,QAAQ;AAAA,cACR;AAAA,cACA;AAAA,YACF;AAAA,UACF;AAAA,QACF,SAAS,KAAK;AACZ,gBAAM,QAAQ,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC;AAChE,iBAAO;AAAA,YACL,YAAY;AAAA,YACZ,QAAQ,MAAM;AAAA,cACZ;AAAA,gBACE,MAAM,MAAM;AAAA,gBACZ,OAAO;AAAA,gBACP,OAAO,MAAM;AAAA,cACf;AAAA,cACA,QAAQ;AAAA,cACR,QAAQ;AAAA,cACR;AAAA,cACA;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAEA,UAAM,UAA2D,CAAC;AAClE,QAAI,kBAAkB;AAEtB,eAAW,WAAW,UAAU;AAC9B,UAAI,QAAQ,YAAY;AACtB,2BAAmB;AACnB;AAAA,MACF;AAEA,cAAQ,QAAQ,OAAO,IAAI,IAAI,QAAQ;AAAA,IACzC;AAEA,UAAM,cAAc,OAAO,OAAO,OAAO,EAAE,OAAO,CAAC,WAAW,OAAO,KAAK,EAAE;AAC5E,UAAM,WAAW,OAAO,KAAK,OAAO,EAAE;AAEtC,aAAS,cAAc;AAAA,MACrB,mCAAmC,kBAAkB;AAAA,MACrD,iCAAiC;AAAA,MACjC,yCAAyC;AAAA,MACzC,oCAAoC;AAAA,IACtC,CAAC;AAED,QAAI,cAAc,GAAG;AACnB,eAAS,UAAU;AAAA,QACjB,MAAM,2BAAe;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AAAA,IACH,OAAO;AACL,eAAS,UAAU,EAAE,MAAM,2BAAe,GAAG,CAAC;AAAA,IAChD;AAEA,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,UAAM,QAAQ,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC;AAChE,aAAS,gBAAgB,KAAK;AAC9B,aAAS,UAAU;AAAA,MACjB,MAAM,2BAAe;AAAA,MACrB,SAAS,MAAM;AAAA,IACjB,CAAC;AACD,WAAO,CAAC;AAAA,EACV,UAAE;AACA,aAAS,IAAI;AAAA,EACf;AACF;","names":["import_api","import_api","import_api"]}
1	+ {"version":3,"sources":["../../src/evals/online.ts","../../src/online-evals/onlineEval.ts","../../src/otel/initAxiomAI.ts","../../package.json","../../src/otel/utils/redaction.ts","../../src/online-evals/executor.ts","../../src/otel/semconv/attributes.ts","../../src/otel/semconv/eval_proposal.ts","../../src/evals/normalize-score.ts"],"sourcesContent":["export { onlineEval } from '../online-evals/onlineEval';\nexport type { OnlineEvalMeta, OnlineEvalOptions } from '../online-evals/onlineEval';\nexport type {\n OnlineEvalScorerEntry,\n OnlineEvalScorerInput,\n SampledOnlineEvalScorer,\n ScorerSampling,\n} from '../online-evals/types';\n","import { context, trace, SpanStatusCode, type SpanContext } from '@opentelemetry/api';\nimport { getGlobalTracer } from '../otel/initAxiomAI';\nimport type {\n OnlineEvalScorerEntry,\n OnlineEvalScorerInput,\n SampledOnlineEvalScorer,\n Scorer,\n ScorerResult,\n ScorerSampling,\n} from './types';\nimport { executeScorer } from './executor';\nimport { Attr } from '../otel/semconv/attributes';\n\ntype ScorerEntry<TInput, TOutput> = OnlineEvalScorerEntry<TInput, TOutput, any>;\ntype ScorerInput<TInput, TOutput> = OnlineEvalScorerInput<TInput, TOutput, any>;\n\ntype InferScorerMetadata<TScorerEntry> =\n TScorerEntry extends SampledOnlineEvalScorer<any, any, any>\n ? InferScorerMetadata<TScorerEntry['scorer']>\n : TScorerEntry extends Scorer<any, any, infer TMetadata>\n ? TMetadata\n : TScorerEntry extends ScorerResult<infer TMetadata>\n ? TMetadata\n : never;\n\ntype InferScorerName<TScorerEntry> =\n TScorerEntry extends SampledOnlineEvalScorer<any, any, any>\n ? InferScorerName<TScorerEntry['scorer']>\n : TScorerEntry extends ScorerResult<any>\n ? TScorerEntry['name']\n : TScorerEntry extends Scorer<any, any, any>\n ? string\n : never;\n\ntype IsBroadString<T extends string> = string extends T ? true : false;\n\ntype DuplicateScorerNames<\n TEntries extends readonly unknown[],\n Seen extends string = never,\n Duplicates extends string = never,\n> = TEntries extends readonly [infer Head, ...infer Tail]\n ? InferScorerName<Head> extends infer ScorerName\n ? ScorerName extends string\n ? IsBroadString<ScorerName> extends true\n ? DuplicateScorerNames<Tail extends readonly unknown[] ? Tail : never, Seen, Duplicates>\n : ScorerName extends Seen\n ? DuplicateScorerNames<\n Tail extends readonly unknown[] ? Tail : never,\n Seen,\n Duplicates \| ScorerName\n >\n : DuplicateScorerNames<\n Tail extends readonly unknown[] ? Tail : never,\n Seen \| ScorerName,\n Duplicates\n >\n : DuplicateScorerNames<Tail extends readonly unknown[] ? Tail : never, Seen, Duplicates>\n : never\n : Duplicates;\n\ntype EnsureUniqueScorerNames<TEntries extends readonly unknown[]> = [\n DuplicateScorerNames<TEntries>,\n] extends [never]\n ? TEntries\n : never & {\n __axiomDuplicateScorerNames__: DuplicateScorerNames<TEntries>;\n };\n\ntype InferOnlineEvalResult<TScorers extends readonly unknown[]> = ScorerResult<\n InferScorerMetadata<TScorers[number]>\n>;\n\ntype InferOnlineEvalResultRecord<TScorers extends readonly unknown[]> = Partial<\n Record<Extract<InferScorerName<TScorers[number]>, string>, InferOnlineEvalResult<TScorers>>\n>;\n\ntype NormalizedScorerEntry<TInput, TOutput> = {\n name: string;\n scorer: ScorerInput<TInput, TOutput>;\n sampling?: ScorerSampling<TInput, TOutput>;\n};\n\n/*\n Metadata for categorizing online evaluations.\n /\nexport type OnlineEvalMeta = {\n /* High-level capability being evaluated (e.g., 'qa', 'summarization') /\n capability: string;\n /* Specific step within the capability (e.g., 'answer', 'extract') /\n step?: string;\n /\n Explicit SpanContext(s) to link the eval span to originating generation span(s).\n * When omitted, the active span's context is used automatically.\n * Use this for deferred evaluation when onlineEval is called after the\n * originating span has completed.\n * Supports both single context and multiple contexts for multi-span linking.\n /\n links?: SpanContext \| SpanContext[];\n};\n\n/\n Options for online evaluation.\n /\nexport type OnlineEvalOptions<\n TInput,\n TOutput,\n TScorers extends readonly ScorerEntry<TInput, TOutput>[] = readonly ScorerEntry<\n TInput,\n TOutput\n >[],\n> = {\n /* Input to pass to scorers (optional - only needed for input+output scorers) /\n input?: TInput;\n /* Output to evaluate /\n output: TOutput;\n /* Scorers or precomputed scores. Supports optional per-scorer sampling. /\n scorers: EnsureUniqueScorerNames<TScorers>;\n};\n\n/\n Determines if an individual scorer should run based on sampling configuration.\n /\nasync function shouldSample<TInput, TOutput>(\n sampling: ScorerSampling<TInput, TOutput> \| undefined,\n args: { input?: TInput; output: TOutput },\n): Promise<boolean> {\n if (sampling === undefined) return true;\n if (typeof sampling === 'number') {\n if (sampling >= 1) return true;\n if (sampling <= 0) return false;\n return Math.random() < sampling;\n }\n return Boolean(await sampling(args));\n}\n\nfunction isSampledScorerEntry<TInput, TOutput>(\n entry: ScorerEntry<TInput, TOutput>,\n): entry is SampledOnlineEvalScorer<TInput, TOutput, any> {\n return typeof entry === 'object' && entry !== null && 'scorer' in entry;\n}\n\nfunction resolveScorerName<TInput, TOutput>(scorer: ScorerInput<TInput, TOutput>): string {\n if (typeof scorer === 'function') {\n return scorer.name \|\| 'unknown';\n }\n return scorer.name;\n}\n\nfunction normalizeScorerEntry<TInput, TOutput>(\n entry: ScorerEntry<TInput, TOutput>,\n): NormalizedScorerEntry<TInput, TOutput> {\n if (isSampledScorerEntry(entry)) {\n return {\n name: resolveScorerName(entry.scorer),\n scorer: entry.scorer,\n sampling: entry.sampling,\n };\n }\n\n return {\n name: resolveScorerName(entry),\n scorer: entry,\n };\n}\n\nfunction getDuplicateScorerNames<TInput, TOutput>(\n entries: readonly NormalizedScorerEntry<TInput, TOutput>[],\n): string[] {\n const nameCounts = new Map<string, number>();\n\n for (const entry of entries) {\n nameCounts.set(entry.name, (nameCounts.get(entry.name) ?? 0) + 1);\n }\n\n return [...nameCounts.entries()]\n .filter(([, count]) => count > 1)\n .map(([name]) => name)\n .sort();\n}\n\n/\n Run online evaluation scorers against production outputs.\n \n Returns a promise that resolves to scorer results. Use `void onlineEval(...)`\n * for fire-and-forget, or `await onlineEval(...)` when you need to wait for\n * completion (e.g., before flushing telemetry in short-lived processes).\n \n Each eval span links back to the originating generation span via an\n * OpenTelemetry span link. Parent/child hierarchy follows natural context\n * propagation — inside `withSpan` the eval is a child, outside it depends\n * on the active context.\n \n ## Usage Patterns\n \n Inside withSpan (recommended):\n * Active span is automatically detected and linked.\n * ```ts\n * await withSpan({ capability: 'qa', step: 'answer' }, async () => {\n * const response = await generateText({ ... });\n * void onlineEval(\n * { capability: 'qa', step: 'answer' },\n * { output: response.text, scorers: [formatScorer] }\n * );\n * return response.text;\n * });\n * ```\n \n Deferred evaluation with explicit link:\n * Pass the originating span's context for linking when evaluating later.\n * Supports single or multiple span contexts.\n * ```ts\n * let spanCtx: SpanContext;\n * const result = await withSpan({ ... }, async (span) => {\n * spanCtx = span.spanContext();\n * return await generateText({ ... });\n * });\n * void onlineEval({ ..., links: spanCtx }, { output: result, scorers });\n \n // Or link to multiple spans:\n * void onlineEval({ ..., links: [spanCtx1, spanCtx2] }, { output, scorers });\n * ```\n \n Awaiting for flush (short-lived processes):\n * ```ts\n * await onlineEval({ ... }, { output, scorers });\n * await flushTelemetry();\n * ```\n \n @param meta - Evaluation metadata for categorization\n * @param meta.capability - High-level capability being evaluated\n * @param meta.step - Optional step within the capability\n * @param meta.links - Optional SpanContext(s) to link to (auto-detected if omitted)\n * @param options - Evaluation configuration\n * @param options.input - Input to pass to scorers\n * @param options.output - Output to evaluate\n * @param options.scorers - Scorer entries with optional per-scorer sampling\n * @returns Promise resolving to scorer results keyed by scorer name\n /\nexport function onlineEval<\n TInput,\n TOutput,\n const TScorers extends readonly ScorerEntry<TInput, TOutput>[],\n>(\n meta: OnlineEvalMeta,\n options: OnlineEvalOptions<TInput, TOutput, TScorers>,\n): Promise<InferOnlineEvalResultRecord<TScorers>> {\n if (options.scorers.length === 0) {\n return Promise.resolve({});\n }\n\n const rawLinks = meta.links ?? trace.getSpan(context.active())?.spanContext();\n const linkContexts = rawLinks ? (Array.isArray(rawLinks) ? rawLinks : [rawLinks]) : [];\n\n return executeOnlineEvalInternal(meta, options, linkContexts);\n}\n\nasync function executeOnlineEvalInternal<\n TInput,\n TOutput,\n const TScorers extends readonly ScorerEntry<TInput, TOutput>[],\n>(\n meta: OnlineEvalMeta,\n options: OnlineEvalOptions<TInput, TOutput, TScorers>,\n linkContexts: SpanContext[],\n): Promise<InferOnlineEvalResultRecord<TScorers>> {\n const tracer = getGlobalTracer();\n\n const spanName = meta.step ? `eval ${meta.capability}/${meta.step}` : `eval ${meta.capability}`;\n\n const evalSpan = tracer.startSpan(\n spanName,\n linkContexts.length > 0 ? { links: linkContexts.map((ctx) => ({ context: ctx })) } : {},\n );\n\n const evalAttrs: Record<string, string> = {\n [Attr.GenAI.Operation.Name]: 'eval',\n [Attr.Eval.Capability.Name]: meta.capability,\n [Attr.Eval.Tags]: JSON.stringify(['online']),\n };\n if (meta.step) {\n evalAttrs[Attr.Eval.Step.Name] = meta.step;\n }\n evalSpan.setAttributes(evalAttrs);\n\n try {\n const normalizedScorers = options.scorers.map((entry) => normalizeScorerEntry(entry));\n const duplicateScorerNames = getDuplicateScorerNames(normalizedScorers);\n if (duplicateScorerNames.length > 0) {\n throw new Error(\n `Duplicate scorer names are not allowed: ${duplicateScorerNames\n .map((name) => `\"${name}\"`)\n .join(', ')}`,\n );\n }\n\n const outcomes = await Promise.all(\n normalizedScorers.map(async (entry) => {\n try {\n const sampledIn = await shouldSample(entry.sampling, {\n input: options.input,\n output: options.output,\n });\n\n if (!sampledIn) {\n return { sampledOut: true as const };\n }\n\n return {\n sampledOut: false as const,\n result: await executeScorer(\n entry.scorer,\n options.input,\n options.output,\n evalSpan,\n meta,\n ),\n };\n } catch (err) {\n const error = err instanceof Error ? err : new Error(String(err));\n return {\n sampledOut: false as const,\n result: await executeScorer(\n {\n name: entry.name,\n score: null,\n error: error.message,\n },\n options.input,\n options.output,\n evalSpan,\n meta,\n ),\n };\n }\n }),\n );\n\n const results: Record<string, InferOnlineEvalResult<TScorers>> = {};\n let sampledOutCount = 0;\n\n for (const outcome of outcomes) {\n if (outcome.sampledOut) {\n sampledOutCount += 1;\n continue;\n }\n\n results[outcome.result.name] = outcome.result as InferOnlineEvalResult<TScorers>;\n }\n\n const failedCount = Object.values(results).filter((result) => result.error).length;\n const ranCount = Object.keys(results).length;\n\n const scoresSummary: Record<string, ScorerResult> = {};\n for (const [name, result] of Object.entries(results)) {\n scoresSummary[name] = {\n name: result.name,\n score: result.score,\n ...(result.metadata &&\n Object.keys(result.metadata).length > 0 && { metadata: result.metadata }),\n ...(result.error && { error: result.error }),\n };\n }\n\n evalSpan.setAttributes({\n [Attr.Eval.Case.Scores]: JSON.stringify(scoresSummary),\n 'axiom.eval.online.scorers.total': normalizedScorers.length,\n 'axiom.eval.online.scorers.ran': ranCount,\n 'axiom.eval.online.scorers.sampled_out': sampledOutCount,\n 'axiom.eval.online.scorers.failed': failedCount,\n });\n\n if (failedCount > 0) {\n evalSpan.setStatus({\n code: SpanStatusCode.ERROR,\n message: 'One or more scorers failed',\n });\n } else {\n evalSpan.setStatus({ code: SpanStatusCode.OK });\n }\n\n return results as InferOnlineEvalResultRecord<TScorers>;\n } catch (err) {\n const error = err instanceof Error ? err : new Error(String(err));\n evalSpan.recordException(error);\n evalSpan.setStatus({\n code: SpanStatusCode.ERROR,\n message: error.message,\n });\n return {};\n } finally {\n evalSpan.end();\n }\n}\n","import type { Tracer } from '@opentelemetry/api';\nimport { trace } from '@opentelemetry/api';\nimport packageJson from '../../package.json';\nimport { AXIOM_AI_REDACTION_KEY, type AxiomAIRedactionPolicy } from './utils/redaction';\n\n// Global key to store tracer scope information across all execution contexts\nconst AXIOM_AI_SCOPE_KEY = Symbol.for('__axiom_ai_scope__');\n\ninterface TracerScope {\n name: string;\n version?: string;\n}\n\n/\n Extract scope information from a tracer, with fallback to package.json\n /\nfunction extractTracerScope(tracer: Tracer): TracerScope {\n const tracerAny = tracer as any;\n\n // Guard access to private fields with optional chaining\n // Note: These are internal OTEL fields that may change in future versions\n // _instrumentationScope is modern, instrumentationLibrary is legacy (<1.16)\n const name =\n tracerAny._instrumentationScope?.name \|\|\n tracerAny.instrumentationLibrary?.name \|\|\n packageJson.name;\n\n const version =\n tracerAny._instrumentationScope?.version \|\|\n tracerAny.instrumentationLibrary?.version \|\|\n packageJson.version;\n\n return { name, version };\n}\n\n/\n Register this in your `instrumentation.ts` to set up axiom.\n * This function stores the tracer's scope information globally to enable Context Propagation\n * and Instrumentation Scope. The tracer will be available across all execution contexts including Next.js.\n \n This function is idempotent - calling it multiple times with the same scope has no additional cost.\n \n @param config\n * @param config.tracer - The tracer that you are using in your application.\n * @param config.redactionPolicy - Optional redaction policy to control what data is captured in spans.\n /\nexport function initAxiomAI(config: { tracer: Tracer; redactionPolicy?: AxiomAIRedactionPolicy }) {\n const newScope = extractTracerScope(config.tracer);\n const existingScope = (globalThis as any)[AXIOM_AI_SCOPE_KEY] as TracerScope \| undefined;\n\n // Check if already initialized with same scope (idempotent behavior)\n if (\n existingScope &&\n existingScope.name === newScope.name &&\n existingScope.version === newScope.version\n ) {\n return;\n }\n\n // Warn about double initialization with different scopes\n if (existingScope) {\n console.warn(\n '[AxiomAI] initAxiomAI() called multiple times with different scopes. ' +\n `Previous: ${existingScope.name}@${existingScope.version}, ` +\n `New: ${newScope.name}@${newScope.version}`,\n );\n }\n\n // Store scope information globally (works across Next.js contexts)\n (globalThis as any)[AXIOM_AI_SCOPE_KEY] = newScope;\n\n // Store redaction policy globally if provided\n if (config.redactionPolicy) {\n (globalThis as any)[AXIOM_AI_REDACTION_KEY] = config.redactionPolicy;\n }\n}\n\n/\n Get a tracer using the globally stored scope information\n * Fall back to package.json defaults if not set\n /\nexport function getGlobalTracer(): Tracer {\n // Get stored scope information or fall back to package defaults\n const scope = (globalThis as any)[AXIOM_AI_SCOPE_KEY] as TracerScope \| undefined;\n\n // Warn if initAxiomAI was never called\n if (!scope) {\n const isDebug = process.env.AXIOM_DEBUG === 'true';\n if (!isDebug) {\n console.warn(\n '[AxiomAI] AXIOM_AI_SCOPE_KEY is undefined. This probably means that ' +\n 'initAxiomAI() was never called. ' +\n 'Make sure to call initAxiomAI({ tracer }) in your instrumentation setup.',\n );\n }\n }\n\n let { name, version } = scope \|\| { name: packageJson.name, version: packageJson.version };\n\n if (!name \|\| !version) {\n name = packageJson.name;\n version = packageJson.version;\n if (!name \|\| !version) {\n name = 'axiom';\n version = 'unknown';\n }\n }\n\n // Use OpenTelemetry's standard global provider mechanism\n return trace.getTracer(name, version);\n}\n\n/\n Reset AxiomAI configuration (useful for testing)\n /\nexport function resetAxiomAI() {\n (globalThis as any)[AXIOM_AI_SCOPE_KEY] = undefined;\n (globalThis as any)[AXIOM_AI_REDACTION_KEY] = undefined;\n}\n","{\n \"name\": \"axiom\",\n \"version\": \"0.45.0\",\n \"type\": \"module\",\n \"author\": \"Axiom, Inc.\",\n \"contributors\": [\n \"Islam Shehata <islam@axiom.co>\",\n \"Chris Ehrlich <chris@axiom.co>\",\n \"Gabriel de Andrade <gabriel@axiom.co>\"\n ],\n \"scripts\": {\n \"dev\": \"tsup --watch\",\n \"build\": \"tsup && chmod +x dist/bin.js && pnpm check:vitest-entrypoints\",\n \"check:vitest-entrypoints\": \"node ./scripts/check-vitest-entrypoints.mjs\",\n \"format\": \"prettier --write .\",\n \"format:check\": \"prettier --check .\",\n \"lint\": \"eslint './/.{js,ts}'\",\n \"typecheck\": \"tsc --noEmit\",\n \"test\": \"vitest run\",\n \"test:watch\": \"vitest --watch\",\n \"publint\": \"npx publint\"\n },\n \"types\": \"./dist/index.d.ts\",\n \"main\": \"./dist/index.cjs\",\n \"module\": \"./dist/index.js\",\n \"bin\": {\n \"axiom\": \"./dist/bin.js\"\n },\n \"exports\": {\n \"./ai\": {\n \"import\": {\n \"types\": \"./dist/index.d.ts\",\n \"default\": \"./dist/index.js\"\n },\n \"require\": {\n \"types\": \"./dist/index.d.cts\",\n \"default\": \"./dist/index.cjs\"\n }\n },\n \"./ai/evals\": {\n \"import\": {\n \"types\": \"./dist/evals.d.ts\",\n \"default\": \"./dist/evals.js\"\n },\n \"require\": {\n \"types\": \"./dist/evals.d.cts\",\n \"default\": \"./dist/evals.cjs\"\n }\n },\n \"./ai/evals/aggregations\": {\n \"import\": {\n \"types\": \"./dist/evals/aggregations.d.ts\",\n \"default\": \"./dist/evals/aggregations.js\"\n },\n \"require\": {\n \"types\": \"./dist/evals/aggregations.d.cts\",\n \"default\": \"./dist/evals/aggregations.cjs\"\n }\n },\n \"./ai/evals/scorers\": {\n \"import\": {\n \"types\": \"./dist/evals/scorers.d.ts\",\n \"default\": \"./dist/evals/scorers.js\"\n },\n \"require\": {\n \"types\": \"./dist/evals/scorers.d.cts\",\n \"default\": \"./dist/evals/scorers.cjs\"\n }\n },\n \"./ai/evals/online\": {\n \"import\": {\n \"types\": \"./dist/evals/online.d.ts\",\n \"default\": \"./dist/evals/online.js\"\n },\n \"require\": {\n \"types\": \"./dist/evals/online.d.cts\",\n \"default\": \"./dist/evals/online.cjs\"\n }\n },\n \"./ai/config\": {\n \"import\": {\n \"types\": \"./dist/config.d.ts\",\n \"default\": \"./dist/config.js\"\n },\n \"require\": {\n \"types\": \"./dist/config.d.cts\",\n \"default\": \"./dist/config.cjs\"\n }\n },\n \"./ai/feedback\": {\n \"import\": {\n \"types\": \"./dist/feedback.d.ts\",\n \"default\": \"./dist/feedback.js\"\n },\n \"require\": {\n \"types\": \"./dist/feedback.d.cts\",\n \"default\": \"./dist/feedback.cjs\"\n }\n }\n },\n \"keywords\": [\n \"axiom\",\n \"logging\",\n \"ai\",\n \"otel\",\n \"opentelemetry\"\n ],\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/axiomhq/ai.git\",\n \"directory\": \"packages/ai\"\n },\n \"license\": \"MIT\",\n \"dependencies\": {\n \"@next/env\": \"^15.4.2\",\n \"@opentelemetry/auto-instrumentations-node\": \"^0.60.1\",\n \"@opentelemetry/context-async-hooks\": \"^2.0.1\",\n \"@opentelemetry/exporter-trace-otlp-http\": \"^0.202.0\",\n \"@opentelemetry/resources\": \"^2.0.1\",\n \"@opentelemetry/sdk-trace-node\": \"^2.0.1\",\n \"@opentelemetry/semantic-conventions\": \"^1.38.0\",\n \"@sinclair/typebox\": \"^0.34.37\",\n \"c12\": \"^3.3.3\",\n \"commander\": \"^14.0.0\",\n \"defu\": \"^6.1.4\",\n \"handlebars\": \"^4.7.8\",\n \"nanoid\": \"^5.1.5\",\n \"open\": \"^10.1.0\",\n \"vite-tsconfig-paths\": \"^5.1.4\",\n \"vitest\": \"^4.0.0\"\n },\n \"peerDependencies\": {\n \"@opentelemetry/api\": \"^1.9.0\",\n \"zod\": \"^3.25.0 \|\| ^4.0.0\"\n },\n \"devDependencies\": {\n \"@ai-sdk/anthropicv1\": \"npm:@ai-sdk/anthropic@^1.2.12\",\n \"@ai-sdk/anthropicv2\": \"npm:@ai-sdk/anthropic@^2.0.57\",\n \"@ai-sdk/anthropicv3\": \"npm:@ai-sdk/anthropic@^3.0.9\",\n \"@ai-sdk/openaiv1\": \"npm:@ai-sdk/openai@^1.3.24\",\n \"@ai-sdk/openaiv2\": \"npm:@ai-sdk/openai@^2.0.88\",\n \"@ai-sdk/openaiv3\": \"npm:@ai-sdk/openai@^3.0.7\",\n \"@ai-sdk/providerv1\": \"npm:@ai-sdk/provider@^1.1.3\",\n \"@ai-sdk/providerv2\": \"npm:@ai-sdk/provider@^2.0.1\",\n \"@ai-sdk/providerv3\": \"npm:@ai-sdk/provider@^3.0.2\",\n \"@opentelemetry/api\": \"^1.9.0\",\n \"@opentelemetry/core\": \"^2.0.1\",\n \"@opentelemetry/sdk-trace-base\": \"^2.0.1\",\n \"@opentelemetry/sdk-trace-node\": \"^2.0.1\",\n \"@repo/eslint-config\": \"workspace:\",\n \"@types/node\": \"^22.15.29\",\n \"@vitest/coverage-v8\": \"^4.0.0\",\n \"aiv4\": \"npm:ai@^4.3.19\",\n \"aiv5\": \"npm:ai@^5.0.118\",\n \"aiv6\": \"npm:ai@^6.0.6\",\n \"esbuild\": \"^0.25.8\",\n \"eslint\": \"catalog:\",\n \"msw\": \"^2.12.2\",\n \"prettier\": \"catalog:\",\n \"tinyrainbow\": \"^2.0.0\",\n \"tsup\": \"catalog:\",\n \"typescript\": \"catalog:\",\n \"vitest\": \"catalog:\",\n \"zod\": \"catalog:\"\n },\n \"files\": [\n \"dist\"\n ],\n \"packageManager\": \"pnpm@10.16.1\"\n}\n","import { context, propagation, type Baggage, type Span } from '@opentelemetry/api';\nimport { WITHSPAN_REDACTION_POLICY_KEY } from '../withSpanBaggageKey';\n\ntype CaptureMessageContent = 'full' \| 'off';\n\nexport type AxiomAIRedactionPolicy = {\n captureMessageContent?: CaptureMessageContent;\n mirrorToolPayloadOnToolSpan?: boolean;\n};\n\nexport const RedactionPolicy = {\n /\n Includes message content on chat spans, and mirrors tool\n * payload on tool spans for more convenient querying.\n /\n AxiomDefault: {\n captureMessageContent: 'full',\n mirrorToolPayloadOnToolSpan: true,\n },\n /\n Redacts message content on chat spans, and does not put\n * tool payload on tool spans.\n /\n OpenTelemetryDefault: {\n captureMessageContent: 'off',\n mirrorToolPayloadOnToolSpan: false,\n },\n} as const satisfies Record<string, AxiomAIRedactionPolicy>;\n\n// Global key to store redaction policy across all execution contexts\nexport const AXIOM_AI_REDACTION_KEY = Symbol.for('__axiom_ai_redaction__');\n\n/\n Get the globally stored redaction policy\n * @returns The global redaction policy or undefined if not set\n /\nfunction getGlobalRedactionPolicy(): AxiomAIRedactionPolicy \| undefined {\n return (globalThis as any)[AXIOM_AI_REDACTION_KEY];\n}\n\n/\n Resolves the effective redaction policy by merging global and per-call policies.\n * Per-call policies take precedence over global policies.\n \n @param globalPolicy - The global redaction policy from initAxiomAI\n * @param localPolicy - The per-call redaction policy from withSpan or middleware\n * @returns The effective redaction policy with all fields resolved\n /\nfunction getEffectiveRedactionPolicy(\n globalPolicy?: AxiomAIRedactionPolicy,\n localPolicy?: AxiomAIRedactionPolicy,\n): AxiomAIRedactionPolicy {\n // Per-call policy overrides global policy, with defaults\n return {\n captureMessageContent:\n localPolicy?.captureMessageContent ??\n globalPolicy?.captureMessageContent ??\n RedactionPolicy.AxiomDefault.captureMessageContent,\n mirrorToolPayloadOnToolSpan:\n localPolicy?.mirrorToolPayloadOnToolSpan ??\n globalPolicy?.mirrorToolPayloadOnToolSpan ??\n RedactionPolicy.AxiomDefault.mirrorToolPayloadOnToolSpan,\n };\n}\n\n/\n Get the active redaction policy\n /\nexport function getRedactionPolicy() {\n return getEffectiveRedactionPolicy(getGlobalRedactionPolicy(), getPerCallRedactionPolicy());\n}\n\n/\n Conditionally sets a span attribute based on the capture message content policy.\n * When policy is 'off', the attribute is not set at all.\n \n @param span - The OpenTelemetry span to set the attribute on\n * @param attribute - The attribute name/key\n * @param value - The attribute value\n * @param captureMessageContent - The capture policy ('full' sets attribute, 'off' skips it)\n /\nexport function handleMaybeRedactedAttribute(\n span: Span,\n attribute: string,\n value: any,\n captureMessageContent?: CaptureMessageContent,\n): void {\n if (captureMessageContent === 'full') {\n span.setAttribute(attribute, value);\n }\n\n // For 'off', don't set the attribute at all\n // Future: Could add callback-based redaction here\n}\n\n/\n Extracts the per-call redaction policy from OpenTelemetry baggage context.\n * This allows middleware and tools to access redaction policies passed via withSpan.\n \n @returns The per-call redaction policy or undefined if not set\n /\nfunction getPerCallRedactionPolicy(): AxiomAIRedactionPolicy \| undefined {\n const baggage: Baggage = propagation.getBaggage(context.active()) \|\| propagation.createBaggage();\n const serializedPolicy = baggage.getEntry(WITHSPAN_REDACTION_POLICY_KEY)?.value;\n\n if (!serializedPolicy) {\n return undefined;\n }\n\n try {\n return JSON.parse(serializedPolicy) as AxiomAIRedactionPolicy;\n } catch (error) {\n console.warn('[AxiomAI] Failed to parse redaction policy from baggage:', error);\n return undefined;\n }\n}\n","import { context, trace, SpanStatusCode, type Span } from '@opentelemetry/api';\nimport { normalizeBooleanScore } from '../evals/normalize-score';\nimport { getGlobalTracer } from '../otel/initAxiomAI';\nimport type { Scorer, ScorerResult } from './types';\nimport type { OnlineEvalMeta } from './onlineEval';\nimport { Attr } from '../otel/semconv/attributes';\n\ntype OnlineEvalScorerInput<TInput, TOutput> = Scorer<TInput, TOutput, any> \| ScorerResult<any>;\n\nfunction setScorerSpanAttrs(\n scorerSpan: Span,\n scorerName: string,\n result: Pick<ScorerResult<any>, 'score' \| 'metadata'>,\n meta?: OnlineEvalMeta,\n): void {\n const { score: scoreValue, metadata } = normalizeBooleanScore(result.score, result.metadata);\n\n const attrs: Record<string, string \| number \| boolean \| undefined> = {\n [Attr.GenAI.Operation.Name]: 'eval.score',\n [Attr.Eval.Score.Name]: scorerName,\n [Attr.Eval.Tags]: JSON.stringify(['online']),\n [Attr.Eval.Score.Value]: scoreValue ?? undefined,\n [Attr.Eval.Capability.Name]: meta?.capability,\n [Attr.Eval.Step.Name]: meta?.step,\n };\n\n if (metadata && Object.keys(metadata).length > 0) {\n attrs[Attr.Eval.Score.Metadata] = JSON.stringify(metadata);\n }\n\n scorerSpan.setAttributes(attrs);\n}\n\n/\n Executes a single scorer or emits a precomputed scorer result.\n /\nexport async function executeScorer<TInput, TOutput>(\n scorer: OnlineEvalScorerInput<TInput, TOutput>,\n input: TInput \| undefined,\n output: TOutput,\n parentSpan: Span,\n meta?: OnlineEvalMeta,\n): Promise<ScorerResult<any>> {\n const tracer = getGlobalTracer();\n const parentContext = trace.setSpan(context.active(), parentSpan);\n\n return context.with(parentContext, async () => {\n const scorerName =\n typeof scorer === 'function'\n ? // undefined/unknown case shouldn't happen, but better safe than sorry\n scorer.name \|\| 'unknown'\n : scorer.name;\n const scorerSpan = tracer.startSpan(`score ${scorerName}`);\n\n try {\n const result =\n typeof scorer === 'function'\n ? ({\n ...(await scorer({\n input,\n output,\n })),\n name: scorerName,\n } satisfies ScorerResult)\n : scorer;\n\n setScorerSpanAttrs(scorerSpan, scorerName, result, meta);\n if (result.error) {\n const error = new Error(result.error);\n scorerSpan.recordException(error);\n scorerSpan.setAttributes({\n [Attr.Error.Message]: error.message,\n [Attr.Error.Type]: error.name,\n });\n scorerSpan.setStatus({\n code: SpanStatusCode.ERROR,\n message: error.message,\n });\n } else {\n scorerSpan.setStatus({ code: SpanStatusCode.OK });\n }\n\n return result;\n } catch (err) {\n const error = err instanceof Error ? err : new Error(String(err));\n const failedResult: ScorerResult = {\n name: scorerName,\n score: null,\n error: error.message,\n };\n\n setScorerSpanAttrs(scorerSpan, scorerName, failedResult, meta);\n\n scorerSpan.recordException(error);\n scorerSpan.setAttributes({\n [Attr.Error.Message]: error.message,\n [Attr.Error.Type]: error.name,\n });\n scorerSpan.setStatus({\n code: SpanStatusCode.ERROR,\n message: error.message,\n });\n\n return failedResult;\n } finally {\n scorerSpan.end();\n }\n });\n}\n","import {\n ATTR_ERROR_TYPE,\n ATTR_HTTP_RESPONSE_STATUS_CODE,\n} from '@opentelemetry/semantic-conventions';\n\nimport {\n ATTR_EVAL_CASE_INDEX,\n ATTR_EVAL_CASE_EXPECTED,\n ATTR_EVAL_CASE_INPUT,\n ATTR_EVAL_CASE_OUTPUT,\n ATTR_EVAL_CASE_METADATA,\n ATTR_EVAL_CASE_SCORES,\n ATTR_EVAL_CASE_TRIALS,\n ATTR_EVAL_SCORE_PASSED,\n ATTR_EVAL_SCORE_IS_BOOLEAN,\n ATTR_EVAL_SCORE_VALUE,\n ATTR_EVAL_SCORE_NAME,\n ATTR_EVAL_SCORE_THRESHOLD,\n ATTR_EVAL_SCORE_METADATA,\n ATTR_EVAL_SCORE_AGGREGATION,\n ATTR_EVAL_SCORE_TRIALS,\n ATTR_EVAL_TASK_TYPE,\n ATTR_EVAL_TASK_NAME,\n ATTR_EVAL_TASK_OUTPUT,\n ATTR_EVAL_COLLECTION_NAME,\n ATTR_EVAL_COLLECTION_SIZE,\n ATTR_EVAL_ID,\n ATTR_EVAL_BASELINE_ID,\n ATTR_EVAL_BASELINE_NAME,\n ATTR_EVAL_BASELINE_VERSION,\n ATTR_EVAL_NAME,\n ATTR_EVAL_TAGS,\n ATTR_EVAL_TYPE,\n ATTR_EVAL_TRIALS,\n ATTR_EVAL_TRIAL_INDEX,\n ATTR_EVAL_TRIAL_ERROR,\n ATTR_EVAL_COLLECTION_ID,\n ATTR_EVAL_USER_NAME,\n ATTR_EVAL_USER_EMAIL,\n ATTR_EVAL_VERSION,\n ATTR_EVAL_RUN_ID,\n ATTR_EVAL_METADATA,\n ATTR_EVAL_CONFIG_FLAGS,\n ATTR_EVAL_CAPABILITY_NAME,\n ATTR_EVAL_STEP_NAME,\n} from './eval_proposal';\n\nimport {\n ATTR_ERROR_MESSAGE,\n ATTR_GEN_AI_AGENT_DESCRIPTION,\n ATTR_GEN_AI_AGENT_ID,\n ATTR_GEN_AI_AGENT_NAME,\n ATTR_GEN_AI_CONVERSATION_ID,\n ATTR_GEN_AI_INPUT_MESSAGES,\n ATTR_GEN_AI_OPERATION_NAME,\n ATTR_GEN_AI_OUTPUT_MESSAGES,\n ATTR_GEN_AI_OUTPUT_TYPE,\n ATTR_GEN_AI_PROVIDER_NAME,\n ATTR_GEN_AI_REQUEST_CHOICE_COUNT,\n ATTR_GEN_AI_REQUEST_ENCODING_FORMATS,\n ATTR_GEN_AI_REQUEST_FREQUENCY_PENALTY,\n ATTR_GEN_AI_REQUEST_MAX_TOKENS,\n ATTR_GEN_AI_REQUEST_MODEL,\n ATTR_GEN_AI_REQUEST_PRESENCE_PENALTY,\n ATTR_GEN_AI_REQUEST_SEED,\n ATTR_GEN_AI_REQUEST_STOP_SEQUENCES,\n ATTR_GEN_AI_REQUEST_TEMPERATURE,\n ATTR_GEN_AI_REQUEST_TOP_K,\n ATTR_GEN_AI_REQUEST_TOP_P,\n ATTR_GEN_AI_RESPONSE_FINISH_REASONS,\n ATTR_GEN_AI_RESPONSE_ID,\n ATTR_GEN_AI_RESPONSE_MODEL,\n ATTR_GEN_AI_TOOL_CALL_ID,\n ATTR_GEN_AI_TOOL_DESCRIPTION,\n ATTR_GEN_AI_TOOL_NAME,\n ATTR_GEN_AI_TOOL_TYPE,\n ATTR_GEN_AI_USAGE_INPUT_TOKENS,\n ATTR_GEN_AI_USAGE_OUTPUT_TOKENS,\n GEN_AI_OPERATION_NAME_VALUE_CHAT,\n GEN_AI_OPERATION_NAME_VALUE_CREATE_AGENT,\n GEN_AI_OPERATION_NAME_VALUE_EMBEDDINGS,\n GEN_AI_OPERATION_NAME_VALUE_EXECUTE_TOOL,\n GEN_AI_OPERATION_NAME_VALUE_GENERATE_CONTENT,\n GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT,\n GEN_AI_OUTPUT_TYPE_VALUE_IMAGE,\n GEN_AI_OUTPUT_TYPE_VALUE_JSON,\n GEN_AI_OUTPUT_TYPE_VALUE_SPEECH,\n GEN_AI_OUTPUT_TYPE_VALUE_TEXT,\n GEN_AI_PROVIDER_NAME_VALUE_ANTHROPIC,\n GEN_AI_PROVIDER_NAME_VALUE_AWS_BEDROCK,\n GEN_AI_PROVIDER_NAME_VALUE_AZURE_AI_INFERENCE,\n GEN_AI_PROVIDER_NAME_VALUE_AZURE_AI_OPENAI,\n GEN_AI_PROVIDER_NAME_VALUE_COHERE,\n GEN_AI_PROVIDER_NAME_VALUE_DEEPSEEK,\n GEN_AI_PROVIDER_NAME_VALUE_GCP_GEMINI,\n GEN_AI_PROVIDER_NAME_VALUE_GCP_GEN_AI,\n GEN_AI_PROVIDER_NAME_VALUE_GCP_VERTEX_AI,\n GEN_AI_PROVIDER_NAME_VALUE_GROQ,\n GEN_AI_PROVIDER_NAME_VALUE_IBM_WATSONX_AI,\n GEN_AI_PROVIDER_NAME_VALUE_MISTRAL_AI,\n GEN_AI_PROVIDER_NAME_VALUE_OPENAI,\n GEN_AI_PROVIDER_NAME_VALUE_PERPLEXITY,\n GEN_AI_PROVIDER_NAME_VALUE_X_AI,\n} from '@opentelemetry/semantic-conventions/incubating';\n\n/\n PROPRIETARY ATTRIBUTES (o11y)\n \n @see: https://axiom.co/docs/ai-engineering/semantic-conventions\n /\n\nconst ATTR_AXIOM_GEN_AI_SCHEMA_URL = 'axiom.gen_ai.schema_url';\nconst ATTR_AXIOM_GEN_AI_SDK_NAME = 'axiom.gen_ai.sdk.name';\nconst ATTR_AXIOM_GEN_AI_SDK_VERSION = 'axiom.gen_ai.sdk.version';\nconst ATTR_GEN_AI_CAPABILITY_NAME = 'gen_ai.capability.name';\nconst ATTR_GEN_AI_STEP_NAME = 'gen_ai.step.name';\nconst ATTR_GEN_AI_TOOL_ARGUMENTS = 'gen_ai.tool.arguments'; // deprecated by OTel\nconst ATTR_GEN_AI_TOOL_MESSAGE = 'gen_ai.tool.message'; // deprecated by OTel\n\nconst GEN_AI_PROVIDER_NAME_VALUE_ASSEMBLYAI = 'assemblyai';\nconst GEN_AI_PROVIDER_NAME_VALUE_CEREBRAS = 'cerebras';\nconst GEN_AI_PROVIDER_NAME_VALUE_DEEPGRAM = 'deepgram';\nconst GEN_AI_PROVIDER_NAME_VALUE_DEEPINFRA = 'deepinfra';\nconst GEN_AI_PROVIDER_NAME_VALUE_ELEVENLABS = 'elevenlabs';\nconst GEN_AI_PROVIDER_NAME_VALUE_FAL = 'fal';\nconst GEN_AI_PROVIDER_NAME_VALUE_FIREWORKS = 'fireworks';\nconst GEN_AI_PROVIDER_NAME_VALUE_GLADIA = 'gladia';\nconst GEN_AI_PROVIDER_NAME_VALUE_HUME = 'hume';\nconst GEN_AI_PROVIDER_NAME_VALUE_LMNT = 'lmnt';\nconst GEN_AI_PROVIDER_NAME_VALUE_LUMA = 'luma';\nconst GEN_AI_PROVIDER_NAME_VALUE_REPLICATE = 'replicate';\nconst GEN_AI_PROVIDER_NAME_VALUE_REVAI = 'revai';\nconst GEN_AI_PROVIDER_NAME_VALUE_TOGETHERAI = 'togetherai';\nconst GEN_AI_PROVIDER_NAME_VALUE_VERCEL = 'vercel';\n\n/\n When adding something new here, please:\n * 1. Make sure it doesn't already exist as part of OTel Semantic Conventions (use that instead)\n * 2. Make sure to use standard naming schema, ie snake_case\n * 3. If a specific feature has an attribute you would like to use, extract it to the shared section\n \n Also Experimental Attributes should always be imported here and then used from the CustomAttributes object\n * because they are unstable.\n \n @see: https://github.com/open-telemetry/opentelemetry-js/tree/c89cb38d0fec39d54cf3fcb35c429a8129e9c909/semantic-conventions#unstable-semconv\n /\nexport const Attr = {\n __EXPERIMENTAL_Flag: (flagName: string) => `flag.${flagName}`,\n __EXPERIMENTAL_Fact: (factName: string) => `fact.${factName}`,\n Axiom: {\n GenAI: {\n SchemaURL: ATTR_AXIOM_GEN_AI_SCHEMA_URL,\n SDK: {\n Name: ATTR_AXIOM_GEN_AI_SDK_NAME,\n Version: ATTR_AXIOM_GEN_AI_SDK_VERSION,\n },\n },\n },\n GenAI: {\n PromptMetadata: {\n ID: 'axiom.gen_ai.prompt.id',\n Name: 'axiom.gen_ai.prompt.name',\n Slug: 'axiom.gen_ai.prompt.slug',\n Version: 'axiom.gen_ai.prompt.version',\n },\n /\n These two are used to identify the span\n /\n Capability: {\n Name: ATTR_GEN_AI_CAPABILITY_NAME,\n },\n Step: {\n Name: ATTR_GEN_AI_STEP_NAME,\n },\n Provider: {\n Name: ATTR_GEN_AI_PROVIDER_NAME,\n Name_Values: {\n Anthropic: GEN_AI_PROVIDER_NAME_VALUE_ANTHROPIC,\n AssemblyAI: GEN_AI_PROVIDER_NAME_VALUE_ASSEMBLYAI,\n AWSBedrock: GEN_AI_PROVIDER_NAME_VALUE_AWS_BEDROCK,\n AzureAIInference: GEN_AI_PROVIDER_NAME_VALUE_AZURE_AI_INFERENCE,\n AzureAIOpenAI: GEN_AI_PROVIDER_NAME_VALUE_AZURE_AI_OPENAI,\n Cerebras: GEN_AI_PROVIDER_NAME_VALUE_CEREBRAS,\n Cohere: GEN_AI_PROVIDER_NAME_VALUE_COHERE,\n Deepgram: GEN_AI_PROVIDER_NAME_VALUE_DEEPGRAM,\n DeepInfra: GEN_AI_PROVIDER_NAME_VALUE_DEEPINFRA,\n Deepseek: GEN_AI_PROVIDER_NAME_VALUE_DEEPSEEK,\n ElevenLabs: GEN_AI_PROVIDER_NAME_VALUE_ELEVENLABS,\n Fal: GEN_AI_PROVIDER_NAME_VALUE_FAL,\n Fireworks: GEN_AI_PROVIDER_NAME_VALUE_FIREWORKS,\n GCPGemini: GEN_AI_PROVIDER_NAME_VALUE_GCP_GEMINI,\n GCPGenAI: GEN_AI_PROVIDER_NAME_VALUE_GCP_GEN_AI,\n GCPVertexAI: GEN_AI_PROVIDER_NAME_VALUE_GCP_VERTEX_AI,\n Gladia: GEN_AI_PROVIDER_NAME_VALUE_GLADIA,\n Groq: GEN_AI_PROVIDER_NAME_VALUE_GROQ,\n Hume: GEN_AI_PROVIDER_NAME_VALUE_HUME,\n IBMWatsonxAI: GEN_AI_PROVIDER_NAME_VALUE_IBM_WATSONX_AI,\n Lmnt: GEN_AI_PROVIDER_NAME_VALUE_LMNT,\n Luma: GEN_AI_PROVIDER_NAME_VALUE_LUMA,\n MistralAI: GEN_AI_PROVIDER_NAME_VALUE_MISTRAL_AI,\n OpenAI: GEN_AI_PROVIDER_NAME_VALUE_OPENAI,\n Perplexity: GEN_AI_PROVIDER_NAME_VALUE_PERPLEXITY,\n Replicate: GEN_AI_PROVIDER_NAME_VALUE_REPLICATE,\n RevAI: GEN_AI_PROVIDER_NAME_VALUE_REVAI,\n TogetherAI: GEN_AI_PROVIDER_NAME_VALUE_TOGETHERAI,\n Vercel: GEN_AI_PROVIDER_NAME_VALUE_VERCEL,\n XAI: GEN_AI_PROVIDER_NAME_VALUE_X_AI,\n },\n },\n /\n Regular attributes\n /\n Agent: {\n Description: ATTR_GEN_AI_AGENT_DESCRIPTION, // not yet used by axiom-ai\n ID: ATTR_GEN_AI_AGENT_ID, // not yet used by axiom-ai\n Name: ATTR_GEN_AI_AGENT_NAME, // not yet used by axiom-ai\n },\n Conversation: {\n ID: ATTR_GEN_AI_CONVERSATION_ID, // not yet used by axiom-ai, anyway probably needs to be provided by user\n },\n Input: {\n Messages: ATTR_GEN_AI_INPUT_MESSAGES,\n },\n Operation: {\n Name: ATTR_GEN_AI_OPERATION_NAME,\n Name_Values: {\n /\n Note that \"text_completion\" is deprecated in favor of \"chat\" for both OpenAI and Anthropic\n /\n Chat: GEN_AI_OPERATION_NAME_VALUE_CHAT,\n CreateAgent: GEN_AI_OPERATION_NAME_VALUE_CREATE_AGENT,\n Embeddings: GEN_AI_OPERATION_NAME_VALUE_EMBEDDINGS,\n ExecuteTool: GEN_AI_OPERATION_NAME_VALUE_EXECUTE_TOOL,\n GenerateContent: GEN_AI_OPERATION_NAME_VALUE_GENERATE_CONTENT,\n InvokeAgent: GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT,\n },\n },\n Output: {\n Messages: ATTR_GEN_AI_OUTPUT_MESSAGES,\n Type: ATTR_GEN_AI_OUTPUT_TYPE,\n Type_Values: {\n Text: GEN_AI_OUTPUT_TYPE_VALUE_TEXT,\n Json: GEN_AI_OUTPUT_TYPE_VALUE_JSON,\n Image: GEN_AI_OUTPUT_TYPE_VALUE_IMAGE,\n Speech: GEN_AI_OUTPUT_TYPE_VALUE_SPEECH,\n },\n },\n /\n The provider that is hosting the model, eg AWS Bedrock\n * There doesn't seem to be a semconv for this\n /\n Request: {\n ChoiceCount: ATTR_GEN_AI_REQUEST_CHOICE_COUNT, // not yet used by axiom-ai\n EncodingFormats: ATTR_GEN_AI_REQUEST_ENCODING_FORMATS, // not yet used by axiom-ai\n FrequencyPenalty: ATTR_GEN_AI_REQUEST_FREQUENCY_PENALTY,\n MaxTokens: ATTR_GEN_AI_REQUEST_MAX_TOKENS,\n /\n The model you asked for\n /\n Model: ATTR_GEN_AI_REQUEST_MODEL,\n PresencePenalty: ATTR_GEN_AI_REQUEST_PRESENCE_PENALTY,\n Seed: ATTR_GEN_AI_REQUEST_SEED,\n StopSequences: ATTR_GEN_AI_REQUEST_STOP_SEQUENCES,\n Temperature: ATTR_GEN_AI_REQUEST_TEMPERATURE,\n TopK: ATTR_GEN_AI_REQUEST_TOP_K,\n TopP: ATTR_GEN_AI_REQUEST_TOP_P,\n },\n Response: {\n FinishReasons: ATTR_GEN_AI_RESPONSE_FINISH_REASONS,\n ID: ATTR_GEN_AI_RESPONSE_ID,\n /\n The model that was actually used (might be different bc routing) - only ever get this from the response, otherwise omit\n /\n Model: ATTR_GEN_AI_RESPONSE_MODEL, // somehow not landing on the span for google models? check up on this...\n },\n Tool: {\n CallID: ATTR_GEN_AI_TOOL_CALL_ID,\n Description: ATTR_GEN_AI_TOOL_DESCRIPTION,\n Name: ATTR_GEN_AI_TOOL_NAME,\n Type: ATTR_GEN_AI_TOOL_TYPE,\n /\n Note, OTel Semantic Convention suggest only putting tool inputs/outputs on the parent chat span\n * But we at least want to give users THE OPTION to put them on the tool spans themselves as well\n * Because it enables a lot of things with querying\n * @see https://github.com/open-telemetry/semantic-conventions/releases/tag/v1.37.0\n /\n Arguments: ATTR_GEN_AI_TOOL_ARGUMENTS,\n /\n Note, OTel Semantic Convention suggest only putting tool inputs/outputs on the parent chat span\n * But we at least want to give users THE OPTION to put them on the tool spans themselves as well\n * Because it enables a lot of things with querying\n * @see https://github.com/open-telemetry/semantic-conventions/releases/tag/v1.37.0\n /\n Message: ATTR_GEN_AI_TOOL_MESSAGE,\n },\n Usage: {\n InputTokens: ATTR_GEN_AI_USAGE_INPUT_TOKENS,\n OutputTokens: ATTR_GEN_AI_USAGE_OUTPUT_TOKENS,\n },\n },\n Eval: {\n ID: ATTR_EVAL_ID,\n Name: ATTR_EVAL_NAME,\n Version: ATTR_EVAL_VERSION,\n Type: ATTR_EVAL_TYPE,\n Trials: ATTR_EVAL_TRIALS,\n Baseline: {\n ID: ATTR_EVAL_BASELINE_ID,\n Name: ATTR_EVAL_BASELINE_NAME,\n Version: ATTR_EVAL_BASELINE_VERSION,\n },\n Capability: {\n Name: ATTR_EVAL_CAPABILITY_NAME,\n },\n Step: {\n Name: ATTR_EVAL_STEP_NAME,\n },\n Tags: ATTR_EVAL_TAGS,\n Metadata: ATTR_EVAL_METADATA,\n Collection: {\n ID: ATTR_EVAL_COLLECTION_ID,\n Name: ATTR_EVAL_COLLECTION_NAME,\n Size: ATTR_EVAL_COLLECTION_SIZE,\n },\n Config: {\n Flags: ATTR_EVAL_CONFIG_FLAGS,\n },\n Run: {\n ID: ATTR_EVAL_RUN_ID,\n },\n Trial: {\n Index: ATTR_EVAL_TRIAL_INDEX,\n Error: ATTR_EVAL_TRIAL_ERROR,\n },\n Case: {\n Index: ATTR_EVAL_CASE_INDEX,\n Input: ATTR_EVAL_CASE_INPUT,\n Output: ATTR_EVAL_CASE_OUTPUT,\n Expected: ATTR_EVAL_CASE_EXPECTED,\n Scores: ATTR_EVAL_CASE_SCORES,\n Metadata: ATTR_EVAL_CASE_METADATA,\n Trials: ATTR_EVAL_CASE_TRIALS,\n },\n Task: {\n Output: ATTR_EVAL_TASK_OUTPUT,\n Name: ATTR_EVAL_TASK_NAME,\n Type: ATTR_EVAL_TASK_TYPE,\n },\n Score: {\n Name: ATTR_EVAL_SCORE_NAME,\n Value: ATTR_EVAL_SCORE_VALUE,\n Threshold: ATTR_EVAL_SCORE_THRESHOLD,\n Passed: ATTR_EVAL_SCORE_PASSED,\n IsBoolean: ATTR_EVAL_SCORE_IS_BOOLEAN,\n Metadata: ATTR_EVAL_SCORE_METADATA,\n Aggregation: ATTR_EVAL_SCORE_AGGREGATION,\n Trials: ATTR_EVAL_SCORE_TRIALS,\n },\n User: {\n Name: ATTR_EVAL_USER_NAME,\n Email: ATTR_EVAL_USER_EMAIL,\n },\n },\n Error: {\n Type: ATTR_ERROR_TYPE,\n Message: ATTR_ERROR_MESSAGE,\n },\n HTTP: {\n Response: {\n StatusCode: ATTR_HTTP_RESPONSE_STATUS_CODE,\n },\n },\n} as const;\n","// experiment\nexport const ATTR_EVAL_ID = 'eval.id' as const;\nexport const ATTR_EVAL_NAME = 'eval.name' as const;\nexport const ATTR_EVAL_VERSION = 'eval.version' as const;\nexport const ATTR_EVAL_TYPE = 'eval.type' as const;\nexport const ATTR_EVAL_TAGS = 'eval.tags' as const;\nexport const ATTR_EVAL_BASELINE_ID = 'eval.baseline.id' as const;\nexport const ATTR_EVAL_BASELINE_NAME = 'eval.baseline.name' as const;\nexport const ATTR_EVAL_BASELINE_VERSION = 'eval.baseline.version' as const;\nexport const ATTR_EVAL_METADATA = 'eval.metadata' as const;\nexport const ATTR_EVAL_TRIALS = 'eval.trials' as const;\n// capability\nexport const ATTR_EVAL_CAPABILITY_NAME = 'eval.capability.name' as const;\nexport const ATTR_EVAL_STEP_NAME = 'eval.step.name' as const;\n// collection\nexport const ATTR_EVAL_COLLECTION_ID = 'eval.collection.id' as const;\nexport const ATTR_EVAL_COLLECTION_SIZE = 'eval.collection.size' as const;\nexport const ATTR_EVAL_COLLECTION_NAME = 'eval.collection.name' as const;\n// config\nexport const ATTR_EVAL_CONFIG_FLAGS = 'eval.config.flags' as const;\n// case\nexport const ATTR_EVAL_CASE_INDEX = 'eval.case.index' as const;\nexport const ATTR_EVAL_CASE_INPUT = 'eval.case.input' as const;\nexport const ATTR_EVAL_CASE_OUTPUT = 'eval.case.output' as const;\nexport const ATTR_EVAL_CASE_EXPECTED = 'eval.case.expected' as const;\nexport const ATTR_EVAL_CASE_SCORES = 'eval.case.scores' as const;\nexport const ATTR_EVAL_CASE_METADATA = 'eval.case.metadata' as const;\nexport const ATTR_EVAL_CASE_TRIALS = 'eval.case.trials' as const;\n// trial\nexport const ATTR_EVAL_TRIAL_INDEX = 'eval.trial.index' as const;\nexport const ATTR_EVAL_TRIAL_ERROR = 'eval.trial.error' as const;\n// task\nexport const ATTR_EVAL_TASK_OUTPUT = 'eval.task.output' as const;\nexport const ATTR_EVAL_TASK_NAME = 'eval.task.name' as const;\nexport const ATTR_EVAL_TASK_TYPE = 'eval.task.type' as const;\n// run\nexport const ATTR_EVAL_RUN_ID = 'eval.run.id' as const;\n// score\nexport const ATTR_EVAL_SCORE_NAME = 'eval.score.name' as const;\nexport const ATTR_EVAL_SCORE_VALUE = 'eval.score.value' as const;\nexport const ATTR_EVAL_SCORE_THRESHOLD = 'eval.score.threshold' as const;\nexport const ATTR_EVAL_SCORE_PASSED = 'eval.score.passed' as const;\nexport const ATTR_EVAL_SCORE_IS_BOOLEAN = 'eval.score.is_boolean' as const;\nexport const ATTR_EVAL_SCORE_SCORER = 'eval.score.scorer' as const;\nexport const ATTR_EVAL_SCORE_METADATA = 'eval.score.metadata' as const;\nexport const ATTR_EVAL_SCORE_AGGREGATION = 'eval.score.aggregation' as const;\nexport const ATTR_EVAL_SCORE_TRIALS = 'eval.score.trials' as const;\n// user\nexport const ATTR_EVAL_USER_NAME = 'eval.user.name';\nexport const ATTR_EVAL_USER_EMAIL = 'eval.user.email';\n","import { Attr } from '../otel/semconv/attributes';\nimport type { Score } from './scorer.types';\n\n/\n Normalizes a boolean score to numeric (1/0) and adds eval.score.is_boolean\n * to metadata. Numbers and null pass through unchanged.\n */\nexport function normalizeBooleanScore(\n score: Score['score'],\n metadata?: Record<string, unknown>,\n): { score: number \| null; metadata?: Record<string, unknown> } {\n if (typeof score !== 'boolean') {\n return { score, metadata };\n }\n return {\n score: score ? 1 : 0,\n metadata: { ...metadata, [Attr.Eval.Score.IsBoolean]: true },\n };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,IAAAA,cAAiE;;;ACCjE,IAAAC,cAAsB;;;ACDtB;AAAA,EACE,MAAQ;AAAA,EACR,SAAW;AAAA,EACX,MAAQ;AAAA,EACR,QAAU;AAAA,EACV,cAAgB;AAAA,IACd;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAAA,EACA,SAAW;AAAA,IACT,KAAO;AAAA,IACP,OAAS;AAAA,IACT,4BAA4B;AAAA,IAC5B,QAAU;AAAA,IACV,gBAAgB;AAAA,IAChB,MAAQ;AAAA,IACR,WAAa;AAAA,IACb,MAAQ;AAAA,IACR,cAAc;AAAA,IACd,SAAW;AAAA,EACb;AAAA,EACA,OAAS;AAAA,EACT,MAAQ;AAAA,EACR,QAAU;AAAA,EACV,KAAO;AAAA,IACL,OAAS;AAAA,EACX;AAAA,EACA,SAAW;AAAA,IACT,QAAQ;AAAA,MACN,QAAU;AAAA,QACR,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,MACA,SAAW;AAAA,QACT,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,IACF;AAAA,IACA,cAAc;AAAA,MACZ,QAAU;AAAA,QACR,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,MACA,SAAW;AAAA,QACT,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,IACF;AAAA,IACA,2BAA2B;AAAA,MACzB,QAAU;AAAA,QACR,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,MACA,SAAW;AAAA,QACT,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,IACF;AAAA,IACA,sBAAsB;AAAA,MACpB,QAAU;AAAA,QACR,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,MACA,SAAW;AAAA,QACT,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,IACF;AAAA,IACA,qBAAqB;AAAA,MACnB,QAAU;AAAA,QACR,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,MACA,SAAW;AAAA,QACT,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,IACF;AAAA,IACA,eAAe;AAAA,MACb,QAAU;AAAA,QACR,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,MACA,SAAW;AAAA,QACT,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,IACF;AAAA,IACA,iBAAiB;AAAA,MACf,QAAU;AAAA,QACR,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,MACA,SAAW;AAAA,QACT,OAAS;AAAA,QACT,SAAW;AAAA,MACb;AAAA,IACF;AAAA,EACF;AAAA,EACA,UAAY;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAAA,EACA,YAAc;AAAA,IACZ,MAAQ;AAAA,IACR,KAAO;AAAA,IACP,WAAa;AAAA,EACf;AAAA,EACA,SAAW;AAAA,EACX,cAAgB;AAAA,IACd,aAAa;AAAA,IACb,6CAA6C;AAAA,IAC7C,sCAAsC;AAAA,IACtC,2CAA2C;AAAA,IAC3C,4BAA4B;AAAA,IAC5B,iCAAiC;AAAA,IACjC,uCAAuC;AAAA,IACvC,qBAAqB;AAAA,IACrB,KAAO;AAAA,IACP,WAAa;AAAA,IACb,MAAQ;AAAA,IACR,YAAc;AAAA,IACd,QAAU;AAAA,IACV,MAAQ;AAAA,IACR,uBAAuB;AAAA,IACvB,QAAU;AAAA,EACZ;AAAA,EACA,kBAAoB;AAAA,IAClB,sBAAsB;AAAA,IACtB,KAAO;AAAA,EACT;AAAA,EACA,iBAAmB;AAAA,IACjB,uBAAuB;AAAA,IACvB,uBAAuB;AAAA,IACvB,uBAAuB;AAAA,IACvB,oBAAoB;AAAA,IACpB,oBAAoB;AAAA,IACpB,oBAAoB;AAAA,IACpB,sBAAsB;AAAA,IACtB,sBAAsB;AAAA,IACtB,sBAAsB;AAAA,IACtB,sBAAsB;AAAA,IACtB,uBAAuB;AAAA,IACvB,iCAAiC;AAAA,IACjC,iCAAiC;AAAA,IACjC,uBAAuB;AAAA,IACvB,eAAe;AAAA,IACf,uBAAuB;AAAA,IACvB,MAAQ;AAAA,IACR,MAAQ;AAAA,IACR,MAAQ;AAAA,IACR,SAAW;AAAA,IACX,QAAU;AAAA,IACV,KAAO;AAAA,IACP,UAAY;AAAA,IACZ,aAAe;AAAA,IACf,MAAQ;AAAA,IACR,YAAc;AAAA,IACd,QAAU;AAAA,IACV,KAAO;AAAA,EACT;AAAA,EACA,OAAS;AAAA,IACP;AAAA,EACF;AAAA,EACA,gBAAkB;AACpB;;;ACzKA,iBAA8D;AA8BvD,IAAM,yBAAyB,OAAO,IAAI,wBAAwB;;;AFxBzE,IAAM,qBAAqB,OAAO,IAAI,oBAAoB;AA2EnD,SAAS,kBAA0B;AAExC,QAAM,QAAS,WAAmB,kBAAkB;AAGpD,MAAI,CAAC,OAAO;AACV,UAAM,UAAU,QAAQ,IAAI,gBAAgB;AAC5C,QAAI,CAAC,SAAS;AACZ,cAAQ;AAAA,QACN;AAAA,MAGF;AAAA,IACF;AAAA,EACF;AAEA,MAAI,EAAE,MAAM,QAAQ,IAAI,SAAS,EAAE,MAAM,gBAAY,MAAM,SAAS,gBAAY,QAAQ;AAExF,MAAI,CAAC,QAAQ,CAAC,SAAS;AACrB,WAAO,gBAAY;AACnB,cAAU,gBAAY;AACtB,QAAI,CAAC,QAAQ,CAAC,SAAS;AACrB,aAAO;AACP,gBAAU;AAAA,IACZ;AAAA,EACF;AAGA,SAAO,kBAAM,UAAU,MAAM,OAAO;AACtC;;;AG9GA,IAAAC,cAA0D;;;ACA1D,kCAGO;;;ACFA,IAAM,eAAe;AACrB,IAAM,iBAAiB;AACvB,IAAM,oBAAoB;AAC1B,IAAM,iBAAiB;AACvB,IAAM,iBAAiB;AACvB,IAAM,wBAAwB;AAC9B,IAAM,0BAA0B;AAChC,IAAM,6BAA6B;AACnC,IAAM,qBAAqB;AAC3B,IAAM,mBAAmB;AAEzB,IAAM,4BAA4B;AAClC,IAAM,sBAAsB;AAE5B,IAAM,0BAA0B;AAChC,IAAM,4BAA4B;AAClC,IAAM,4BAA4B;AAElC,IAAM,yBAAyB;AAE/B,IAAM,uBAAuB;AAC7B,IAAM,uBAAuB;AAC7B,IAAM,wBAAwB;AAC9B,IAAM,0BAA0B;AAChC,IAAM,wBAAwB;AAC9B,IAAM,0BAA0B;AAChC,IAAM,wBAAwB;AAE9B,IAAM,wBAAwB;AAC9B,IAAM,wBAAwB;AAE9B,IAAM,wBAAwB;AAC9B,IAAM,sBAAsB;AAC5B,IAAM,sBAAsB;AAE5B,IAAM,mBAAmB;AAEzB,IAAM,uBAAuB;AAC7B,IAAM,wBAAwB;AAC9B,IAAM,4BAA4B;AAClC,IAAM,yBAAyB;AAC/B,IAAM,6BAA6B;AAEnC,IAAM,2BAA2B;AACjC,IAAM,8BAA8B;AACpC,IAAM,yBAAyB;AAE/B,IAAM,sBAAsB;AAC5B,IAAM,uBAAuB;;;ADFpC,wBAwDO;AAQP,IAAM,+BAA+B;AACrC,IAAM,6BAA6B;AACnC,IAAM,gCAAgC;AACtC,IAAM,8BAA8B;AACpC,IAAM,wBAAwB;AAC9B,IAAM,6BAA6B;AACnC,IAAM,2BAA2B;AAEjC,IAAM,wCAAwC;AAC9C,IAAM,sCAAsC;AAC5C,IAAM,sCAAsC;AAC5C,IAAM,uCAAuC;AAC7C,IAAM,wCAAwC;AAC9C,IAAM,iCAAiC;AACvC,IAAM,uCAAuC;AAC7C,IAAM,oCAAoC;AAC1C,IAAM,kCAAkC;AACxC,IAAM,kCAAkC;AACxC,IAAM,kCAAkC;AACxC,IAAM,uCAAuC;AAC7C,IAAM,mCAAmC;AACzC,IAAM,wCAAwC;AAC9C,IAAM,oCAAoC;AAanC,IAAM,OAAO;AAAA,EAClB,qBAAqB,CAAC,aAAqB,QAAQ,QAAQ;AAAA,EAC3D,qBAAqB,CAAC,aAAqB,QAAQ,QAAQ;AAAA,EAC3D,OAAO;AAAA,IACL,OAAO;AAAA,MACL,WAAW;AAAA,MACX,KAAK;AAAA,QACH,MAAM;AAAA,QACN,SAAS;AAAA,MACX;AAAA,IACF;AAAA,EACF;AAAA,EACA,OAAO;AAAA,IACL,gBAAgB;AAAA,MACd,IAAI;AAAA,MACJ,MAAM;AAAA,MACN,MAAM;AAAA,MACN,SAAS;AAAA,IACX;AAAA;AAAA;AAAA;AAAA,IAIA,YAAY;AAAA,MACV,MAAM;AAAA,IACR;AAAA,IACA,MAAM;AAAA,MACJ,MAAM;AAAA,IACR;AAAA,IACA,UAAU;AAAA,MACR,MAAM;AAAA,MACN,aAAa;AAAA,QACX,WAAW;AAAA,QACX,YAAY;AAAA,QACZ,YAAY;AAAA,QACZ,kBAAkB;AAAA,QAClB,eAAe;AAAA,QACf,UAAU;AAAA,QACV,QAAQ;AAAA,QACR,UAAU;AAAA,QACV,WAAW;AAAA,QACX,UAAU;AAAA,QACV,YAAY;AAAA,QACZ,KAAK;AAAA,QACL,WAAW;AAAA,QACX,WAAW;AAAA,QACX,UAAU;AAAA,QACV,aAAa;AAAA,QACb,QAAQ;AAAA,QACR,MAAM;AAAA,QACN,MAAM;AAAA,QACN,cAAc;AAAA,QACd,MAAM;AAAA,QACN,MAAM;AAAA,QACN,WAAW;AAAA,QACX,QAAQ;AAAA,QACR,YAAY;AAAA,QACZ,WAAW;AAAA,QACX,OAAO;AAAA,QACP,YAAY;AAAA,QACZ,QAAQ;AAAA,QACR,KAAK;AAAA,MACP;AAAA,IACF;AAAA;AAAA;AAAA;AAAA,IAIA,OAAO;AAAA,MACL,aAAa;AAAA;AAAA,MACb,IAAI;AAAA;AAAA,MACJ,MAAM;AAAA;AAAA,IACR;AAAA,IACA,cAAc;AAAA,MACZ,IAAI;AAAA;AAAA,IACN;AAAA,IACA,OAAO;AAAA,MACL,UAAU;AAAA,IACZ;AAAA,IACA,WAAW;AAAA,MACT,MAAM;AAAA,MACN,aAAa;AAAA;AAAA;AAAA;AAAA,QAIX,MAAM;AAAA,QACN,aAAa;AAAA,QACb,YAAY;AAAA,QACZ,aAAa;AAAA,QACb,iBAAiB;AAAA,QACjB,aAAa;AAAA,MACf;AAAA,IACF;AAAA,IACA,QAAQ;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,aAAa;AAAA,QACX,MAAM;AAAA,QACN,MAAM;AAAA,QACN,OAAO;AAAA,QACP,QAAQ;AAAA,MACV;AAAA,IACF;AAAA;AAAA;AAAA;AAAA;AAAA,IAKA,SAAS;AAAA,MACP,aAAa;AAAA;AAAA,MACb,iBAAiB;AAAA;AAAA,MACjB,kBAAkB;AAAA,MAClB,WAAW;AAAA;AAAA;AAAA;AAAA,MAIX,OAAO;AAAA,MACP,iBAAiB;AAAA,MACjB,MAAM;AAAA,MACN,eAAe;AAAA,MACf,aAAa;AAAA,MACb,MAAM;AAAA,MACN,MAAM;AAAA,IACR;AAAA,IACA,UAAU;AAAA,MACR,eAAe;AAAA,MACf,IAAI;AAAA;AAAA;AAAA;AAAA,MAIJ,OAAO;AAAA;AAAA,IACT;AAAA,IACA,MAAM;AAAA,MACJ,QAAQ;AAAA,MACR,aAAa;AAAA,MACb,MAAM;AAAA,MACN,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAON,WAAW;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAOX,SAAS;AAAA,IACX;AAAA,IACA,OAAO;AAAA,MACL,aAAa;AAAA,MACb,cAAc;AAAA,IAChB;AAAA,EACF;AAAA,EACA,MAAM;AAAA,IACJ,IAAI;AAAA,IACJ,MAAM;AAAA,IACN,SAAS;AAAA,IACT,MAAM;AAAA,IACN,QAAQ;AAAA,IACR,UAAU;AAAA,MACR,IAAI;AAAA,MACJ,MAAM;AAAA,MACN,SAAS;AAAA,IACX;AAAA,IACA,YAAY;AAAA,MACV,MAAM;AAAA,IACR;AAAA,IACA,MAAM;AAAA,MACJ,MAAM;AAAA,IACR;AAAA,IACA,MAAM;AAAA,IACN,UAAU;AAAA,IACV,YAAY;AAAA,MACV,IAAI;AAAA,MACJ,MAAM;AAAA,MACN,MAAM;AAAA,IACR;AAAA,IACA,QAAQ;AAAA,MACN,OAAO;AAAA,IACT;AAAA,IACA,KAAK;AAAA,MACH,IAAI;AAAA,IACN;AAAA,IACA,OAAO;AAAA,MACL,OAAO;AAAA,MACP,OAAO;AAAA,IACT;AAAA,IACA,MAAM;AAAA,MACJ,OAAO;AAAA,MACP,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,UAAU;AAAA,MACV,QAAQ;AAAA,MACR,UAAU;AAAA,MACV,QAAQ;AAAA,IACV;AAAA,IACA,MAAM;AAAA,MACJ,QAAQ;AAAA,MACR,MAAM;AAAA,MACN,MAAM;AAAA,IACR;AAAA,IACA,OAAO;AAAA,MACL,MAAM;AAAA,MACN,OAAO;AAAA,MACP,WAAW;AAAA,MACX,QAAQ;AAAA,MACR,WAAW;AAAA,MACX,UAAU;AAAA,MACV,aAAa;AAAA,MACb,QAAQ;AAAA,IACV;AAAA,IACA,MAAM;AAAA,MACJ,MAAM;AAAA,MACN,OAAO;AAAA,IACT;AAAA,EACF;AAAA,EACA,OAAO;AAAA,IACL,MAAM;AAAA,IACN,SAAS;AAAA,EACX;AAAA,EACA,MAAM;AAAA,IACJ,UAAU;AAAA,MACR,YAAY;AAAA,IACd;AAAA,EACF;AACF;;;AE7WO,SAAS,sBACd,OACA,UAC8D;AAC9D,MAAI,OAAO,UAAU,WAAW;AAC9B,WAAO,EAAE,OAAO,SAAS;AAAA,EAC3B;AACA,SAAO;AAAA,IACL,OAAO,QAAQ,IAAI;AAAA,IACnB,UAAU,EAAE,GAAG,UAAU,CAAC,KAAK,KAAK,MAAM,SAAS,GAAG,KAAK;AAAA,EAC7D;AACF;;;AHTA,SAAS,mBACP,YACA,YACA,QACA,MACM;AACN,QAAM,EAAE,OAAO,YAAY,SAAS,IAAI,sBAAsB,OAAO,OAAO,OAAO,QAAQ;AAE3F,QAAM,QAA+D;AAAA,IACnE,CAAC,KAAK,MAAM,UAAU,IAAI,GAAG;AAAA,IAC7B,CAAC,KAAK,KAAK,MAAM,IAAI,GAAG;AAAA,IACxB,CAAC,KAAK,KAAK,IAAI,GAAG,KAAK,UAAU,CAAC,QAAQ,CAAC;AAAA,IAC3C,CAAC,KAAK,KAAK,MAAM,KAAK,GAAG,cAAc;AAAA,IACvC,CAAC,KAAK,KAAK,WAAW,IAAI,GAAG,MAAM;AAAA,IACnC,CAAC,KAAK,KAAK,KAAK,IAAI,GAAG,MAAM;AAAA,EAC/B;AAEA,MAAI,YAAY,OAAO,KAAK,QAAQ,EAAE,SAAS,GAAG;AAChD,UAAM,KAAK,KAAK,MAAM,QAAQ,IAAI,KAAK,UAAU,QAAQ;AAAA,EAC3D;AAEA,aAAW,cAAc,KAAK;AAChC;AAKA,eAAsB,cACpB,QACA,OACA,QACA,YACA,MAC4B;AAC5B,QAAM,SAAS,gBAAgB;AAC/B,QAAM,gBAAgB,kBAAM,QAAQ,oBAAQ,OAAO,GAAG,UAAU;AAEhE,SAAO,oBAAQ,KAAK,eAAe,YAAY;AAC7C,UAAM,aACJ,OAAO,WAAW;AAAA;AAAA,MAEd,OAAO,QAAQ;AAAA,QACf,OAAO;AACb,UAAM,aAAa,OAAO,UAAU,SAAS,UAAU,EAAE;AAEzD,QAAI;AACF,YAAM,SACJ,OAAO,WAAW,aACb;AAAA,QACC,GAAI,MAAM,OAAO;AAAA,UACf;AAAA,UACA;AAAA,QACF,CAAC;AAAA,QACD,MAAM;AAAA,MACR,IACA;AAEN,yBAAmB,YAAY,YAAY,QAAQ,IAAI;AACvD,UAAI,OAAO,OAAO;AAChB,cAAM,QAAQ,IAAI,MAAM,OAAO,KAAK;AACpC,mBAAW,gBAAgB,KAAK;AAChC,mBAAW,cAAc;AAAA,UACvB,CAAC,KAAK,MAAM,OAAO,GAAG,MAAM;AAAA,UAC5B,CAAC,KAAK,MAAM,IAAI,GAAG,MAAM;AAAA,QAC3B,CAAC;AACD,mBAAW,UAAU;AAAA,UACnB,MAAM,2BAAe;AAAA,UACrB,SAAS,MAAM;AAAA,QACjB,CAAC;AAAA,MACH,OAAO;AACL,mBAAW,UAAU,EAAE,MAAM,2BAAe,GAAG,CAAC;AAAA,MAClD;AAEA,aAAO;AAAA,IACT,SAAS,KAAK;AACZ,YAAM,QAAQ,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC;AAChE,YAAM,eAA6B;AAAA,QACjC,MAAM;AAAA,QACN,OAAO;AAAA,QACP,OAAO,MAAM;AAAA,MACf;AAEA,yBAAmB,YAAY,YAAY,cAAc,IAAI;AAE7D,iBAAW,gBAAgB,KAAK;AAChC,iBAAW,cAAc;AAAA,QACvB,CAAC,KAAK,MAAM,OAAO,GAAG,MAAM;AAAA,QAC5B,CAAC,KAAK,MAAM,IAAI,GAAG,MAAM;AAAA,MAC3B,CAAC;AACD,iBAAW,UAAU;AAAA,QACnB,MAAM,2BAAe;AAAA,QACrB,SAAS,MAAM;AAAA,MACjB,CAAC;AAED,aAAO;AAAA,IACT,UAAE;AACA,iBAAW,IAAI;AAAA,IACjB;AAAA,EACF,CAAC;AACH;;;AJcA,eAAe,aACb,UACA,MACkB;AAClB,MAAI,aAAa,OAAW,QAAO;AACnC,MAAI,OAAO,aAAa,UAAU;AAChC,QAAI,YAAY,EAAG,QAAO;AAC1B,QAAI,YAAY,EAAG,QAAO;AAC1B,WAAO,KAAK,OAAO,IAAI;AAAA,EACzB;AACA,SAAO,QAAQ,MAAM,SAAS,IAAI,CAAC;AACrC;AAEA,SAAS,qBACP,OACwD;AACxD,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,YAAY;AACpE;AAEA,SAAS,kBAAmC,QAA8C;AACxF,MAAI,OAAO,WAAW,YAAY;AAChC,WAAO,OAAO,QAAQ;AAAA,EACxB;AACA,SAAO,OAAO;AAChB;AAEA,SAAS,qBACP,OACwC;AACxC,MAAI,qBAAqB,KAAK,GAAG;AAC/B,WAAO;AAAA,MACL,MAAM,kBAAkB,MAAM,MAAM;AAAA,MACpC,QAAQ,MAAM;AAAA,MACd,UAAU,MAAM;AAAA,IAClB;AAAA,EACF;AAEA,SAAO;AAAA,IACL,MAAM,kBAAkB,KAAK;AAAA,IAC7B,QAAQ;AAAA,EACV;AACF;AAEA,SAAS,wBACP,SACU;AACV,QAAM,aAAa,oBAAI,IAAoB;AAE3C,aAAW,SAAS,SAAS;AAC3B,eAAW,IAAI,MAAM,OAAO,WAAW,IAAI,MAAM,IAAI,KAAK,KAAK,CAAC;AAAA,EAClE;AAEA,SAAO,CAAC,GAAG,WAAW,QAAQ,CAAC,EAC5B,OAAO,CAAC,CAAC,EAAE,KAAK,MAAM,QAAQ,CAAC,EAC/B,IAAI,CAAC,CAAC,IAAI,MAAM,IAAI,EACpB,KAAK;AACV;AA4DO,SAAS,WAKd,MACA,SACgD;AAChD,MAAI,QAAQ,QAAQ,WAAW,GAAG;AAChC,WAAO,QAAQ,QAAQ,CAAC,CAAC;AAAA,EAC3B;AAEA,QAAM,WAAW,KAAK,SAAS,kBAAM,QAAQ,oBAAQ,OAAO,CAAC,GAAG,YAAY;AAC5E,QAAM,eAAe,WAAY,MAAM,QAAQ,QAAQ,IAAI,WAAW,CAAC,QAAQ,IAAK,CAAC;AAErF,SAAO,0BAA0B,MAAM,SAAS,YAAY;AAC9D;AAEA,eAAe,0BAKb,MACA,SACA,cACgD;AAChD,QAAM,SAAS,gBAAgB;AAE/B,QAAM,WAAW,KAAK,OAAO,QAAQ,KAAK,UAAU,IAAI,KAAK,IAAI,KAAK,QAAQ,KAAK,UAAU;AAE7F,QAAM,WAAW,OAAO;AAAA,IACtB;AAAA,IACA,aAAa,SAAS,IAAI,EAAE,OAAO,aAAa,IAAI,CAAC,SAAS,EAAE,SAAS,IAAI,EAAE,EAAE,IAAI,CAAC;AAAA,EACxF;AAEA,QAAM,YAAoC;AAAA,IACxC,CAAC,KAAK,MAAM,UAAU,IAAI,GAAG;AAAA,IAC7B,CAAC,KAAK,KAAK,WAAW,IAAI,GAAG,KAAK;AAAA,IAClC,CAAC,KAAK,KAAK,IAAI,GAAG,KAAK,UAAU,CAAC,QAAQ,CAAC;AAAA,EAC7C;AACA,MAAI,KAAK,MAAM;AACb,cAAU,KAAK,KAAK,KAAK,IAAI,IAAI,KAAK;AAAA,EACxC;AACA,WAAS,cAAc,SAAS;AAEhC,MAAI;AACF,UAAM,oBAAoB,QAAQ,QAAQ,IAAI,CAAC,UAAU,qBAAqB,KAAK,CAAC;AACpF,UAAM,uBAAuB,wBAAwB,iBAAiB;AACtE,QAAI,qBAAqB,SAAS,GAAG;AACnC,YAAM,IAAI;AAAA,QACR,2CAA2C,qBACxC,IAAI,CAAC,SAAS,IAAI,IAAI,GAAG,EACzB,KAAK,IAAI,CAAC;AAAA,MACf;AAAA,IACF;AAEA,UAAM,WAAW,MAAM,QAAQ;AAAA,MAC7B,kBAAkB,IAAI,OAAO,UAAU;AACrC,YAAI;AACF,gBAAM,YAAY,MAAM,aAAa,MAAM,UAAU;AAAA,YACnD,OAAO,QAAQ;AAAA,YACf,QAAQ,QAAQ;AAAA,UAClB,CAAC;AAED,cAAI,CAAC,WAAW;AACd,mBAAO,EAAE,YAAY,KAAc;AAAA,UACrC;AAEA,iBAAO;AAAA,YACL,YAAY;AAAA,YACZ,QAAQ,MAAM;AAAA,cACZ,MAAM;AAAA,cACN,QAAQ;AAAA,cACR,QAAQ;AAAA,cACR;AAAA,cACA;AAAA,YACF;AAAA,UACF;AAAA,QACF,SAAS,KAAK;AACZ,gBAAM,QAAQ,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC;AAChE,iBAAO;AAAA,YACL,YAAY;AAAA,YACZ,QAAQ,MAAM;AAAA,cACZ;AAAA,gBACE,MAAM,MAAM;AAAA,gBACZ,OAAO;AAAA,gBACP,OAAO,MAAM;AAAA,cACf;AAAA,cACA,QAAQ;AAAA,cACR,QAAQ;AAAA,cACR;AAAA,cACA;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAEA,UAAM,UAA2D,CAAC;AAClE,QAAI,kBAAkB;AAEtB,eAAW,WAAW,UAAU;AAC9B,UAAI,QAAQ,YAAY;AACtB,2BAAmB;AACnB;AAAA,MACF;AAEA,cAAQ,QAAQ,OAAO,IAAI,IAAI,QAAQ;AAAA,IACzC;AAEA,UAAM,cAAc,OAAO,OAAO,OAAO,EAAE,OAAO,CAAC,WAAW,OAAO,KAAK,EAAE;AAC5E,UAAM,WAAW,OAAO,KAAK,OAAO,EAAE;AAEtC,UAAM,gBAA8C,CAAC;AACrD,eAAW,CAAC,MAAM,MAAM,KAAK,OAAO,QAAQ,OAAO,GAAG;AACpD,oBAAc,IAAI,IAAI;AAAA,QACpB,MAAM,OAAO;AAAA,QACb,OAAO,OAAO;AAAA,QACd,GAAI,OAAO,YACT,OAAO,KAAK,OAAO,QAAQ,EAAE,SAAS,KAAK,EAAE,UAAU,OAAO,SAAS;AAAA,QACzE,GAAI,OAAO,SAAS,EAAE,OAAO,OAAO,MAAM;AAAA,MAC5C;AAAA,IACF;AAEA,aAAS,cAAc;AAAA,MACrB,CAAC,KAAK,KAAK,KAAK,MAAM,GAAG,KAAK,UAAU,aAAa;AAAA,MACrD,mCAAmC,kBAAkB;AAAA,MACrD,iCAAiC;AAAA,MACjC,yCAAyC;AAAA,MACzC,oCAAoC;AAAA,IACtC,CAAC;AAED,QAAI,cAAc,GAAG;AACnB,eAAS,UAAU;AAAA,QACjB,MAAM,2BAAe;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AAAA,IACH,OAAO;AACL,eAAS,UAAU,EAAE,MAAM,2BAAe,GAAG,CAAC;AAAA,IAChD;AAEA,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,UAAM,QAAQ,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC;AAChE,aAAS,gBAAgB,KAAK;AAC9B,aAAS,UAAU;AAAA,MACjB,MAAM,2BAAe;AAAA,MACrB,SAAS,MAAM;AAAA,IACjB,CAAC;AACD,WAAO,CAAC;AAAA,EACV,UAAE;AACA,aAAS,IAAI;AAAA,EACf;AACF;","names":["import_api","import_api","import_api"]}

package/dist/evals/online.d.cts CHANGED Viewed

@@ -55,12 +55,13 @@ type OnlineEvalMeta = {
     /** Specific step within the capability (e.g., 'answer', 'extract') */
     step?: string;
     /**
-     * Explicit SpanContext to link the eval span to the originating generation span.
+     * Explicit SpanContext(s) to link the eval span to originating generation span(s).
      * When omitted, the active span's context is used automatically.
      * Use this for deferred evaluation when onlineEval is called after the
      * originating span has completed.
+     * Supports both single context and multiple contexts for multi-span linking.
      */
-    link?: SpanContext;
+    links?: SpanContext | SpanContext[];
 };
 /**
  * Options for online evaluation.
@@ -102,13 +103,17 @@ type OnlineEvalOptions<TInput, TOutput, TScorers extends readonly ScorerEntry<TI
  *
  * **Deferred evaluation with explicit link:**
  * Pass the originating span's context for linking when evaluating later.
+ * Supports single or multiple span contexts.
  * ```ts
  * let spanCtx: SpanContext;
  * const result = await withSpan({ ... }, async (span) => {
  *   spanCtx = span.spanContext();
  *   return await generateText({ ... });
  * });
- * void onlineEval({ ..., link: spanCtx }, { output: result, scorers });
+ * void onlineEval({ ..., links: spanCtx }, { output: result, scorers });
+ *
+ * // Or link to multiple spans:
+ * void onlineEval({ ..., links: [spanCtx1, spanCtx2] }, { output, scorers });
  * ```
  *
  * **Awaiting for flush (short-lived processes):**
@@ -120,7 +125,7 @@ type OnlineEvalOptions<TInput, TOutput, TScorers extends readonly ScorerEntry<TI
  * @param meta - Evaluation metadata for categorization
  * @param meta.capability - High-level capability being evaluated
  * @param meta.step - Optional step within the capability
- * @param meta.link - Optional SpanContext to link to (auto-detected if omitted)
+ * @param meta.links - Optional SpanContext(s) to link to (auto-detected if omitted)
  * @param options - Evaluation configuration
  * @param options.input - Input to pass to scorers
  * @param options.output - Output to evaluate

package/dist/evals/online.d.ts CHANGED Viewed

@@ -55,12 +55,13 @@ type OnlineEvalMeta = {
     /** Specific step within the capability (e.g., 'answer', 'extract') */
     step?: string;
     /**
-     * Explicit SpanContext to link the eval span to the originating generation span.
+     * Explicit SpanContext(s) to link the eval span to originating generation span(s).
      * When omitted, the active span's context is used automatically.
      * Use this for deferred evaluation when onlineEval is called after the
      * originating span has completed.
+     * Supports both single context and multiple contexts for multi-span linking.
      */
-    link?: SpanContext;
+    links?: SpanContext | SpanContext[];
 };
 /**
  * Options for online evaluation.
@@ -102,13 +103,17 @@ type OnlineEvalOptions<TInput, TOutput, TScorers extends readonly ScorerEntry<TI
  *
  * **Deferred evaluation with explicit link:**
  * Pass the originating span's context for linking when evaluating later.
+ * Supports single or multiple span contexts.
  * ```ts
  * let spanCtx: SpanContext;
  * const result = await withSpan({ ... }, async (span) => {
  *   spanCtx = span.spanContext();
  *   return await generateText({ ... });
  * });
- * void onlineEval({ ..., link: spanCtx }, { output: result, scorers });
+ * void onlineEval({ ..., links: spanCtx }, { output: result, scorers });
+ *
+ * // Or link to multiple spans:
+ * void onlineEval({ ..., links: [spanCtx1, spanCtx2] }, { output, scorers });
  * ```
  *
  * **Awaiting for flush (short-lived processes):**
@@ -120,7 +125,7 @@ type OnlineEvalOptions<TInput, TOutput, TScorers extends readonly ScorerEntry<TI
  * @param meta - Evaluation metadata for categorization
  * @param meta.capability - High-level capability being evaluated
  * @param meta.step - Optional step within the capability
- * @param meta.link - Optional SpanContext to link to (auto-detected if omitted)
+ * @param meta.links - Optional SpanContext(s) to link to (auto-detected if omitted)
  * @param options - Evaluation configuration
  * @param options.input - Input to pass to scorers
  * @param options.output - Output to evaluate

package/dist/evals/online.js CHANGED Viewed

@@ -1,7 +1,8 @@
 import {
   onlineEval
-} from "../chunk-AAYRMTT6.js";
-import "../chunk-HW6E4M7S.js";
+} from "../chunk-SGAIMXRV.js";
+import "../chunk-N3RHEQMQ.js";
+import "../chunk-3Q36GUCI.js";
 import "../chunk-TWE3LIRZ.js";
 import {
   init_esm_shims

package/dist/evals/online.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"sources":["../../src/evals/online.ts"],"sourcesContent":["export { onlineEval } from '../online-evals/onlineEval';\nexport type { OnlineEvalMeta, OnlineEvalOptions } from '../online-evals/onlineEval';\nexport type {\n OnlineEvalScorerEntry,\n OnlineEvalScorerInput,\n SampledOnlineEvalScorer,\n ScorerSampling,\n} from '../online-evals/types';\n"],"mappings":"~~;;;;;;;;;;~~AAAA;","names":[]}
1	+ {"version":3,"sources":["../../src/evals/online.ts"],"sourcesContent":["export { onlineEval } from '../online-evals/onlineEval';\nexport type { OnlineEvalMeta, OnlineEvalOptions } from '../online-evals/onlineEval';\nexport type {\n OnlineEvalScorerEntry,\n OnlineEvalScorerInput,\n SampledOnlineEvalScorer,\n ScorerSampling,\n} from '../online-evals/types';\n"],"mappings":";;;;;;;;;;;AAAA;","names":[]}

package/dist/evals/scorers.cjs CHANGED Viewed

@@ -327,6 +327,17 @@ var Attr = {
   }
 };
+// src/evals/normalize-score.ts
+function normalizeBooleanScore(score, metadata) {
+  if (typeof score !== "boolean") {
+    return { score, metadata };
+  }
+  return {
+    score: score ? 1 : 0,
+    metadata: { ...metadata, [Attr.Eval.Score.IsBoolean]: true }
+  };
+}
 // src/evals/scorer.factory.ts
 function createScorer(name, fn, options) {
   const normalizeScore = (res) => {
@@ -334,23 +345,9 @@ function createScorer(name, fn, options) {
       return { score: res };
     }
     if (typeof res === "boolean") {
-      return {
-        score: res ? 1 : 0,
-        metadata: {
-          [Attr.Eval.Score.IsBoolean]: true
-        }
-      };
-    }
-    if (typeof res.score === "boolean") {
-      return {
-        score: res.score ? 1 : 0,
-        metadata: {
-          ...res.metadata,
-          [Attr.Eval.Score.IsBoolean]: true
-        }
-      };
+      return normalizeBooleanScore(res);
     }
-    return res;
+    return normalizeBooleanScore(res.score, res.metadata);
   };
   const scorer = (args) => {
     const res = fn(args);

package/dist/evals/scorers.cjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"sources":["../../src/evals/scorers.ts","../../src/otel/semconv/attributes.ts","../../src/otel/semconv/eval_proposal.ts","../../src/evals/scorer.factory.ts"],"sourcesContent":["export type { Score, ScoreWithName, ScorerOptions, ScorerLike } from './scorer.types';\nexport type { Scorer as ScorerType } from './scorer.types';\nexport { createScorer as Scorer } from './scorer.factory';\n","import {\n ATTR_ERROR_TYPE,\n ATTR_HTTP_RESPONSE_STATUS_CODE,\n} from '@opentelemetry/semantic-conventions';\n\nimport {\n ATTR_EVAL_CASE_INDEX,\n ATTR_EVAL_CASE_EXPECTED,\n ATTR_EVAL_CASE_INPUT,\n ATTR_EVAL_CASE_OUTPUT,\n ATTR_EVAL_CASE_METADATA,\n ATTR_EVAL_CASE_SCORES,\n ATTR_EVAL_CASE_TRIALS,\n ATTR_EVAL_SCORE_PASSED,\n ATTR_EVAL_SCORE_IS_BOOLEAN,\n ATTR_EVAL_SCORE_VALUE,\n ATTR_EVAL_SCORE_NAME,\n ATTR_EVAL_SCORE_THRESHOLD,\n ATTR_EVAL_SCORE_METADATA,\n ATTR_EVAL_SCORE_AGGREGATION,\n ATTR_EVAL_SCORE_TRIALS,\n ATTR_EVAL_TASK_TYPE,\n ATTR_EVAL_TASK_NAME,\n ATTR_EVAL_TASK_OUTPUT,\n ATTR_EVAL_COLLECTION_NAME,\n ATTR_EVAL_COLLECTION_SIZE,\n ATTR_EVAL_ID,\n ATTR_EVAL_BASELINE_ID,\n ATTR_EVAL_BASELINE_NAME,\n ATTR_EVAL_BASELINE_VERSION,\n ATTR_EVAL_NAME,\n ATTR_EVAL_TAGS,\n ATTR_EVAL_TYPE,\n ATTR_EVAL_TRIALS,\n ATTR_EVAL_TRIAL_INDEX,\n ATTR_EVAL_TRIAL_ERROR,\n ATTR_EVAL_COLLECTION_ID,\n ATTR_EVAL_USER_NAME,\n ATTR_EVAL_USER_EMAIL,\n ATTR_EVAL_VERSION,\n ATTR_EVAL_RUN_ID,\n ATTR_EVAL_METADATA,\n ATTR_EVAL_CONFIG_FLAGS,\n ATTR_EVAL_CAPABILITY_NAME,\n ATTR_EVAL_STEP_NAME,\n} from './eval_proposal';\n\nimport {\n ATTR_ERROR_MESSAGE,\n ATTR_GEN_AI_AGENT_DESCRIPTION,\n ATTR_GEN_AI_AGENT_ID,\n ATTR_GEN_AI_AGENT_NAME,\n ATTR_GEN_AI_CONVERSATION_ID,\n ATTR_GEN_AI_INPUT_MESSAGES,\n ATTR_GEN_AI_OPERATION_NAME,\n ATTR_GEN_AI_OUTPUT_MESSAGES,\n ATTR_GEN_AI_OUTPUT_TYPE,\n ATTR_GEN_AI_PROVIDER_NAME,\n ATTR_GEN_AI_REQUEST_CHOICE_COUNT,\n ATTR_GEN_AI_REQUEST_ENCODING_FORMATS,\n ATTR_GEN_AI_REQUEST_FREQUENCY_PENALTY,\n ATTR_GEN_AI_REQUEST_MAX_TOKENS,\n ATTR_GEN_AI_REQUEST_MODEL,\n ATTR_GEN_AI_REQUEST_PRESENCE_PENALTY,\n ATTR_GEN_AI_REQUEST_SEED,\n ATTR_GEN_AI_REQUEST_STOP_SEQUENCES,\n ATTR_GEN_AI_REQUEST_TEMPERATURE,\n ATTR_GEN_AI_REQUEST_TOP_K,\n ATTR_GEN_AI_REQUEST_TOP_P,\n ATTR_GEN_AI_RESPONSE_FINISH_REASONS,\n ATTR_GEN_AI_RESPONSE_ID,\n ATTR_GEN_AI_RESPONSE_MODEL,\n ATTR_GEN_AI_TOOL_CALL_ID,\n ATTR_GEN_AI_TOOL_DESCRIPTION,\n ATTR_GEN_AI_TOOL_NAME,\n ATTR_GEN_AI_TOOL_TYPE,\n ATTR_GEN_AI_USAGE_INPUT_TOKENS,\n ATTR_GEN_AI_USAGE_OUTPUT_TOKENS,\n GEN_AI_OPERATION_NAME_VALUE_CHAT,\n GEN_AI_OPERATION_NAME_VALUE_CREATE_AGENT,\n GEN_AI_OPERATION_NAME_VALUE_EMBEDDINGS,\n GEN_AI_OPERATION_NAME_VALUE_EXECUTE_TOOL,\n GEN_AI_OPERATION_NAME_VALUE_GENERATE_CONTENT,\n GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT,\n GEN_AI_OUTPUT_TYPE_VALUE_IMAGE,\n GEN_AI_OUTPUT_TYPE_VALUE_JSON,\n GEN_AI_OUTPUT_TYPE_VALUE_SPEECH,\n GEN_AI_OUTPUT_TYPE_VALUE_TEXT,\n GEN_AI_PROVIDER_NAME_VALUE_ANTHROPIC,\n GEN_AI_PROVIDER_NAME_VALUE_AWS_BEDROCK,\n GEN_AI_PROVIDER_NAME_VALUE_AZURE_AI_INFERENCE,\n GEN_AI_PROVIDER_NAME_VALUE_AZURE_AI_OPENAI,\n GEN_AI_PROVIDER_NAME_VALUE_COHERE,\n GEN_AI_PROVIDER_NAME_VALUE_DEEPSEEK,\n GEN_AI_PROVIDER_NAME_VALUE_GCP_GEMINI,\n GEN_AI_PROVIDER_NAME_VALUE_GCP_GEN_AI,\n GEN_AI_PROVIDER_NAME_VALUE_GCP_VERTEX_AI,\n GEN_AI_PROVIDER_NAME_VALUE_GROQ,\n GEN_AI_PROVIDER_NAME_VALUE_IBM_WATSONX_AI,\n GEN_AI_PROVIDER_NAME_VALUE_MISTRAL_AI,\n GEN_AI_PROVIDER_NAME_VALUE_OPENAI,\n GEN_AI_PROVIDER_NAME_VALUE_PERPLEXITY,\n GEN_AI_PROVIDER_NAME_VALUE_X_AI,\n} from '@opentelemetry/semantic-conventions/incubating';\n\n/*\n PROPRIETARY ATTRIBUTES (o11y)\n \n @see: https://axiom.co/docs/ai-engineering/semantic-conventions\n /\n\nconst ATTR_AXIOM_GEN_AI_SCHEMA_URL = 'axiom.gen_ai.schema_url';\nconst ATTR_AXIOM_GEN_AI_SDK_NAME = 'axiom.gen_ai.sdk.name';\nconst ATTR_AXIOM_GEN_AI_SDK_VERSION = 'axiom.gen_ai.sdk.version';\nconst ATTR_GEN_AI_CAPABILITY_NAME = 'gen_ai.capability.name';\nconst ATTR_GEN_AI_STEP_NAME = 'gen_ai.step.name';\nconst ATTR_GEN_AI_TOOL_ARGUMENTS = 'gen_ai.tool.arguments'; // deprecated by OTel\nconst ATTR_GEN_AI_TOOL_MESSAGE = 'gen_ai.tool.message'; // deprecated by OTel\n\nconst GEN_AI_PROVIDER_NAME_VALUE_ASSEMBLYAI = 'assemblyai';\nconst GEN_AI_PROVIDER_NAME_VALUE_CEREBRAS = 'cerebras';\nconst GEN_AI_PROVIDER_NAME_VALUE_DEEPGRAM = 'deepgram';\nconst GEN_AI_PROVIDER_NAME_VALUE_DEEPINFRA = 'deepinfra';\nconst GEN_AI_PROVIDER_NAME_VALUE_ELEVENLABS = 'elevenlabs';\nconst GEN_AI_PROVIDER_NAME_VALUE_FAL = 'fal';\nconst GEN_AI_PROVIDER_NAME_VALUE_FIREWORKS = 'fireworks';\nconst GEN_AI_PROVIDER_NAME_VALUE_GLADIA = 'gladia';\nconst GEN_AI_PROVIDER_NAME_VALUE_HUME = 'hume';\nconst GEN_AI_PROVIDER_NAME_VALUE_LMNT = 'lmnt';\nconst GEN_AI_PROVIDER_NAME_VALUE_LUMA = 'luma';\nconst GEN_AI_PROVIDER_NAME_VALUE_REPLICATE = 'replicate';\nconst GEN_AI_PROVIDER_NAME_VALUE_REVAI = 'revai';\nconst GEN_AI_PROVIDER_NAME_VALUE_TOGETHERAI = 'togetherai';\nconst GEN_AI_PROVIDER_NAME_VALUE_VERCEL = 'vercel';\n\n/\n When adding something new here, please:\n * 1. Make sure it doesn't already exist as part of OTel Semantic Conventions (use that instead)\n * 2. Make sure to use standard naming schema, ie snake_case\n * 3. If a specific feature has an attribute you would like to use, extract it to the shared section\n \n Also Experimental Attributes should always be imported here and then used from the CustomAttributes object\n * because they are unstable.\n \n @see: https://github.com/open-telemetry/opentelemetry-js/tree/c89cb38d0fec39d54cf3fcb35c429a8129e9c909/semantic-conventions#unstable-semconv\n /\nexport const Attr = {\n __EXPERIMENTAL_Flag: (flagName: string) => `flag.${flagName}`,\n __EXPERIMENTAL_Fact: (factName: string) => `fact.${factName}`,\n Axiom: {\n GenAI: {\n SchemaURL: ATTR_AXIOM_GEN_AI_SCHEMA_URL,\n SDK: {\n Name: ATTR_AXIOM_GEN_AI_SDK_NAME,\n Version: ATTR_AXIOM_GEN_AI_SDK_VERSION,\n },\n },\n },\n GenAI: {\n PromptMetadata: {\n ID: 'axiom.gen_ai.prompt.id',\n Name: 'axiom.gen_ai.prompt.name',\n Slug: 'axiom.gen_ai.prompt.slug',\n Version: 'axiom.gen_ai.prompt.version',\n },\n /\n These two are used to identify the span\n /\n Capability: {\n Name: ATTR_GEN_AI_CAPABILITY_NAME,\n },\n Step: {\n Name: ATTR_GEN_AI_STEP_NAME,\n },\n Provider: {\n Name: ATTR_GEN_AI_PROVIDER_NAME,\n Name_Values: {\n Anthropic: GEN_AI_PROVIDER_NAME_VALUE_ANTHROPIC,\n AssemblyAI: GEN_AI_PROVIDER_NAME_VALUE_ASSEMBLYAI,\n AWSBedrock: GEN_AI_PROVIDER_NAME_VALUE_AWS_BEDROCK,\n AzureAIInference: GEN_AI_PROVIDER_NAME_VALUE_AZURE_AI_INFERENCE,\n AzureAIOpenAI: GEN_AI_PROVIDER_NAME_VALUE_AZURE_AI_OPENAI,\n Cerebras: GEN_AI_PROVIDER_NAME_VALUE_CEREBRAS,\n Cohere: GEN_AI_PROVIDER_NAME_VALUE_COHERE,\n Deepgram: GEN_AI_PROVIDER_NAME_VALUE_DEEPGRAM,\n DeepInfra: GEN_AI_PROVIDER_NAME_VALUE_DEEPINFRA,\n Deepseek: GEN_AI_PROVIDER_NAME_VALUE_DEEPSEEK,\n ElevenLabs: GEN_AI_PROVIDER_NAME_VALUE_ELEVENLABS,\n Fal: GEN_AI_PROVIDER_NAME_VALUE_FAL,\n Fireworks: GEN_AI_PROVIDER_NAME_VALUE_FIREWORKS,\n GCPGemini: GEN_AI_PROVIDER_NAME_VALUE_GCP_GEMINI,\n GCPGenAI: GEN_AI_PROVIDER_NAME_VALUE_GCP_GEN_AI,\n GCPVertexAI: GEN_AI_PROVIDER_NAME_VALUE_GCP_VERTEX_AI,\n Gladia: GEN_AI_PROVIDER_NAME_VALUE_GLADIA,\n Groq: GEN_AI_PROVIDER_NAME_VALUE_GROQ,\n Hume: GEN_AI_PROVIDER_NAME_VALUE_HUME,\n IBMWatsonxAI: GEN_AI_PROVIDER_NAME_VALUE_IBM_WATSONX_AI,\n Lmnt: GEN_AI_PROVIDER_NAME_VALUE_LMNT,\n Luma: GEN_AI_PROVIDER_NAME_VALUE_LUMA,\n MistralAI: GEN_AI_PROVIDER_NAME_VALUE_MISTRAL_AI,\n OpenAI: GEN_AI_PROVIDER_NAME_VALUE_OPENAI,\n Perplexity: GEN_AI_PROVIDER_NAME_VALUE_PERPLEXITY,\n Replicate: GEN_AI_PROVIDER_NAME_VALUE_REPLICATE,\n RevAI: GEN_AI_PROVIDER_NAME_VALUE_REVAI,\n TogetherAI: GEN_AI_PROVIDER_NAME_VALUE_TOGETHERAI,\n Vercel: GEN_AI_PROVIDER_NAME_VALUE_VERCEL,\n XAI: GEN_AI_PROVIDER_NAME_VALUE_X_AI,\n },\n },\n /\n Regular attributes\n /\n Agent: {\n Description: ATTR_GEN_AI_AGENT_DESCRIPTION, // not yet used by axiom-ai\n ID: ATTR_GEN_AI_AGENT_ID, // not yet used by axiom-ai\n Name: ATTR_GEN_AI_AGENT_NAME, // not yet used by axiom-ai\n },\n Conversation: {\n ID: ATTR_GEN_AI_CONVERSATION_ID, // not yet used by axiom-ai, anyway probably needs to be provided by user\n },\n Input: {\n Messages: ATTR_GEN_AI_INPUT_MESSAGES,\n },\n Operation: {\n Name: ATTR_GEN_AI_OPERATION_NAME,\n Name_Values: {\n /\n Note that \"text_completion\" is deprecated in favor of \"chat\" for both OpenAI and Anthropic\n /\n Chat: GEN_AI_OPERATION_NAME_VALUE_CHAT,\n CreateAgent: GEN_AI_OPERATION_NAME_VALUE_CREATE_AGENT,\n Embeddings: GEN_AI_OPERATION_NAME_VALUE_EMBEDDINGS,\n ExecuteTool: GEN_AI_OPERATION_NAME_VALUE_EXECUTE_TOOL,\n GenerateContent: GEN_AI_OPERATION_NAME_VALUE_GENERATE_CONTENT,\n InvokeAgent: GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT,\n },\n },\n Output: {\n Messages: ATTR_GEN_AI_OUTPUT_MESSAGES,\n Type: ATTR_GEN_AI_OUTPUT_TYPE,\n Type_Values: {\n Text: GEN_AI_OUTPUT_TYPE_VALUE_TEXT,\n Json: GEN_AI_OUTPUT_TYPE_VALUE_JSON,\n Image: GEN_AI_OUTPUT_TYPE_VALUE_IMAGE,\n Speech: GEN_AI_OUTPUT_TYPE_VALUE_SPEECH,\n },\n },\n /\n The provider that is hosting the model, eg AWS Bedrock\n * There doesn't seem to be a semconv for this\n /\n Request: {\n ChoiceCount: ATTR_GEN_AI_REQUEST_CHOICE_COUNT, // not yet used by axiom-ai\n EncodingFormats: ATTR_GEN_AI_REQUEST_ENCODING_FORMATS, // not yet used by axiom-ai\n FrequencyPenalty: ATTR_GEN_AI_REQUEST_FREQUENCY_PENALTY,\n MaxTokens: ATTR_GEN_AI_REQUEST_MAX_TOKENS,\n /\n The model you asked for\n /\n Model: ATTR_GEN_AI_REQUEST_MODEL,\n PresencePenalty: ATTR_GEN_AI_REQUEST_PRESENCE_PENALTY,\n Seed: ATTR_GEN_AI_REQUEST_SEED,\n StopSequences: ATTR_GEN_AI_REQUEST_STOP_SEQUENCES,\n Temperature: ATTR_GEN_AI_REQUEST_TEMPERATURE,\n TopK: ATTR_GEN_AI_REQUEST_TOP_K,\n TopP: ATTR_GEN_AI_REQUEST_TOP_P,\n },\n Response: {\n FinishReasons: ATTR_GEN_AI_RESPONSE_FINISH_REASONS,\n ID: ATTR_GEN_AI_RESPONSE_ID,\n /\n The model that was actually used (might be different bc routing) - only ever get this from the response, otherwise omit\n /\n Model: ATTR_GEN_AI_RESPONSE_MODEL, // somehow not landing on the span for google models? check up on this...\n },\n Tool: {\n CallID: ATTR_GEN_AI_TOOL_CALL_ID,\n Description: ATTR_GEN_AI_TOOL_DESCRIPTION,\n Name: ATTR_GEN_AI_TOOL_NAME,\n Type: ATTR_GEN_AI_TOOL_TYPE,\n /\n Note, OTel Semantic Convention suggest only putting tool inputs/outputs on the parent chat span\n * But we at least want to give users THE OPTION to put them on the tool spans themselves as well\n * Because it enables a lot of things with querying\n * @see https://github.com/open-telemetry/semantic-conventions/releases/tag/v1.37.0\n /\n Arguments: ATTR_GEN_AI_TOOL_ARGUMENTS,\n /\n Note, OTel Semantic Convention suggest only putting tool inputs/outputs on the parent chat span\n * But we at least want to give users THE OPTION to put them on the tool spans themselves as well\n * Because it enables a lot of things with querying\n * @see https://github.com/open-telemetry/semantic-conventions/releases/tag/v1.37.0\n /\n Message: ATTR_GEN_AI_TOOL_MESSAGE,\n },\n Usage: {\n InputTokens: ATTR_GEN_AI_USAGE_INPUT_TOKENS,\n OutputTokens: ATTR_GEN_AI_USAGE_OUTPUT_TOKENS,\n },\n },\n Eval: {\n ID: ATTR_EVAL_ID,\n Name: ATTR_EVAL_NAME,\n Version: ATTR_EVAL_VERSION,\n Type: ATTR_EVAL_TYPE,\n Trials: ATTR_EVAL_TRIALS,\n Baseline: {\n ID: ATTR_EVAL_BASELINE_ID,\n Name: ATTR_EVAL_BASELINE_NAME,\n Version: ATTR_EVAL_BASELINE_VERSION,\n },\n Capability: {\n Name: ATTR_EVAL_CAPABILITY_NAME,\n },\n Step: {\n Name: ATTR_EVAL_STEP_NAME,\n },\n Tags: ATTR_EVAL_TAGS,\n Metadata: ATTR_EVAL_METADATA,\n Collection: {\n ID: ATTR_EVAL_COLLECTION_ID,\n Name: ATTR_EVAL_COLLECTION_NAME,\n Size: ATTR_EVAL_COLLECTION_SIZE,\n },\n Config: {\n Flags: ATTR_EVAL_CONFIG_FLAGS,\n },\n Run: {\n ID: ATTR_EVAL_RUN_ID,\n },\n Trial: {\n Index: ATTR_EVAL_TRIAL_INDEX,\n Error: ATTR_EVAL_TRIAL_ERROR,\n },\n Case: {\n Index: ATTR_EVAL_CASE_INDEX,\n Input: ATTR_EVAL_CASE_INPUT,\n Output: ATTR_EVAL_CASE_OUTPUT,\n Expected: ATTR_EVAL_CASE_EXPECTED,\n Scores: ATTR_EVAL_CASE_SCORES,\n Metadata: ATTR_EVAL_CASE_METADATA,\n Trials: ATTR_EVAL_CASE_TRIALS,\n },\n Task: {\n Output: ATTR_EVAL_TASK_OUTPUT,\n Name: ATTR_EVAL_TASK_NAME,\n Type: ATTR_EVAL_TASK_TYPE,\n },\n Score: {\n Name: ATTR_EVAL_SCORE_NAME,\n Value: ATTR_EVAL_SCORE_VALUE,\n Threshold: ATTR_EVAL_SCORE_THRESHOLD,\n Passed: ATTR_EVAL_SCORE_PASSED,\n IsBoolean: ATTR_EVAL_SCORE_IS_BOOLEAN,\n Metadata: ATTR_EVAL_SCORE_METADATA,\n Aggregation: ATTR_EVAL_SCORE_AGGREGATION,\n Trials: ATTR_EVAL_SCORE_TRIALS,\n },\n User: {\n Name: ATTR_EVAL_USER_NAME,\n Email: ATTR_EVAL_USER_EMAIL,\n },\n },\n Error: {\n Type: ATTR_ERROR_TYPE,\n Message: ATTR_ERROR_MESSAGE,\n },\n HTTP: {\n Response: {\n StatusCode: ATTR_HTTP_RESPONSE_STATUS_CODE,\n },\n },\n} as const;\n","// experiment\nexport const ATTR_EVAL_ID = 'eval.id' as const;\nexport const ATTR_EVAL_NAME = 'eval.name' as const;\nexport const ATTR_EVAL_VERSION = 'eval.version' as const;\nexport const ATTR_EVAL_TYPE = 'eval.type' as const;\nexport const ATTR_EVAL_TAGS = 'eval.tags' as const;\nexport const ATTR_EVAL_BASELINE_ID = 'eval.baseline.id' as const;\nexport const ATTR_EVAL_BASELINE_NAME = 'eval.baseline.name' as const;\nexport const ATTR_EVAL_BASELINE_VERSION = 'eval.baseline.version' as const;\nexport const ATTR_EVAL_METADATA = 'eval.metadata' as const;\nexport const ATTR_EVAL_TRIALS = 'eval.trials' as const;\n// capability\nexport const ATTR_EVAL_CAPABILITY_NAME = 'eval.capability.name' as const;\nexport const ATTR_EVAL_STEP_NAME = 'eval.step.name' as const;\n// collection\nexport const ATTR_EVAL_COLLECTION_ID = 'eval.collection.id' as const;\nexport const ATTR_EVAL_COLLECTION_SIZE = 'eval.collection.size' as const;\nexport const ATTR_EVAL_COLLECTION_NAME = 'eval.collection.name' as const;\n// config\nexport const ATTR_EVAL_CONFIG_FLAGS = 'eval.config.flags' as const;\n// case\nexport const ATTR_EVAL_CASE_INDEX = 'eval.case.index' as const;\nexport const ATTR_EVAL_CASE_INPUT = 'eval.case.input' as const;\nexport const ATTR_EVAL_CASE_OUTPUT = 'eval.case.output' as const;\nexport const ATTR_EVAL_CASE_EXPECTED = 'eval.case.expected' as const;\nexport const ATTR_EVAL_CASE_SCORES = 'eval.case.scores' as const;\nexport const ATTR_EVAL_CASE_METADATA = 'eval.case.metadata' as const;\nexport const ATTR_EVAL_CASE_TRIALS = 'eval.case.trials' as const;\n// trial\nexport const ATTR_EVAL_TRIAL_INDEX = 'eval.trial.index' as const;\nexport const ATTR_EVAL_TRIAL_ERROR = 'eval.trial.error' as const;\n// task\nexport const ATTR_EVAL_TASK_OUTPUT = 'eval.task.output' as const;\nexport const ATTR_EVAL_TASK_NAME = 'eval.task.name' as const;\nexport const ATTR_EVAL_TASK_TYPE = 'eval.task.type' as const;\n// run\nexport const ATTR_EVAL_RUN_ID = 'eval.run.id' as const;\n// score\nexport const ATTR_EVAL_SCORE_NAME = 'eval.score.name' as const;\nexport const ATTR_EVAL_SCORE_VALUE = 'eval.score.value' as const;\nexport const ATTR_EVAL_SCORE_THRESHOLD = 'eval.score.threshold' as const;\nexport const ATTR_EVAL_SCORE_PASSED = 'eval.score.passed' as const;\nexport const ATTR_EVAL_SCORE_IS_BOOLEAN = 'eval.score.is_boolean' as const;\nexport const ATTR_EVAL_SCORE_SCORER = 'eval.score.scorer' as const;\nexport const ATTR_EVAL_SCORE_METADATA = 'eval.score.metadata' as const;\nexport const ATTR_EVAL_SCORE_AGGREGATION = 'eval.score.aggregation' as const;\nexport const ATTR_EVAL_SCORE_TRIALS = 'eval.score.trials' as const;\n// user\nexport const ATTR_EVAL_USER_NAME = 'eval.user.name';\nexport const ATTR_EVAL_USER_EMAIL = 'eval.user.email';\n","import { Attr } from '../otel/semconv/attributes';\nimport type { ValidateName } from '../util/name-validation';\nimport type { Score, Scorer, ScorerOptions } from './scorer.types';\n\n// Helper to force TypeScript to evaluate/simplify types\ntype Simplify<T> = { [K in keyof T]: T[K] } & {};\n\ntype ScorerReturnValue = number \| boolean \| Score;\ntype AwaitedValue<T> = T extends Promise<infer U> ? U : T;\ntype InferScorerMetadata<T> =\n AwaitedValue<T> extends Score<infer TMetadata> ? TMetadata : Record<string, any>;\ntype NormalizeScorerReturn<T, TMetadata extends Record<string, any>> =\n T extends Promise<any> ? Promise<Score<TMetadata>> : Score<TMetadata>;\n\n/\n Creates a scorer to be used in evals.\n \n Scorers need to return a number or a boolean. If returning a number, it is\n * suggested that this number is between 0 and 1.\n \n @example\n * const scorer = createScorer('exact-match',\n * (args: { output: string; expected: string; }) => {\n * return args.output === args.expected ? true : false;\n * }\n * );\n \n @example\n * // With aggregation for trials\n * import { PassAtK } from '@axiomhq/ai/evals/aggregations';\n * const scorer = createScorer('tool-called',\n * (args: { output: string }) => args.output.includes('tool') ? 1 : 0,\n * { aggregation: PassAtK({ threshold: 0.8 }) }\n * );\n /\nexport function createScorer<\n TArgs extends Record<string, any> = {},\n // Use tuple wrapping to prevent distributive conditional types\n TInput = [TArgs] extends [{ input: infer I }] ? I : unknown,\n TExpected = [TArgs] extends [{ expected: infer E }] ? Exclude<E, undefined> : unknown,\n TOutput = [TArgs] extends [{ output: infer O }] ? Exclude<O, undefined> : never,\n TExtra extends Record<string, any> = Simplify<\n Omit<TArgs, 'input' \| 'expected' \| 'output' \| 'trialIndex'>\n >,\n TReturn extends ScorerReturnValue \| Promise<ScorerReturnValue> =\n \| ScorerReturnValue\n \| Promise<ScorerReturnValue>,\n TName extends string = string,\n>(\n /\n The name of the scorer\n /\n name: ValidateName<TName>,\n /\n The scorer function. Can be sync or async.\n /\n fn: (args: TArgs) => TReturn,\n /\n Optional configuration for the scorer, including aggregation for trials.\n */\n options?: ScorerOptions,\n): [TOutput] extends [never]\n ? never\n : Scorer<\n TInput,\n TExpected,\n TOutput,\n TExtra,\n InferScorerMetadata<TReturn>,\n NormalizeScorerReturn<TReturn, InferScorerMetadata<TReturn>>\n > {\n const normalizeScore = (res: ScorerReturnValue): Score => {\n if (typeof res === 'number') {\n return { score: res };\n }\n if (typeof res === 'boolean') {\n return {\n score: res ? 1 : 0,\n metadata: {\n [Attr.Eval.Score.IsBoolean]: true,\n },\n };\n }\n // Score object with boolean score - convert and merge is_boolean into metadata\n if (typeof res.score === 'boolean') {\n return {\n score: res.score ? 1 : 0,\n metadata: {\n ...res.metadata,\n [Attr.Eval.Score.IsBoolean]: true,\n },\n };\n }\n return res;\n };\n\n const scorer: any = (args: TArgs) => {\n const res = fn(args);\n\n // If user returned a Promise, handle async\n if (res instanceof Promise) {\n return res.then(normalizeScore);\n }\n\n // Otherwise handle sync\n return normalizeScore(res);\n };\n\n // Attach name property to function for pre-execution access\n Object.defineProperty(scorer, 'name', {\n value: name,\n configurable: true,\n enumerable: true,\n });\n\n // Attach aggregation config if provided\n if (options?.aggregation) {\n Object.defineProperty(scorer, 'aggregation', {\n value: options.aggregation,\n configurable: true,\n enumerable: true,\n });\n }\n\n return scorer as [TOutput] extends [never]\n ? never\n : Scorer<\n TInput,\n TExpected,\n TOutput,\n TExtra,\n InferScorerMetadata<TReturn>,\n NormalizeScorerReturn<TReturn, InferScorerMetadata<TReturn>>\n >;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,kCAGO;;;ACFA,IAAM,eAAe;AACrB,IAAM,iBAAiB;AACvB,IAAM,oBAAoB;AAC1B,IAAM,iBAAiB;AACvB,IAAM,iBAAiB;AACvB,IAAM,wBAAwB;AAC9B,IAAM,0BAA0B;AAChC,IAAM,6BAA6B;AACnC,IAAM,qBAAqB;AAC3B,IAAM,mBAAmB;AAEzB,IAAM,4BAA4B;AAClC,IAAM,sBAAsB;AAE5B,IAAM,0BAA0B;AAChC,IAAM,4BAA4B;AAClC,IAAM,4BAA4B;AAElC,IAAM,yBAAyB;AAE/B,IAAM,uBAAuB;AAC7B,IAAM,uBAAuB;AAC7B,IAAM,wBAAwB;AAC9B,IAAM,0BAA0B;AAChC,IAAM,wBAAwB;AAC9B,IAAM,0BAA0B;AAChC,IAAM,wBAAwB;AAE9B,IAAM,wBAAwB;AAC9B,IAAM,wBAAwB;AAE9B,IAAM,wBAAwB;AAC9B,IAAM,sBAAsB;AAC5B,IAAM,sBAAsB;AAE5B,IAAM,mBAAmB;AAEzB,IAAM,uBAAuB;AAC7B,IAAM,wBAAwB;AAC9B,IAAM,4BAA4B;AAClC,IAAM,yBAAyB;AAC/B,IAAM,6BAA6B;AAEnC,IAAM,2BAA2B;AACjC,IAAM,8BAA8B;AACpC,IAAM,yBAAyB;AAE/B,IAAM,sBAAsB;AAC5B,IAAM,uBAAuB;;;ADFpC,wBAwDO;AAQP,IAAM,+BAA+B;AACrC,IAAM,6BAA6B;AACnC,IAAM,gCAAgC;AACtC,IAAM,8BAA8B;AACpC,IAAM,wBAAwB;AAC9B,IAAM,6BAA6B;AACnC,IAAM,2BAA2B;AAEjC,IAAM,wCAAwC;AAC9C,IAAM,sCAAsC;AAC5C,IAAM,sCAAsC;AAC5C,IAAM,uCAAuC;AAC7C,IAAM,wCAAwC;AAC9C,IAAM,iCAAiC;AACvC,IAAM,uCAAuC;AAC7C,IAAM,oCAAoC;AAC1C,IAAM,kCAAkC;AACxC,IAAM,kCAAkC;AACxC,IAAM,kCAAkC;AACxC,IAAM,uCAAuC;AAC7C,IAAM,mCAAmC;AACzC,IAAM,wCAAwC;AAC9C,IAAM,oCAAoC;AAanC,IAAM,OAAO;AAAA,EAClB,qBAAqB,CAAC,aAAqB,QAAQ,QAAQ;AAAA,EAC3D,qBAAqB,CAAC,aAAqB,QAAQ,QAAQ;AAAA,EAC3D,OAAO;AAAA,IACL,OAAO;AAAA,MACL,WAAW;AAAA,MACX,KAAK;AAAA,QACH,MAAM;AAAA,QACN,SAAS;AAAA,MACX;AAAA,IACF;AAAA,EACF;AAAA,EACA,OAAO;AAAA,IACL,gBAAgB;AAAA,MACd,IAAI;AAAA,MACJ,MAAM;AAAA,MACN,MAAM;AAAA,MACN,SAAS;AAAA,IACX;AAAA;AAAA;AAAA;AAAA,IAIA,YAAY;AAAA,MACV,MAAM;AAAA,IACR;AAAA,IACA,MAAM;AAAA,MACJ,MAAM;AAAA,IACR;AAAA,IACA,UAAU;AAAA,MACR,MAAM;AAAA,MACN,aAAa;AAAA,QACX,WAAW;AAAA,QACX,YAAY;AAAA,QACZ,YAAY;AAAA,QACZ,kBAAkB;AAAA,QAClB,eAAe;AAAA,QACf,UAAU;AAAA,QACV,QAAQ;AAAA,QACR,UAAU;AAAA,QACV,WAAW;AAAA,QACX,UAAU;AAAA,QACV,YAAY;AAAA,QACZ,KAAK;AAAA,QACL,WAAW;AAAA,QACX,WAAW;AAAA,QACX,UAAU;AAAA,QACV,aAAa;AAAA,QACb,QAAQ;AAAA,QACR,MAAM;AAAA,QACN,MAAM;AAAA,QACN,cAAc;AAAA,QACd,MAAM;AAAA,QACN,MAAM;AAAA,QACN,WAAW;AAAA,QACX,QAAQ;AAAA,QACR,YAAY;AAAA,QACZ,WAAW;AAAA,QACX,OAAO;AAAA,QACP,YAAY;AAAA,QACZ,QAAQ;AAAA,QACR,KAAK;AAAA,MACP;AAAA,IACF;AAAA;AAAA;AAAA;AAAA,IAIA,OAAO;AAAA,MACL,aAAa;AAAA;AAAA,MACb,IAAI;AAAA;AAAA,MACJ,MAAM;AAAA;AAAA,IACR;AAAA,IACA,cAAc;AAAA,MACZ,IAAI;AAAA;AAAA,IACN;AAAA,IACA,OAAO;AAAA,MACL,UAAU;AAAA,IACZ;AAAA,IACA,WAAW;AAAA,MACT,MAAM;AAAA,MACN,aAAa;AAAA;AAAA;AAAA;AAAA,QAIX,MAAM;AAAA,QACN,aAAa;AAAA,QACb,YAAY;AAAA,QACZ,aAAa;AAAA,QACb,iBAAiB;AAAA,QACjB,aAAa;AAAA,MACf;AAAA,IACF;AAAA,IACA,QAAQ;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,aAAa;AAAA,QACX,MAAM;AAAA,QACN,MAAM;AAAA,QACN,OAAO;AAAA,QACP,QAAQ;AAAA,MACV;AAAA,IACF;AAAA;AAAA;AAAA;AAAA;AAAA,IAKA,SAAS;AAAA,MACP,aAAa;AAAA;AAAA,MACb,iBAAiB;AAAA;AAAA,MACjB,kBAAkB;AAAA,MAClB,WAAW;AAAA;AAAA;AAAA;AAAA,MAIX,OAAO;AAAA,MACP,iBAAiB;AAAA,MACjB,MAAM;AAAA,MACN,eAAe;AAAA,MACf,aAAa;AAAA,MACb,MAAM;AAAA,MACN,MAAM;AAAA,IACR;AAAA,IACA,UAAU;AAAA,MACR,eAAe;AAAA,MACf,IAAI;AAAA;AAAA;AAAA;AAAA,MAIJ,OAAO;AAAA;AAAA,IACT;AAAA,IACA,MAAM;AAAA,MACJ,QAAQ;AAAA,MACR,aAAa;AAAA,MACb,MAAM;AAAA,MACN,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAON,WAAW;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAOX,SAAS;AAAA,IACX;AAAA,IACA,OAAO;AAAA,MACL,aAAa;AAAA,MACb,cAAc;AAAA,IAChB;AAAA,EACF;AAAA,EACA,MAAM;AAAA,IACJ,IAAI;AAAA,IACJ,MAAM;AAAA,IACN,SAAS;AAAA,IACT,MAAM;AAAA,IACN,QAAQ;AAAA,IACR,UAAU;AAAA,MACR,IAAI;AAAA,MACJ,MAAM;AAAA,MACN,SAAS;AAAA,IACX;AAAA,IACA,YAAY;AAAA,MACV,MAAM;AAAA,IACR;AAAA,IACA,MAAM;AAAA,MACJ,MAAM;AAAA,IACR;AAAA,IACA,MAAM;AAAA,IACN,UAAU;AAAA,IACV,YAAY;AAAA,MACV,IAAI;AAAA,MACJ,MAAM;AAAA,MACN,MAAM;AAAA,IACR;AAAA,IACA,QAAQ;AAAA,MACN,OAAO;AAAA,IACT;AAAA,IACA,KAAK;AAAA,MACH,IAAI;AAAA,IACN;AAAA,IACA,OAAO;AAAA,MACL,OAAO;AAAA,MACP,OAAO;AAAA,IACT;AAAA,IACA,MAAM;AAAA,MACJ,OAAO;AAAA,MACP,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,UAAU;AAAA,MACV,QAAQ;AAAA,MACR,UAAU;AAAA,MACV,QAAQ;AAAA,IACV;AAAA,IACA,MAAM;AAAA,MACJ,QAAQ;AAAA,MACR,MAAM;AAAA,MACN,MAAM;AAAA,IACR;AAAA,IACA,OAAO;AAAA,MACL,MAAM;AAAA,MACN,OAAO;AAAA,MACP,WAAW;AAAA,MACX,QAAQ;AAAA,MACR,WAAW;AAAA,MACX,UAAU;AAAA,MACV,aAAa;AAAA,MACb,QAAQ;AAAA,IACV;AAAA,IACA,MAAM;AAAA,MACJ,MAAM;AAAA,MACN,OAAO;AAAA,IACT;AAAA,EACF;AAAA,EACA,OAAO;AAAA,IACL,MAAM;AAAA,IACN,SAAS;AAAA,EACX;AAAA,EACA,MAAM;AAAA,IACJ,UAAU;AAAA,MACR,YAAY;AAAA,IACd;AAAA,EACF;AACF;;;AEjVO,SAAS,aAiBd,MAIA,IAIA,SAUI;AACJ,QAAM,iBAAiB,CAAC,QAAkC;AACxD,QAAI,OAAO,QAAQ,UAAU;AAC3B,aAAO,EAAE,OAAO,IAAI;AAAA,IACtB;AACA,QAAI,OAAO,QAAQ,WAAW;AAC5B,aAAO;AAAA,QACL,OAAO,MAAM,IAAI;AAAA,QACjB,UAAU;AAAA,UACR,CAAC,KAAK,KAAK,MAAM,SAAS,GAAG;AAAA,QAC/B;AAAA,MACF;AAAA,IACF;AAEA,QAAI,OAAO,IAAI,UAAU,WAAW;AAClC,aAAO;AAAA,QACL,OAAO,IAAI,QAAQ,IAAI;AAAA,QACvB,UAAU;AAAA,UACR,GAAG,IAAI;AAAA,UACP,CAAC,KAAK,KAAK,MAAM,SAAS,GAAG;AAAA,QAC/B;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAEA,QAAM,SAAc,CAAC,SAAgB;AACnC,UAAM,MAAM,GAAG,IAAI;AAGnB,QAAI,eAAe,SAAS;AAC1B,aAAO,IAAI,KAAK,cAAc;AAAA,IAChC;AAGA,WAAO,eAAe,GAAG;AAAA,EAC3B;AAGA,SAAO,eAAe,QAAQ,QAAQ;AAAA,IACpC,OAAO;AAAA,IACP,cAAc;AAAA,IACd,YAAY;AAAA,EACd,CAAC;AAGD,MAAI,SAAS,aAAa;AACxB,WAAO,eAAe,QAAQ,eAAe;AAAA,MAC3C,OAAO,QAAQ;AAAA,MACf,cAAc;AAAA,MACd,YAAY;AAAA,IACd,CAAC;AAAA,EACH;AAEA,SAAO;AAUT;","names":[]}
1	+ {"version":3,"sources":["../../src/evals/scorers.ts","../../src/otel/semconv/attributes.ts","../../src/otel/semconv/eval_proposal.ts","../../src/evals/normalize-score.ts","../../src/evals/scorer.factory.ts"],"sourcesContent":["export type { Score, ScoreWithName, ScorerOptions, ScorerLike } from './scorer.types';\nexport type { Scorer as ScorerType } from './scorer.types';\nexport { createScorer as Scorer } from './scorer.factory';\n","import {\n ATTR_ERROR_TYPE,\n ATTR_HTTP_RESPONSE_STATUS_CODE,\n} from '@opentelemetry/semantic-conventions';\n\nimport {\n ATTR_EVAL_CASE_INDEX,\n ATTR_EVAL_CASE_EXPECTED,\n ATTR_EVAL_CASE_INPUT,\n ATTR_EVAL_CASE_OUTPUT,\n ATTR_EVAL_CASE_METADATA,\n ATTR_EVAL_CASE_SCORES,\n ATTR_EVAL_CASE_TRIALS,\n ATTR_EVAL_SCORE_PASSED,\n ATTR_EVAL_SCORE_IS_BOOLEAN,\n ATTR_EVAL_SCORE_VALUE,\n ATTR_EVAL_SCORE_NAME,\n ATTR_EVAL_SCORE_THRESHOLD,\n ATTR_EVAL_SCORE_METADATA,\n ATTR_EVAL_SCORE_AGGREGATION,\n ATTR_EVAL_SCORE_TRIALS,\n ATTR_EVAL_TASK_TYPE,\n ATTR_EVAL_TASK_NAME,\n ATTR_EVAL_TASK_OUTPUT,\n ATTR_EVAL_COLLECTION_NAME,\n ATTR_EVAL_COLLECTION_SIZE,\n ATTR_EVAL_ID,\n ATTR_EVAL_BASELINE_ID,\n ATTR_EVAL_BASELINE_NAME,\n ATTR_EVAL_BASELINE_VERSION,\n ATTR_EVAL_NAME,\n ATTR_EVAL_TAGS,\n ATTR_EVAL_TYPE,\n ATTR_EVAL_TRIALS,\n ATTR_EVAL_TRIAL_INDEX,\n ATTR_EVAL_TRIAL_ERROR,\n ATTR_EVAL_COLLECTION_ID,\n ATTR_EVAL_USER_NAME,\n ATTR_EVAL_USER_EMAIL,\n ATTR_EVAL_VERSION,\n ATTR_EVAL_RUN_ID,\n ATTR_EVAL_METADATA,\n ATTR_EVAL_CONFIG_FLAGS,\n ATTR_EVAL_CAPABILITY_NAME,\n ATTR_EVAL_STEP_NAME,\n} from './eval_proposal';\n\nimport {\n ATTR_ERROR_MESSAGE,\n ATTR_GEN_AI_AGENT_DESCRIPTION,\n ATTR_GEN_AI_AGENT_ID,\n ATTR_GEN_AI_AGENT_NAME,\n ATTR_GEN_AI_CONVERSATION_ID,\n ATTR_GEN_AI_INPUT_MESSAGES,\n ATTR_GEN_AI_OPERATION_NAME,\n ATTR_GEN_AI_OUTPUT_MESSAGES,\n ATTR_GEN_AI_OUTPUT_TYPE,\n ATTR_GEN_AI_PROVIDER_NAME,\n ATTR_GEN_AI_REQUEST_CHOICE_COUNT,\n ATTR_GEN_AI_REQUEST_ENCODING_FORMATS,\n ATTR_GEN_AI_REQUEST_FREQUENCY_PENALTY,\n ATTR_GEN_AI_REQUEST_MAX_TOKENS,\n ATTR_GEN_AI_REQUEST_MODEL,\n ATTR_GEN_AI_REQUEST_PRESENCE_PENALTY,\n ATTR_GEN_AI_REQUEST_SEED,\n ATTR_GEN_AI_REQUEST_STOP_SEQUENCES,\n ATTR_GEN_AI_REQUEST_TEMPERATURE,\n ATTR_GEN_AI_REQUEST_TOP_K,\n ATTR_GEN_AI_REQUEST_TOP_P,\n ATTR_GEN_AI_RESPONSE_FINISH_REASONS,\n ATTR_GEN_AI_RESPONSE_ID,\n ATTR_GEN_AI_RESPONSE_MODEL,\n ATTR_GEN_AI_TOOL_CALL_ID,\n ATTR_GEN_AI_TOOL_DESCRIPTION,\n ATTR_GEN_AI_TOOL_NAME,\n ATTR_GEN_AI_TOOL_TYPE,\n ATTR_GEN_AI_USAGE_INPUT_TOKENS,\n ATTR_GEN_AI_USAGE_OUTPUT_TOKENS,\n GEN_AI_OPERATION_NAME_VALUE_CHAT,\n GEN_AI_OPERATION_NAME_VALUE_CREATE_AGENT,\n GEN_AI_OPERATION_NAME_VALUE_EMBEDDINGS,\n GEN_AI_OPERATION_NAME_VALUE_EXECUTE_TOOL,\n GEN_AI_OPERATION_NAME_VALUE_GENERATE_CONTENT,\n GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT,\n GEN_AI_OUTPUT_TYPE_VALUE_IMAGE,\n GEN_AI_OUTPUT_TYPE_VALUE_JSON,\n GEN_AI_OUTPUT_TYPE_VALUE_SPEECH,\n GEN_AI_OUTPUT_TYPE_VALUE_TEXT,\n GEN_AI_PROVIDER_NAME_VALUE_ANTHROPIC,\n GEN_AI_PROVIDER_NAME_VALUE_AWS_BEDROCK,\n GEN_AI_PROVIDER_NAME_VALUE_AZURE_AI_INFERENCE,\n GEN_AI_PROVIDER_NAME_VALUE_AZURE_AI_OPENAI,\n GEN_AI_PROVIDER_NAME_VALUE_COHERE,\n GEN_AI_PROVIDER_NAME_VALUE_DEEPSEEK,\n GEN_AI_PROVIDER_NAME_VALUE_GCP_GEMINI,\n GEN_AI_PROVIDER_NAME_VALUE_GCP_GEN_AI,\n GEN_AI_PROVIDER_NAME_VALUE_GCP_VERTEX_AI,\n GEN_AI_PROVIDER_NAME_VALUE_GROQ,\n GEN_AI_PROVIDER_NAME_VALUE_IBM_WATSONX_AI,\n GEN_AI_PROVIDER_NAME_VALUE_MISTRAL_AI,\n GEN_AI_PROVIDER_NAME_VALUE_OPENAI,\n GEN_AI_PROVIDER_NAME_VALUE_PERPLEXITY,\n GEN_AI_PROVIDER_NAME_VALUE_X_AI,\n} from '@opentelemetry/semantic-conventions/incubating';\n\n/*\n PROPRIETARY ATTRIBUTES (o11y)\n \n @see: https://axiom.co/docs/ai-engineering/semantic-conventions\n /\n\nconst ATTR_AXIOM_GEN_AI_SCHEMA_URL = 'axiom.gen_ai.schema_url';\nconst ATTR_AXIOM_GEN_AI_SDK_NAME = 'axiom.gen_ai.sdk.name';\nconst ATTR_AXIOM_GEN_AI_SDK_VERSION = 'axiom.gen_ai.sdk.version';\nconst ATTR_GEN_AI_CAPABILITY_NAME = 'gen_ai.capability.name';\nconst ATTR_GEN_AI_STEP_NAME = 'gen_ai.step.name';\nconst ATTR_GEN_AI_TOOL_ARGUMENTS = 'gen_ai.tool.arguments'; // deprecated by OTel\nconst ATTR_GEN_AI_TOOL_MESSAGE = 'gen_ai.tool.message'; // deprecated by OTel\n\nconst GEN_AI_PROVIDER_NAME_VALUE_ASSEMBLYAI = 'assemblyai';\nconst GEN_AI_PROVIDER_NAME_VALUE_CEREBRAS = 'cerebras';\nconst GEN_AI_PROVIDER_NAME_VALUE_DEEPGRAM = 'deepgram';\nconst GEN_AI_PROVIDER_NAME_VALUE_DEEPINFRA = 'deepinfra';\nconst GEN_AI_PROVIDER_NAME_VALUE_ELEVENLABS = 'elevenlabs';\nconst GEN_AI_PROVIDER_NAME_VALUE_FAL = 'fal';\nconst GEN_AI_PROVIDER_NAME_VALUE_FIREWORKS = 'fireworks';\nconst GEN_AI_PROVIDER_NAME_VALUE_GLADIA = 'gladia';\nconst GEN_AI_PROVIDER_NAME_VALUE_HUME = 'hume';\nconst GEN_AI_PROVIDER_NAME_VALUE_LMNT = 'lmnt';\nconst GEN_AI_PROVIDER_NAME_VALUE_LUMA = 'luma';\nconst GEN_AI_PROVIDER_NAME_VALUE_REPLICATE = 'replicate';\nconst GEN_AI_PROVIDER_NAME_VALUE_REVAI = 'revai';\nconst GEN_AI_PROVIDER_NAME_VALUE_TOGETHERAI = 'togetherai';\nconst GEN_AI_PROVIDER_NAME_VALUE_VERCEL = 'vercel';\n\n/\n When adding something new here, please:\n * 1. Make sure it doesn't already exist as part of OTel Semantic Conventions (use that instead)\n * 2. Make sure to use standard naming schema, ie snake_case\n * 3. If a specific feature has an attribute you would like to use, extract it to the shared section\n \n Also Experimental Attributes should always be imported here and then used from the CustomAttributes object\n * because they are unstable.\n \n @see: https://github.com/open-telemetry/opentelemetry-js/tree/c89cb38d0fec39d54cf3fcb35c429a8129e9c909/semantic-conventions#unstable-semconv\n /\nexport const Attr = {\n __EXPERIMENTAL_Flag: (flagName: string) => `flag.${flagName}`,\n __EXPERIMENTAL_Fact: (factName: string) => `fact.${factName}`,\n Axiom: {\n GenAI: {\n SchemaURL: ATTR_AXIOM_GEN_AI_SCHEMA_URL,\n SDK: {\n Name: ATTR_AXIOM_GEN_AI_SDK_NAME,\n Version: ATTR_AXIOM_GEN_AI_SDK_VERSION,\n },\n },\n },\n GenAI: {\n PromptMetadata: {\n ID: 'axiom.gen_ai.prompt.id',\n Name: 'axiom.gen_ai.prompt.name',\n Slug: 'axiom.gen_ai.prompt.slug',\n Version: 'axiom.gen_ai.prompt.version',\n },\n /\n These two are used to identify the span\n /\n Capability: {\n Name: ATTR_GEN_AI_CAPABILITY_NAME,\n },\n Step: {\n Name: ATTR_GEN_AI_STEP_NAME,\n },\n Provider: {\n Name: ATTR_GEN_AI_PROVIDER_NAME,\n Name_Values: {\n Anthropic: GEN_AI_PROVIDER_NAME_VALUE_ANTHROPIC,\n AssemblyAI: GEN_AI_PROVIDER_NAME_VALUE_ASSEMBLYAI,\n AWSBedrock: GEN_AI_PROVIDER_NAME_VALUE_AWS_BEDROCK,\n AzureAIInference: GEN_AI_PROVIDER_NAME_VALUE_AZURE_AI_INFERENCE,\n AzureAIOpenAI: GEN_AI_PROVIDER_NAME_VALUE_AZURE_AI_OPENAI,\n Cerebras: GEN_AI_PROVIDER_NAME_VALUE_CEREBRAS,\n Cohere: GEN_AI_PROVIDER_NAME_VALUE_COHERE,\n Deepgram: GEN_AI_PROVIDER_NAME_VALUE_DEEPGRAM,\n DeepInfra: GEN_AI_PROVIDER_NAME_VALUE_DEEPINFRA,\n Deepseek: GEN_AI_PROVIDER_NAME_VALUE_DEEPSEEK,\n ElevenLabs: GEN_AI_PROVIDER_NAME_VALUE_ELEVENLABS,\n Fal: GEN_AI_PROVIDER_NAME_VALUE_FAL,\n Fireworks: GEN_AI_PROVIDER_NAME_VALUE_FIREWORKS,\n GCPGemini: GEN_AI_PROVIDER_NAME_VALUE_GCP_GEMINI,\n GCPGenAI: GEN_AI_PROVIDER_NAME_VALUE_GCP_GEN_AI,\n GCPVertexAI: GEN_AI_PROVIDER_NAME_VALUE_GCP_VERTEX_AI,\n Gladia: GEN_AI_PROVIDER_NAME_VALUE_GLADIA,\n Groq: GEN_AI_PROVIDER_NAME_VALUE_GROQ,\n Hume: GEN_AI_PROVIDER_NAME_VALUE_HUME,\n IBMWatsonxAI: GEN_AI_PROVIDER_NAME_VALUE_IBM_WATSONX_AI,\n Lmnt: GEN_AI_PROVIDER_NAME_VALUE_LMNT,\n Luma: GEN_AI_PROVIDER_NAME_VALUE_LUMA,\n MistralAI: GEN_AI_PROVIDER_NAME_VALUE_MISTRAL_AI,\n OpenAI: GEN_AI_PROVIDER_NAME_VALUE_OPENAI,\n Perplexity: GEN_AI_PROVIDER_NAME_VALUE_PERPLEXITY,\n Replicate: GEN_AI_PROVIDER_NAME_VALUE_REPLICATE,\n RevAI: GEN_AI_PROVIDER_NAME_VALUE_REVAI,\n TogetherAI: GEN_AI_PROVIDER_NAME_VALUE_TOGETHERAI,\n Vercel: GEN_AI_PROVIDER_NAME_VALUE_VERCEL,\n XAI: GEN_AI_PROVIDER_NAME_VALUE_X_AI,\n },\n },\n /\n Regular attributes\n /\n Agent: {\n Description: ATTR_GEN_AI_AGENT_DESCRIPTION, // not yet used by axiom-ai\n ID: ATTR_GEN_AI_AGENT_ID, // not yet used by axiom-ai\n Name: ATTR_GEN_AI_AGENT_NAME, // not yet used by axiom-ai\n },\n Conversation: {\n ID: ATTR_GEN_AI_CONVERSATION_ID, // not yet used by axiom-ai, anyway probably needs to be provided by user\n },\n Input: {\n Messages: ATTR_GEN_AI_INPUT_MESSAGES,\n },\n Operation: {\n Name: ATTR_GEN_AI_OPERATION_NAME,\n Name_Values: {\n /\n Note that \"text_completion\" is deprecated in favor of \"chat\" for both OpenAI and Anthropic\n /\n Chat: GEN_AI_OPERATION_NAME_VALUE_CHAT,\n CreateAgent: GEN_AI_OPERATION_NAME_VALUE_CREATE_AGENT,\n Embeddings: GEN_AI_OPERATION_NAME_VALUE_EMBEDDINGS,\n ExecuteTool: GEN_AI_OPERATION_NAME_VALUE_EXECUTE_TOOL,\n GenerateContent: GEN_AI_OPERATION_NAME_VALUE_GENERATE_CONTENT,\n InvokeAgent: GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT,\n },\n },\n Output: {\n Messages: ATTR_GEN_AI_OUTPUT_MESSAGES,\n Type: ATTR_GEN_AI_OUTPUT_TYPE,\n Type_Values: {\n Text: GEN_AI_OUTPUT_TYPE_VALUE_TEXT,\n Json: GEN_AI_OUTPUT_TYPE_VALUE_JSON,\n Image: GEN_AI_OUTPUT_TYPE_VALUE_IMAGE,\n Speech: GEN_AI_OUTPUT_TYPE_VALUE_SPEECH,\n },\n },\n /\n The provider that is hosting the model, eg AWS Bedrock\n * There doesn't seem to be a semconv for this\n /\n Request: {\n ChoiceCount: ATTR_GEN_AI_REQUEST_CHOICE_COUNT, // not yet used by axiom-ai\n EncodingFormats: ATTR_GEN_AI_REQUEST_ENCODING_FORMATS, // not yet used by axiom-ai\n FrequencyPenalty: ATTR_GEN_AI_REQUEST_FREQUENCY_PENALTY,\n MaxTokens: ATTR_GEN_AI_REQUEST_MAX_TOKENS,\n /\n The model you asked for\n /\n Model: ATTR_GEN_AI_REQUEST_MODEL,\n PresencePenalty: ATTR_GEN_AI_REQUEST_PRESENCE_PENALTY,\n Seed: ATTR_GEN_AI_REQUEST_SEED,\n StopSequences: ATTR_GEN_AI_REQUEST_STOP_SEQUENCES,\n Temperature: ATTR_GEN_AI_REQUEST_TEMPERATURE,\n TopK: ATTR_GEN_AI_REQUEST_TOP_K,\n TopP: ATTR_GEN_AI_REQUEST_TOP_P,\n },\n Response: {\n FinishReasons: ATTR_GEN_AI_RESPONSE_FINISH_REASONS,\n ID: ATTR_GEN_AI_RESPONSE_ID,\n /\n The model that was actually used (might be different bc routing) - only ever get this from the response, otherwise omit\n /\n Model: ATTR_GEN_AI_RESPONSE_MODEL, // somehow not landing on the span for google models? check up on this...\n },\n Tool: {\n CallID: ATTR_GEN_AI_TOOL_CALL_ID,\n Description: ATTR_GEN_AI_TOOL_DESCRIPTION,\n Name: ATTR_GEN_AI_TOOL_NAME,\n Type: ATTR_GEN_AI_TOOL_TYPE,\n /\n Note, OTel Semantic Convention suggest only putting tool inputs/outputs on the parent chat span\n * But we at least want to give users THE OPTION to put them on the tool spans themselves as well\n * Because it enables a lot of things with querying\n * @see https://github.com/open-telemetry/semantic-conventions/releases/tag/v1.37.0\n /\n Arguments: ATTR_GEN_AI_TOOL_ARGUMENTS,\n /\n Note, OTel Semantic Convention suggest only putting tool inputs/outputs on the parent chat span\n * But we at least want to give users THE OPTION to put them on the tool spans themselves as well\n * Because it enables a lot of things with querying\n * @see https://github.com/open-telemetry/semantic-conventions/releases/tag/v1.37.0\n /\n Message: ATTR_GEN_AI_TOOL_MESSAGE,\n },\n Usage: {\n InputTokens: ATTR_GEN_AI_USAGE_INPUT_TOKENS,\n OutputTokens: ATTR_GEN_AI_USAGE_OUTPUT_TOKENS,\n },\n },\n Eval: {\n ID: ATTR_EVAL_ID,\n Name: ATTR_EVAL_NAME,\n Version: ATTR_EVAL_VERSION,\n Type: ATTR_EVAL_TYPE,\n Trials: ATTR_EVAL_TRIALS,\n Baseline: {\n ID: ATTR_EVAL_BASELINE_ID,\n Name: ATTR_EVAL_BASELINE_NAME,\n Version: ATTR_EVAL_BASELINE_VERSION,\n },\n Capability: {\n Name: ATTR_EVAL_CAPABILITY_NAME,\n },\n Step: {\n Name: ATTR_EVAL_STEP_NAME,\n },\n Tags: ATTR_EVAL_TAGS,\n Metadata: ATTR_EVAL_METADATA,\n Collection: {\n ID: ATTR_EVAL_COLLECTION_ID,\n Name: ATTR_EVAL_COLLECTION_NAME,\n Size: ATTR_EVAL_COLLECTION_SIZE,\n },\n Config: {\n Flags: ATTR_EVAL_CONFIG_FLAGS,\n },\n Run: {\n ID: ATTR_EVAL_RUN_ID,\n },\n Trial: {\n Index: ATTR_EVAL_TRIAL_INDEX,\n Error: ATTR_EVAL_TRIAL_ERROR,\n },\n Case: {\n Index: ATTR_EVAL_CASE_INDEX,\n Input: ATTR_EVAL_CASE_INPUT,\n Output: ATTR_EVAL_CASE_OUTPUT,\n Expected: ATTR_EVAL_CASE_EXPECTED,\n Scores: ATTR_EVAL_CASE_SCORES,\n Metadata: ATTR_EVAL_CASE_METADATA,\n Trials: ATTR_EVAL_CASE_TRIALS,\n },\n Task: {\n Output: ATTR_EVAL_TASK_OUTPUT,\n Name: ATTR_EVAL_TASK_NAME,\n Type: ATTR_EVAL_TASK_TYPE,\n },\n Score: {\n Name: ATTR_EVAL_SCORE_NAME,\n Value: ATTR_EVAL_SCORE_VALUE,\n Threshold: ATTR_EVAL_SCORE_THRESHOLD,\n Passed: ATTR_EVAL_SCORE_PASSED,\n IsBoolean: ATTR_EVAL_SCORE_IS_BOOLEAN,\n Metadata: ATTR_EVAL_SCORE_METADATA,\n Aggregation: ATTR_EVAL_SCORE_AGGREGATION,\n Trials: ATTR_EVAL_SCORE_TRIALS,\n },\n User: {\n Name: ATTR_EVAL_USER_NAME,\n Email: ATTR_EVAL_USER_EMAIL,\n },\n },\n Error: {\n Type: ATTR_ERROR_TYPE,\n Message: ATTR_ERROR_MESSAGE,\n },\n HTTP: {\n Response: {\n StatusCode: ATTR_HTTP_RESPONSE_STATUS_CODE,\n },\n },\n} as const;\n","// experiment\nexport const ATTR_EVAL_ID = 'eval.id' as const;\nexport const ATTR_EVAL_NAME = 'eval.name' as const;\nexport const ATTR_EVAL_VERSION = 'eval.version' as const;\nexport const ATTR_EVAL_TYPE = 'eval.type' as const;\nexport const ATTR_EVAL_TAGS = 'eval.tags' as const;\nexport const ATTR_EVAL_BASELINE_ID = 'eval.baseline.id' as const;\nexport const ATTR_EVAL_BASELINE_NAME = 'eval.baseline.name' as const;\nexport const ATTR_EVAL_BASELINE_VERSION = 'eval.baseline.version' as const;\nexport const ATTR_EVAL_METADATA = 'eval.metadata' as const;\nexport const ATTR_EVAL_TRIALS = 'eval.trials' as const;\n// capability\nexport const ATTR_EVAL_CAPABILITY_NAME = 'eval.capability.name' as const;\nexport const ATTR_EVAL_STEP_NAME = 'eval.step.name' as const;\n// collection\nexport const ATTR_EVAL_COLLECTION_ID = 'eval.collection.id' as const;\nexport const ATTR_EVAL_COLLECTION_SIZE = 'eval.collection.size' as const;\nexport const ATTR_EVAL_COLLECTION_NAME = 'eval.collection.name' as const;\n// config\nexport const ATTR_EVAL_CONFIG_FLAGS = 'eval.config.flags' as const;\n// case\nexport const ATTR_EVAL_CASE_INDEX = 'eval.case.index' as const;\nexport const ATTR_EVAL_CASE_INPUT = 'eval.case.input' as const;\nexport const ATTR_EVAL_CASE_OUTPUT = 'eval.case.output' as const;\nexport const ATTR_EVAL_CASE_EXPECTED = 'eval.case.expected' as const;\nexport const ATTR_EVAL_CASE_SCORES = 'eval.case.scores' as const;\nexport const ATTR_EVAL_CASE_METADATA = 'eval.case.metadata' as const;\nexport const ATTR_EVAL_CASE_TRIALS = 'eval.case.trials' as const;\n// trial\nexport const ATTR_EVAL_TRIAL_INDEX = 'eval.trial.index' as const;\nexport const ATTR_EVAL_TRIAL_ERROR = 'eval.trial.error' as const;\n// task\nexport const ATTR_EVAL_TASK_OUTPUT = 'eval.task.output' as const;\nexport const ATTR_EVAL_TASK_NAME = 'eval.task.name' as const;\nexport const ATTR_EVAL_TASK_TYPE = 'eval.task.type' as const;\n// run\nexport const ATTR_EVAL_RUN_ID = 'eval.run.id' as const;\n// score\nexport const ATTR_EVAL_SCORE_NAME = 'eval.score.name' as const;\nexport const ATTR_EVAL_SCORE_VALUE = 'eval.score.value' as const;\nexport const ATTR_EVAL_SCORE_THRESHOLD = 'eval.score.threshold' as const;\nexport const ATTR_EVAL_SCORE_PASSED = 'eval.score.passed' as const;\nexport const ATTR_EVAL_SCORE_IS_BOOLEAN = 'eval.score.is_boolean' as const;\nexport const ATTR_EVAL_SCORE_SCORER = 'eval.score.scorer' as const;\nexport const ATTR_EVAL_SCORE_METADATA = 'eval.score.metadata' as const;\nexport const ATTR_EVAL_SCORE_AGGREGATION = 'eval.score.aggregation' as const;\nexport const ATTR_EVAL_SCORE_TRIALS = 'eval.score.trials' as const;\n// user\nexport const ATTR_EVAL_USER_NAME = 'eval.user.name';\nexport const ATTR_EVAL_USER_EMAIL = 'eval.user.email';\n","import { Attr } from '../otel/semconv/attributes';\nimport type { Score } from './scorer.types';\n\n/\n Normalizes a boolean score to numeric (1/0) and adds eval.score.is_boolean\n * to metadata. Numbers and null pass through unchanged.\n /\nexport function normalizeBooleanScore(\n score: Score['score'],\n metadata?: Record<string, unknown>,\n): { score: number \| null; metadata?: Record<string, unknown> } {\n if (typeof score !== 'boolean') {\n return { score, metadata };\n }\n return {\n score: score ? 1 : 0,\n metadata: { ...metadata, [Attr.Eval.Score.IsBoolean]: true },\n };\n}\n","import type { ValidateName } from '../util/name-validation';\nimport { normalizeBooleanScore } from './normalize-score';\nimport type { Score, Scorer, ScorerOptions } from './scorer.types';\n\n// Helper to force TypeScript to evaluate/simplify types\ntype Simplify<T> = { [K in keyof T]: T[K] } & {};\n\ntype ScorerReturnValue = number \| boolean \| Score;\ntype AwaitedValue<T> = T extends Promise<infer U> ? U : T;\ntype InferScorerMetadata<T> =\n AwaitedValue<T> extends Score<infer TMetadata> ? TMetadata : Record<string, any>;\ntype NormalizeScorerReturn<T, TMetadata extends Record<string, any>> =\n T extends Promise<any> ? Promise<Score<TMetadata>> : Score<TMetadata>;\n\n/\n Creates a scorer to be used in evals.\n \n Scorers need to return a number or a boolean. If returning a number, it is\n * suggested that this number is between 0 and 1.\n \n @example\n * const scorer = createScorer('exact-match',\n * (args: { output: string; expected: string; }) => {\n * return args.output === args.expected ? true : false;\n * }\n * );\n \n @example\n * // With aggregation for trials\n * import { PassAtK } from '@axiomhq/ai/evals/aggregations';\n * const scorer = createScorer('tool-called',\n * (args: { output: string }) => args.output.includes('tool') ? 1 : 0,\n * { aggregation: PassAtK({ threshold: 0.8 }) }\n * );\n /\nexport function createScorer<\n TArgs extends Record<string, any> = {},\n // Use tuple wrapping to prevent distributive conditional types\n TInput = [TArgs] extends [{ input: infer I }] ? I : unknown,\n TExpected = [TArgs] extends [{ expected: infer E }] ? Exclude<E, undefined> : unknown,\n TOutput = [TArgs] extends [{ output: infer O }] ? Exclude<O, undefined> : never,\n TExtra extends Record<string, any> = Simplify<\n Omit<TArgs, 'input' \| 'expected' \| 'output' \| 'trialIndex'>\n >,\n TReturn extends ScorerReturnValue \| Promise<ScorerReturnValue> =\n \| ScorerReturnValue\n \| Promise<ScorerReturnValue>,\n TName extends string = string,\n>(\n /\n The name of the scorer\n /\n name: ValidateName<TName>,\n /\n The scorer function. Can be sync or async.\n /\n fn: (args: TArgs) => TReturn,\n /\n Optional configuration for the scorer, including aggregation for trials.\n */\n options?: ScorerOptions,\n): [TOutput] extends [never]\n ? never\n : Scorer<\n TInput,\n TExpected,\n TOutput,\n TExtra,\n InferScorerMetadata<TReturn>,\n NormalizeScorerReturn<TReturn, InferScorerMetadata<TReturn>>\n > {\n const normalizeScore = (res: ScorerReturnValue): Score => {\n if (typeof res === 'number') {\n return { score: res };\n }\n if (typeof res === 'boolean') {\n return normalizeBooleanScore(res);\n }\n return normalizeBooleanScore(res.score, res.metadata);\n };\n\n const scorer: any = (args: TArgs) => {\n const res = fn(args);\n\n // If user returned a Promise, handle async\n if (res instanceof Promise) {\n return res.then(normalizeScore);\n }\n\n // Otherwise handle sync\n return normalizeScore(res);\n };\n\n // Attach name property to function for pre-execution access\n Object.defineProperty(scorer, 'name', {\n value: name,\n configurable: true,\n enumerable: true,\n });\n\n // Attach aggregation config if provided\n if (options?.aggregation) {\n Object.defineProperty(scorer, 'aggregation', {\n value: options.aggregation,\n configurable: true,\n enumerable: true,\n });\n }\n\n return scorer as [TOutput] extends [never]\n ? never\n : Scorer<\n TInput,\n TExpected,\n TOutput,\n TExtra,\n InferScorerMetadata<TReturn>,\n NormalizeScorerReturn<TReturn, InferScorerMetadata<TReturn>>\n >;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,kCAGO;;;ACFA,IAAM,eAAe;AACrB,IAAM,iBAAiB;AACvB,IAAM,oBAAoB;AAC1B,IAAM,iBAAiB;AACvB,IAAM,iBAAiB;AACvB,IAAM,wBAAwB;AAC9B,IAAM,0BAA0B;AAChC,IAAM,6BAA6B;AACnC,IAAM,qBAAqB;AAC3B,IAAM,mBAAmB;AAEzB,IAAM,4BAA4B;AAClC,IAAM,sBAAsB;AAE5B,IAAM,0BAA0B;AAChC,IAAM,4BAA4B;AAClC,IAAM,4BAA4B;AAElC,IAAM,yBAAyB;AAE/B,IAAM,uBAAuB;AAC7B,IAAM,uBAAuB;AAC7B,IAAM,wBAAwB;AAC9B,IAAM,0BAA0B;AAChC,IAAM,wBAAwB;AAC9B,IAAM,0BAA0B;AAChC,IAAM,wBAAwB;AAE9B,IAAM,wBAAwB;AAC9B,IAAM,wBAAwB;AAE9B,IAAM,wBAAwB;AAC9B,IAAM,sBAAsB;AAC5B,IAAM,sBAAsB;AAE5B,IAAM,mBAAmB;AAEzB,IAAM,uBAAuB;AAC7B,IAAM,wBAAwB;AAC9B,IAAM,4BAA4B;AAClC,IAAM,yBAAyB;AAC/B,IAAM,6BAA6B;AAEnC,IAAM,2BAA2B;AACjC,IAAM,8BAA8B;AACpC,IAAM,yBAAyB;AAE/B,IAAM,sBAAsB;AAC5B,IAAM,uBAAuB;;;ADFpC,wBAwDO;AAQP,IAAM,+BAA+B;AACrC,IAAM,6BAA6B;AACnC,IAAM,gCAAgC;AACtC,IAAM,8BAA8B;AACpC,IAAM,wBAAwB;AAC9B,IAAM,6BAA6B;AACnC,IAAM,2BAA2B;AAEjC,IAAM,wCAAwC;AAC9C,IAAM,sCAAsC;AAC5C,IAAM,sCAAsC;AAC5C,IAAM,uCAAuC;AAC7C,IAAM,wCAAwC;AAC9C,IAAM,iCAAiC;AACvC,IAAM,uCAAuC;AAC7C,IAAM,oCAAoC;AAC1C,IAAM,kCAAkC;AACxC,IAAM,kCAAkC;AACxC,IAAM,kCAAkC;AACxC,IAAM,uCAAuC;AAC7C,IAAM,mCAAmC;AACzC,IAAM,wCAAwC;AAC9C,IAAM,oCAAoC;AAanC,IAAM,OAAO;AAAA,EAClB,qBAAqB,CAAC,aAAqB,QAAQ,QAAQ;AAAA,EAC3D,qBAAqB,CAAC,aAAqB,QAAQ,QAAQ;AAAA,EAC3D,OAAO;AAAA,IACL,OAAO;AAAA,MACL,WAAW;AAAA,MACX,KAAK;AAAA,QACH,MAAM;AAAA,QACN,SAAS;AAAA,MACX;AAAA,IACF;AAAA,EACF;AAAA,EACA,OAAO;AAAA,IACL,gBAAgB;AAAA,MACd,IAAI;AAAA,MACJ,MAAM;AAAA,MACN,MAAM;AAAA,MACN,SAAS;AAAA,IACX;AAAA;AAAA;AAAA;AAAA,IAIA,YAAY;AAAA,MACV,MAAM;AAAA,IACR;AAAA,IACA,MAAM;AAAA,MACJ,MAAM;AAAA,IACR;AAAA,IACA,UAAU;AAAA,MACR,MAAM;AAAA,MACN,aAAa;AAAA,QACX,WAAW;AAAA,QACX,YAAY;AAAA,QACZ,YAAY;AAAA,QACZ,kBAAkB;AAAA,QAClB,eAAe;AAAA,QACf,UAAU;AAAA,QACV,QAAQ;AAAA,QACR,UAAU;AAAA,QACV,WAAW;AAAA,QACX,UAAU;AAAA,QACV,YAAY;AAAA,QACZ,KAAK;AAAA,QACL,WAAW;AAAA,QACX,WAAW;AAAA,QACX,UAAU;AAAA,QACV,aAAa;AAAA,QACb,QAAQ;AAAA,QACR,MAAM;AAAA,QACN,MAAM;AAAA,QACN,cAAc;AAAA,QACd,MAAM;AAAA,QACN,MAAM;AAAA,QACN,WAAW;AAAA,QACX,QAAQ;AAAA,QACR,YAAY;AAAA,QACZ,WAAW;AAAA,QACX,OAAO;AAAA,QACP,YAAY;AAAA,QACZ,QAAQ;AAAA,QACR,KAAK;AAAA,MACP;AAAA,IACF;AAAA;AAAA;AAAA;AAAA,IAIA,OAAO;AAAA,MACL,aAAa;AAAA;AAAA,MACb,IAAI;AAAA;AAAA,MACJ,MAAM;AAAA;AAAA,IACR;AAAA,IACA,cAAc;AAAA,MACZ,IAAI;AAAA;AAAA,IACN;AAAA,IACA,OAAO;AAAA,MACL,UAAU;AAAA,IACZ;AAAA,IACA,WAAW;AAAA,MACT,MAAM;AAAA,MACN,aAAa;AAAA;AAAA;AAAA;AAAA,QAIX,MAAM;AAAA,QACN,aAAa;AAAA,QACb,YAAY;AAAA,QACZ,aAAa;AAAA,QACb,iBAAiB;AAAA,QACjB,aAAa;AAAA,MACf;AAAA,IACF;AAAA,IACA,QAAQ;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,aAAa;AAAA,QACX,MAAM;AAAA,QACN,MAAM;AAAA,QACN,OAAO;AAAA,QACP,QAAQ;AAAA,MACV;AAAA,IACF;AAAA;AAAA;AAAA;AAAA;AAAA,IAKA,SAAS;AAAA,MACP,aAAa;AAAA;AAAA,MACb,iBAAiB;AAAA;AAAA,MACjB,kBAAkB;AAAA,MAClB,WAAW;AAAA;AAAA;AAAA;AAAA,MAIX,OAAO;AAAA,MACP,iBAAiB;AAAA,MACjB,MAAM;AAAA,MACN,eAAe;AAAA,MACf,aAAa;AAAA,MACb,MAAM;AAAA,MACN,MAAM;AAAA,IACR;AAAA,IACA,UAAU;AAAA,MACR,eAAe;AAAA,MACf,IAAI;AAAA;AAAA;AAAA;AAAA,MAIJ,OAAO;AAAA;AAAA,IACT;AAAA,IACA,MAAM;AAAA,MACJ,QAAQ;AAAA,MACR,aAAa;AAAA,MACb,MAAM;AAAA,MACN,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAON,WAAW;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAOX,SAAS;AAAA,IACX;AAAA,IACA,OAAO;AAAA,MACL,aAAa;AAAA,MACb,cAAc;AAAA,IAChB;AAAA,EACF;AAAA,EACA,MAAM;AAAA,IACJ,IAAI;AAAA,IACJ,MAAM;AAAA,IACN,SAAS;AAAA,IACT,MAAM;AAAA,IACN,QAAQ;AAAA,IACR,UAAU;AAAA,MACR,IAAI;AAAA,MACJ,MAAM;AAAA,MACN,SAAS;AAAA,IACX;AAAA,IACA,YAAY;AAAA,MACV,MAAM;AAAA,IACR;AAAA,IACA,MAAM;AAAA,MACJ,MAAM;AAAA,IACR;AAAA,IACA,MAAM;AAAA,IACN,UAAU;AAAA,IACV,YAAY;AAAA,MACV,IAAI;AAAA,MACJ,MAAM;AAAA,MACN,MAAM;AAAA,IACR;AAAA,IACA,QAAQ;AAAA,MACN,OAAO;AAAA,IACT;AAAA,IACA,KAAK;AAAA,MACH,IAAI;AAAA,IACN;AAAA,IACA,OAAO;AAAA,MACL,OAAO;AAAA,MACP,OAAO;AAAA,IACT;AAAA,IACA,MAAM;AAAA,MACJ,OAAO;AAAA,MACP,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,UAAU;AAAA,MACV,QAAQ;AAAA,MACR,UAAU;AAAA,MACV,QAAQ;AAAA,IACV;AAAA,IACA,MAAM;AAAA,MACJ,QAAQ;AAAA,MACR,MAAM;AAAA,MACN,MAAM;AAAA,IACR;AAAA,IACA,OAAO;AAAA,MACL,MAAM;AAAA,MACN,OAAO;AAAA,MACP,WAAW;AAAA,MACX,QAAQ;AAAA,MACR,WAAW;AAAA,MACX,UAAU;AAAA,MACV,aAAa;AAAA,MACb,QAAQ;AAAA,IACV;AAAA,IACA,MAAM;AAAA,MACJ,MAAM;AAAA,MACN,OAAO;AAAA,IACT;AAAA,EACF;AAAA,EACA,OAAO;AAAA,IACL,MAAM;AAAA,IACN,SAAS;AAAA,EACX;AAAA,EACA,MAAM;AAAA,IACJ,UAAU;AAAA,MACR,YAAY;AAAA,IACd;AAAA,EACF;AACF;;;AE7WO,SAAS,sBACd,OACA,UAC8D;AAC9D,MAAI,OAAO,UAAU,WAAW;AAC9B,WAAO,EAAE,OAAO,SAAS;AAAA,EAC3B;AACA,SAAO;AAAA,IACL,OAAO,QAAQ,IAAI;AAAA,IACnB,UAAU,EAAE,GAAG,UAAU,CAAC,KAAK,KAAK,MAAM,SAAS,GAAG,KAAK;AAAA,EAC7D;AACF;;;ACiBO,SAAS,aAiBd,MAIA,IAIA,SAUI;AACJ,QAAM,iBAAiB,CAAC,QAAkC;AACxD,QAAI,OAAO,QAAQ,UAAU;AAC3B,aAAO,EAAE,OAAO,IAAI;AAAA,IACtB;AACA,QAAI,OAAO,QAAQ,WAAW;AAC5B,aAAO,sBAAsB,GAAG;AAAA,IAClC;AACA,WAAO,sBAAsB,IAAI,OAAO,IAAI,QAAQ;AAAA,EACtD;AAEA,QAAM,SAAc,CAAC,SAAgB;AACnC,UAAM,MAAM,GAAG,IAAI;AAGnB,QAAI,eAAe,SAAS;AAC1B,aAAO,IAAI,KAAK,cAAc;AAAA,IAChC;AAGA,WAAO,eAAe,GAAG;AAAA,EAC3B;AAGA,SAAO,eAAe,QAAQ,QAAQ;AAAA,IACpC,OAAO;AAAA,IACP,cAAc;AAAA,IACd,YAAY;AAAA,EACd,CAAC;AAGD,MAAI,SAAS,aAAa;AACxB,WAAO,eAAe,QAAQ,eAAe;AAAA,MAC3C,OAAO,QAAQ;AAAA,MACf,cAAc;AAAA,MACd,YAAY;AAAA,IACd,CAAC;AAAA,EACH;AAEA,SAAO;AAUT;","names":[]}

package/dist/evals/scorers.js CHANGED Viewed

@@ -1,6 +1,7 @@
 import {
   createScorer
-} from "../chunk-MCKVQ2IB.js";
+} from "../chunk-JSEK2D3F.js";
+import "../chunk-3Q36GUCI.js";
 import "../chunk-TWE3LIRZ.js";
 import {
   init_esm_shims

package/dist/evals/scorers.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"sources":["../../src/evals/scorers.ts"],"sourcesContent":["export type { Score, ScoreWithName, ScorerOptions, ScorerLike } from './scorer.types';\nexport type { Scorer as ScorerType } from './scorer.types';\nexport { createScorer as Scorer } from './scorer.factory';\n"],"mappings":"~~;;;;;;;;;~~AAAA;","names":[]}
1	+ {"version":3,"sources":["../../src/evals/scorers.ts"],"sourcesContent":["export type { Score, ScoreWithName, ScorerOptions, ScorerLike } from './scorer.types';\nexport type { Scorer as ScorerType } from './scorer.types';\nexport { createScorer as Scorer } from './scorer.factory';\n"],"mappings":";;;;;;;;;;AAAA;","names":[]}