@opensearch-project/agent-health 0.3.0 → 0.5.2

package/lib/dist/lib/telemetry/constants.js

@@ -0,0 +1,87 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * OTel Semantic Convention constants for evaluation telemetry.
+ *
+ * In-spec attributes are re-exported from @opentelemetry/semantic-conventions.
+ * Proposed and extension attributes are defined here as string constants.
+ */
+// =============================================================================
+// In-spec attributes (Development stability)
+// Re-exported from @opentelemetry/semantic-conventions for convenience
+// =============================================================================
+export { 
+// Test attributes (model/test/registry.yaml)
+ATTR_TEST_SUITE_NAME, ATTR_TEST_SUITE_RUN_STATUS, ATTR_TEST_CASE_NAME, ATTR_TEST_CASE_RESULT_STATUS, TEST_SUITE_RUN_STATUS_VALUE_SUCCESS, TEST_SUITE_RUN_STATUS_VALUE_FAILURE, TEST_SUITE_RUN_STATUS_VALUE_IN_PROGRESS, TEST_CASE_RESULT_STATUS_VALUE_PASS, TEST_CASE_RESULT_STATUS_VALUE_FAIL, 
+// GenAI evaluation attributes (model/gen-ai/registry.yaml, PR #2563)
+ATTR_GEN_AI_OPERATION_NAME, ATTR_GEN_AI_EVALUATION_NAME, ATTR_GEN_AI_EVALUATION_SCORE_VALUE, ATTR_GEN_AI_EVALUATION_SCORE_LABEL, ATTR_GEN_AI_EVALUATION_EXPLANATION, 
+// GenAI evaluation event (model/gen-ai/events.yaml)
+EVENT_GEN_AI_EVALUATION_RESULT, } from '@opentelemetry/semantic-conventions/incubating';
+// =============================================================================
+// Proposed attributes (Issue #3398 — not yet in spec)
+// https://github.com/open-telemetry/semantic-conventions/issues/3398
+// =============================================================================
+/** Unique identifier for a specific test suite run */
+export const ATTR_TEST_SUITE_RUN_ID = 'test.suite.run.id';
+/** Machine-readable unique identifier for a test case */
+export const ATTR_TEST_CASE_ID = 'test.case.id';
+// =============================================================================
+// IO attributes (used by Python SDK, not yet standardized)
+// =============================================================================
+/** Test case input (prompt) */
+export const ATTR_TEST_CASE_INPUT = 'test.case.input';
+/** Test case output (agent response) */
+export const ATTR_TEST_CASE_OUTPUT = 'test.case.output';
+/** Test case expected outcome */
+export const ATTR_TEST_CASE_EXPECTED = 'test.case.expected';
+// =============================================================================
+// Agent Health extension attributes (agent_health.* prefix)
+// =============================================================================
+/** Model ID used by the LLM judge */
+export const ATTR_AGENT_HEALTH_JUDGE_MODEL_ID = 'agent_health.judge.model_id';
+/** Time spent in LLM judge evaluation (ms) */
+export const ATTR_AGENT_HEALTH_JUDGE_DURATION_MS = 'agent_health.judge.duration_ms';
+/** Number of retry attempts for the LLM judge */
+export const ATTR_AGENT_HEALTH_JUDGE_ATTEMPTS = 'agent_health.judge.attempts';
+/** Time spent in agent execution (ms) */
+export const ATTR_AGENT_HEALTH_AGENT_DURATION_MS = 'agent_health.agent.duration_ms';
+/** Connector protocol used for agent communication */
+export const ATTR_AGENT_HEALTH_CONNECTOR_PROTOCOL = 'agent_health.connector.protocol';
+/**
+ * Agent run ID — links eval spans to the agent execution trace (Strategy B).
+ *
+ * Uses Agent Health's own namespace rather than the OpenTelemetry-reserved
+ * `gen_ai.*` namespace: `gen_ai.request.id` is not a registered semantic
+ * convention attribute (the `gen_ai.request.*` namespace is for request
+ * parameters), and the OTEL naming spec advises against adding app-specific
+ * keys under an existing OTEL namespace. W3C trace context (shared traceId,
+ * Strategy A) remains the primary correlation; this attribute is the loose
+ * link for agents that don't propagate context.
+ */
+export const ATTR_AGENT_HEALTH_AGENT_RUN_ID = 'agent_health.run.id';
+// =============================================================================
+// Constants
+// =============================================================================
+/** Operation name value for evaluation spans */
+export const GEN_AI_OPERATION_NAME_VALUE_EVALUATION = 'evaluation';
+/**
+ * OTEL-registered (incubating) conversation/session identifier —
+ * `gen_ai.conversation.id` (model/gen-ai/registry.yaml): "the unique
+ * identifier for a conversation (session, thread), used to store and correlate
+ * messages within this conversation." We stamp this (= the agent run id) on our
+ * eval + sample-agent spans so trace queries can correlate on the OTEL-standard
+ * attribute in addition to our own `agent_health.run.id`. Declared as a string
+ * literal (rather than re-exported from the incubating module) so it's stable
+ * regardless of semantic-conventions packaging. See AGENTS.md → Trace
+ * correlation conventions.
+ */
+export const ATTR_GEN_AI_CONVERSATION_ID = 'gen_ai.conversation.id';
+/** Maximum length for truncated attribute values */
+export const MAX_ATTRIBUTE_LENGTH = 10000;
+/** Maximum length for judge explanation in events */
+export const MAX_EXPLANATION_LENGTH = 4000;
+/** Tracer name used for evaluation spans */
+export const EVAL_TRACER_NAME = 'agent-health-eval';
+//# sourceMappingURL=constants.js.map

package/lib/dist/lib/telemetry/constants.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../../telemetry/constants.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;GAKG;AAEH,gFAAgF;AAChF,6CAA6C;AAC7C,uEAAuE;AACvE,gFAAgF;AAEhF,OAAO;AACL,6CAA6C;AAC7C,oBAAoB,EACpB,0BAA0B,EAC1B,mBAAmB,EACnB,4BAA4B,EAC5B,mCAAmC,EACnC,mCAAmC,EACnC,uCAAuC,EACvC,kCAAkC,EAClC,kCAAkC;AAElC,qEAAqE;AACrE,0BAA0B,EAC1B,2BAA2B,EAC3B,kCAAkC,EAClC,kCAAkC,EAClC,kCAAkC;AAElC,oDAAoD;AACpD,8BAA8B,GAC/B,MAAM,gDAAgD,CAAC;AAExD,gFAAgF;AAChF,sDAAsD;AACtD,qEAAqE;AACrE,gFAAgF;AAEhF,sDAAsD;AACtD,MAAM,CAAC,MAAM,sBAAsB,GAAG,mBAA4B,CAAC;AAEnE,yDAAyD;AACzD,MAAM,CAAC,MAAM,iBAAiB,GAAG,cAAuB,CAAC;AAEzD,gFAAgF;AAChF,2DAA2D;AAC3D,gFAAgF;AAEhF,+BAA+B;AAC/B,MAAM,CAAC,MAAM,oBAAoB,GAAG,iBAA0B,CAAC;AAE/D,wCAAwC;AACxC,MAAM,CAAC,MAAM,qBAAqB,GAAG,kBAA2B,CAAC;AAEjE,iCAAiC;AACjC,MAAM,CAAC,MAAM,uBAAuB,GAAG,oBAA6B,CAAC;AAErE,gFAAgF;AAChF,4DAA4D;AAC5D,gFAAgF;AAEhF,qCAAqC;AACrC,MAAM,CAAC,MAAM,gCAAgC,GAAG,6BAAsC,CAAC;AAEvF,8CAA8C;AAC9C,MAAM,CAAC,MAAM,mCAAmC,GAAG,gCAAyC,CAAC;AAE7F,iDAAiD;AACjD,MAAM,CAAC,MAAM,gCAAgC,GAAG,6BAAsC,CAAC;AAEvF,yCAAyC;AACzC,MAAM,CAAC,MAAM,mCAAmC,GAAG,gCAAyC,CAAC;AAE7F,sDAAsD;AACtD,MAAM,CAAC,MAAM,oCAAoC,GAAG,iCAA0C,CAAC;AAE/F;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,8BAA8B,GAAG,qBAA8B,CAAC;AAE7E,gFAAgF;AAChF,YAAY;AACZ,gFAAgF;AAEhF,gDAAgD;AAChD,MAAM,CAAC,MAAM,sCAAsC,GAAG,YAAqB,CAAC;AAE5E;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,wBAAiC,CAAC;AAE7E,oDAAoD;AACpD,MAAM,CAAC,MAAM,oBAAoB,GAAG,KAAK,CAAC;AAE1C,qDAAqD;AACrD,MAAM,CAAC,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAE3C,4CAA4C;AAC5C,MAAM,CAAC,MAAM,gBAAgB,GAAG,mBAAmB,CAAC"}

package/lib/dist/lib/telemetry/evalSpans.d.ts

@@ -0,0 +1,61 @@
+/**
+ * OTel span helpers for evaluation telemetry.
+ *
+ * Creates and manages spans for benchmark execution following
+ * OpenTelemetry semantic conventions for test suites and GenAI evaluation.
+ *
+ * Span structure:
+ *   test_suite_run {benchmarkName}     ← root span (1 per benchmark run)
+ *     └── test_case                    ← child span (1 per test case)
+ *           └── Event: gen_ai.evaluation.result
+ */
+import { type Context, type Span, type Link } from '@opentelemetry/api';
+import type { Benchmark, BenchmarkRun, TestCase, TestCaseRun } from '../../types/index.js';
+/**
+ * Start a test_suite_run root span for a benchmark run.
+ *
+ * Returns the span and a context carrying it for child span creation.
+ * The caller MUST call finalizeTestSuiteRunSpan() when the run completes.
+ */
+export declare function startTestSuiteRunSpan(benchmark: Benchmark, run: BenchmarkRun): {
+    span: Span;
+    context: Context;
+} | null;
+/**
+ * Start a test_case child span for a single test case evaluation.
+ *
+ * @param parentCtx - Context from the parent test_suite_run span
+ * @param testCase - The test case being evaluated
+ * @param benchmark - Parent benchmark (for name duplication)
+ * @param run - Parent benchmark run (for run ID duplication)
+ * @param agentRunId - Optional agent execution run ID (links eval span to agent traces)
+ * @param links - Optional span links (e.g., to agent execution trace)
+ */
+export declare function startTestCaseSpan(parentCtx: Context, testCase: TestCase, benchmark: Benchmark, run: BenchmarkRun, agentRunId?: string, links?: Link[]): {
+    span: Span;
+    context: Context;
+} | null;
+/**
+ * Add gen_ai.evaluation.result events to a test case span.
+ *
+ * Emits one event per metric (currently: accuracy with pass/fail label).
+ */
+export declare function addEvaluationResultEvents(span: Span, report: TestCaseRun): void;
+/**
+ * Finalize a test case span with evaluation results and end it.
+ */
+export declare function finalizeTestCaseSpan(span: Span, report: TestCaseRun, endTime?: Date): void;
+/**
+ * Finalize a test_suite_run span and end it.
+ */
+export declare function finalizeTestSuiteRunSpan(span: Span, run: BenchmarkRun): void;
+/**
+ * Emit a complete test case span for a deferred (trace-mode) evaluation.
+ *
+ * Used when the judge runs asynchronously after trace polling completes.
+ * Creates a standalone span with explicit timestamps reflecting the original execution.
+ */
+export declare function emitDeferredTestCaseSpan(testCase: TestCase, report: TestCaseRun, benchmark: {
+    name: string;
+}, runId: string, agentRunId?: string, startTime?: Date, endTime?: Date, agentTraceId?: string): void;
+//# sourceMappingURL=evalSpans.d.ts.map

package/lib/dist/lib/telemetry/evalSpans.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evalSpans.d.ts","sourceRoot":"","sources":["../../../telemetry/evalSpans.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;GAUG;AAEH,OAAO,EAIL,KAAK,OAAO,EACZ,KAAK,IAAI,EAIT,KAAK,IAAI,EACV,MAAM,oBAAoB,CAAC;AA0C5B,OAAO,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAoB9E;;;;;GAKG;AACH,wBAAgB,qBAAqB,CACnC,SAAS,EAAE,SAAS,EACpB,GAAG,EAAE,YAAY,GAChB;IAAE,IAAI,EAAE,IAAI,CAAC;IAAC,OAAO,EAAE,OAAO,CAAA;CAAE,GAAG,IAAI,CAkBzC;AAED;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAC/B,SAAS,EAAE,OAAO,EAClB,QAAQ,EAAE,QAAQ,EAClB,SAAS,EAAE,SAAS,EACpB,GAAG,EAAE,YAAY,EACjB,UAAU,CAAC,EAAE,MAAM,EACnB,KAAK,CAAC,EAAE,IAAI,EAAE,GACb;IAAE,IAAI,EAAE,IAAI,CAAC;IAAC,OAAO,EAAE,OAAO,CAAA;CAAE,GAAG,IAAI,CAsCzC;AAED;;;;GAIG;AACH,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,GAAG,IAAI,CAkC/E;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,IAAI,GAAG,IAAI,CAqC1F;AAED;;GAEG;AACH,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,YAAY,GAAG,IAAI,CAoB5E;AAED;;;;;GAKG;AACH,wBAAgB,wBAAwB,CACtC,QAAQ,EAAE,QAAQ,EAClB,MAAM,EAAE,WAAW,EACnB,SAAS,EAAE;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,EAC3B,KAAK,EAAE,MAAM,EACb,UAAU,CAAC,EAAE,MAAM,EACnB,SAAS,CAAC,EAAE,IAAI,EAChB,OAAO,CAAC,EAAE,IAAI,EACd,YAAY,CAAC,EAAE,MAAM,GACpB,IAAI,CAiDN"}

package/lib/dist/lib/telemetry/evalSpans.js

@@ -0,0 +1,254 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * OTel span helpers for evaluation telemetry.
+ *
+ * Creates and manages spans for benchmark execution following
+ * OpenTelemetry semantic conventions for test suites and GenAI evaluation.
+ *
+ * Span structure:
+ *   test_suite_run {benchmarkName}     ← root span (1 per benchmark run)
+ *     └── test_case                    ← child span (1 per test case)
+ *           └── Event: gen_ai.evaluation.result
+ */
+import { context, trace, TraceFlags, SpanKind, SpanStatusCode, } from '@opentelemetry/api';
+import { getEvalTracer, isEvalTelemetryEnabled, flushEvalTracer } from './provider.js';
+import { 
+// In-spec attributes
+ATTR_TEST_SUITE_NAME, ATTR_TEST_SUITE_RUN_STATUS, ATTR_TEST_CASE_NAME, ATTR_TEST_CASE_RESULT_STATUS, ATTR_GEN_AI_OPERATION_NAME, ATTR_GEN_AI_EVALUATION_NAME, ATTR_GEN_AI_CONVERSATION_ID, ATTR_GEN_AI_EVALUATION_SCORE_VALUE, ATTR_GEN_AI_EVALUATION_SCORE_LABEL, ATTR_GEN_AI_EVALUATION_EXPLANATION, EVENT_GEN_AI_EVALUATION_RESULT, TEST_SUITE_RUN_STATUS_VALUE_SUCCESS, TEST_SUITE_RUN_STATUS_VALUE_FAILURE, TEST_CASE_RESULT_STATUS_VALUE_PASS, TEST_CASE_RESULT_STATUS_VALUE_FAIL, 
+// Proposed attributes (#3398)
+ATTR_TEST_SUITE_RUN_ID, ATTR_TEST_CASE_ID, 
+// IO attributes
+ATTR_TEST_CASE_INPUT, ATTR_TEST_CASE_OUTPUT, ATTR_TEST_CASE_EXPECTED, 
+// Agent Health extensions
+ATTR_AGENT_HEALTH_JUDGE_MODEL_ID, ATTR_AGENT_HEALTH_JUDGE_DURATION_MS, ATTR_AGENT_HEALTH_JUDGE_ATTEMPTS, ATTR_AGENT_HEALTH_AGENT_DURATION_MS, ATTR_AGENT_HEALTH_CONNECTOR_PROTOCOL, ATTR_AGENT_HEALTH_AGENT_RUN_ID, 
+// Constants
+GEN_AI_OPERATION_NAME_VALUE_EVALUATION, MAX_ATTRIBUTE_LENGTH, MAX_EXPLANATION_LENGTH, } from './constants.js';
+/**
+ * Truncate a string to the given max length
+ */
+function truncate(value, maxLength) {
+    if (!value)
+        return undefined;
+    return value.length > maxLength ? value.slice(0, maxLength) + '...' : value;
+}
+/**
+ * Extract the agent's final response from a trajectory
+ */
+function extractAgentResponse(report) {
+    const responseStep = [...report.trajectory].reverse().find(s => s.type === 'response' || s.type === 'assistant');
+    return responseStep?.content;
+}
+/**
+ * Start a test_suite_run root span for a benchmark run.
+ *
+ * Returns the span and a context carrying it for child span creation.
+ * The caller MUST call finalizeTestSuiteRunSpan() when the run completes.
+ */
+export function startTestSuiteRunSpan(benchmark, run) {
+    if (!isEvalTelemetryEnabled())
+        return null;
+    const tracer = getEvalTracer();
+    const span = tracer.startSpan(`test_suite_run ${benchmark.name}`, {
+        kind: SpanKind.INTERNAL,
+        attributes: {
+            [ATTR_TEST_SUITE_NAME]: benchmark.name,
+            [ATTR_TEST_SUITE_RUN_ID]: run.id,
+            [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_EVALUATION,
+        },
+    });
+    const ctx = context.active();
+    const spanContext = trace.setSpan(ctx, span);
+    console.log(`[Telemetry] Started test_suite_run span for "${benchmark.name}" (runId=${run.id})`);
+    return { span, context: spanContext };
+}
+/**
+ * Start a test_case child span for a single test case evaluation.
+ *
+ * @param parentCtx - Context from the parent test_suite_run span
+ * @param testCase - The test case being evaluated
+ * @param benchmark - Parent benchmark (for name duplication)
+ * @param run - Parent benchmark run (for run ID duplication)
+ * @param agentRunId - Optional agent execution run ID (links eval span to agent traces)
+ * @param links - Optional span links (e.g., to agent execution trace)
+ */
+export function startTestCaseSpan(parentCtx, testCase, benchmark, run, agentRunId, links) {
+    if (!isEvalTelemetryEnabled())
+        return null;
+    const tracer = getEvalTracer();
+    const attributes = {
+        [ATTR_TEST_SUITE_NAME]: benchmark.name,
+        [ATTR_TEST_SUITE_RUN_ID]: run.id,
+        [ATTR_TEST_CASE_ID]: testCase.id,
+        [ATTR_TEST_CASE_NAME]: testCase.name,
+        [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_EVALUATION,
+    };
+    if (agentRunId) {
+        attributes[ATTR_AGENT_HEALTH_AGENT_RUN_ID] = agentRunId;
+        // OTEL-standard correlation id (incubating). Set to the same agent run id
+        // so trace queries correlating on `gen_ai.conversation.id` find this eval
+        // span alongside `agent_health.run.id` (Strategy B).
+        attributes[ATTR_GEN_AI_CONVERSATION_ID] = agentRunId;
+    }
+    attributes[ATTR_TEST_CASE_INPUT] = truncate(testCase.initialPrompt || '', MAX_ATTRIBUTE_LENGTH);
+    const expected = truncate(testCase.expectedOutcomes ? JSON.stringify(testCase.expectedOutcomes) : undefined, MAX_ATTRIBUTE_LENGTH);
+    if (expected)
+        attributes[ATTR_TEST_CASE_EXPECTED] = expected;
+    const span = tracer.startSpan('test_case', {
+        kind: SpanKind.INTERNAL,
+        attributes,
+        links,
+    }, parentCtx);
+    const ctx = trace.setSpan(parentCtx, span);
+    console.log(`[Telemetry] Started test_case span for "${testCase.name}" (agentRunId=${agentRunId || 'none'})`);
+    return { span, context: ctx };
+}
+/**
+ * Add gen_ai.evaluation.result events to a test case span.
+ *
+ * Emits one event per metric (currently: accuracy with pass/fail label).
+ */
+export function addEvaluationResultEvents(span, report) {
+    if (!isEvalTelemetryEnabled())
+        return;
+    // Primary metric: accuracy
+    if (report.metrics?.accuracy !== undefined) {
+        span.addEvent(EVENT_GEN_AI_EVALUATION_RESULT, {
+            [ATTR_GEN_AI_EVALUATION_NAME]: 'accuracy',
+            [ATTR_GEN_AI_EVALUATION_SCORE_VALUE]: report.metrics.accuracy,
+            [ATTR_GEN_AI_EVALUATION_SCORE_LABEL]: report.passFailStatus === 'passed' ? 'pass' : 'fail',
+            [ATTR_GEN_AI_EVALUATION_EXPLANATION]: truncate(report.llmJudgeReasoning, MAX_EXPLANATION_LENGTH),
+        });
+    }
+    // Additional metrics (faithfulness, latency_score, trajectory_alignment_score)
+    if (report.metrics?.faithfulness !== undefined) {
+        span.addEvent(EVENT_GEN_AI_EVALUATION_RESULT, {
+            [ATTR_GEN_AI_EVALUATION_NAME]: 'faithfulness',
+            [ATTR_GEN_AI_EVALUATION_SCORE_VALUE]: report.metrics.faithfulness,
+        });
+    }
+    if (report.metrics?.latency_score !== undefined) {
+        span.addEvent(EVENT_GEN_AI_EVALUATION_RESULT, {
+            [ATTR_GEN_AI_EVALUATION_NAME]: 'latency_score',
+            [ATTR_GEN_AI_EVALUATION_SCORE_VALUE]: report.metrics.latency_score,
+        });
+    }
+    if (report.metrics?.trajectory_alignment_score !== undefined) {
+        span.addEvent(EVENT_GEN_AI_EVALUATION_RESULT, {
+            [ATTR_GEN_AI_EVALUATION_NAME]: 'trajectory_alignment_score',
+            [ATTR_GEN_AI_EVALUATION_SCORE_VALUE]: report.metrics.trajectory_alignment_score,
+        });
+    }
+}
+/**
+ * Finalize a test case span with evaluation results and end it.
+ */
+export function finalizeTestCaseSpan(span, report, endTime) {
+    if (!isEvalTelemetryEnabled())
+        return;
+    // Set result status
+    const status = report.passFailStatus === 'passed'
+        ? TEST_CASE_RESULT_STATUS_VALUE_PASS
+        : TEST_CASE_RESULT_STATUS_VALUE_FAIL;
+    span.setAttribute(ATTR_TEST_CASE_RESULT_STATUS, status);
+    // Set output (agent's final response) — always set so UI can distinguish missing vs empty
+    const output = extractAgentResponse(report) || '';
+    span.setAttribute(ATTR_TEST_CASE_OUTPUT, truncate(output, MAX_ATTRIBUTE_LENGTH));
+    // Set Agent Health extension attributes
+    if (report.connectorProtocol) {
+        span.setAttribute(ATTR_AGENT_HEALTH_CONNECTOR_PROTOCOL, report.connectorProtocol);
+    }
+    if (report.performanceMetrics?.agentDurationMs !== undefined) {
+        span.setAttribute(ATTR_AGENT_HEALTH_AGENT_DURATION_MS, report.performanceMetrics.agentDurationMs);
+    }
+    if (report.performanceMetrics?.judgeDurationMs !== undefined) {
+        span.setAttribute(ATTR_AGENT_HEALTH_JUDGE_DURATION_MS, report.performanceMetrics.judgeDurationMs);
+    }
+    if (report.performanceMetrics?.judgeAttempts !== undefined) {
+        span.setAttribute(ATTR_AGENT_HEALTH_JUDGE_ATTEMPTS, report.performanceMetrics.judgeAttempts);
+    }
+    if (report.llmJudgeResponse?.modelId) {
+        span.setAttribute(ATTR_AGENT_HEALTH_JUDGE_MODEL_ID, report.llmJudgeResponse.modelId);
+    }
+    // Set span status based on evaluation outcome
+    if (report.status === 'failed') {
+        span.setStatus({ code: SpanStatusCode.ERROR, message: 'Evaluation failed' });
+    }
+    span.end(endTime);
+    console.log(`[Telemetry] Finalized test_case span (status=${status}, passFailStatus=${report.passFailStatus})`);
+}
+/**
+ * Finalize a test_suite_run span and end it.
+ */
+export function finalizeTestSuiteRunSpan(span, run) {
+    if (!isEvalTelemetryEnabled())
+        return;
+    // Compute overall status from results
+    const results = Object.values(run.results);
+    const hasFailures = results.some(r => r.status === 'failed');
+    const status = hasFailures
+        ? TEST_SUITE_RUN_STATUS_VALUE_FAILURE
+        : TEST_SUITE_RUN_STATUS_VALUE_SUCCESS;
+    span.setAttribute(ATTR_TEST_SUITE_RUN_STATUS, status);
+    if (hasFailures) {
+        const failCount = results.filter(r => r.status === 'failed').length;
+        span.setStatus({ code: SpanStatusCode.ERROR, message: `${failCount} case(s) failed` });
+    }
+    span.end();
+    console.log(`[Telemetry] Finalized test_suite_run span (status=${status}), flushing...`);
+    flushEvalTracer().catch(() => { });
+}
+/**
+ * Emit a complete test case span for a deferred (trace-mode) evaluation.
+ *
+ * Used when the judge runs asynchronously after trace polling completes.
+ * Creates a standalone span with explicit timestamps reflecting the original execution.
+ */
+export function emitDeferredTestCaseSpan(testCase, report, benchmark, runId, agentRunId, startTime, endTime, agentTraceId) {
+    if (!isEvalTelemetryEnabled())
+        return;
+    const tracer = getEvalTracer();
+    const attributes = {
+        [ATTR_TEST_SUITE_NAME]: benchmark.name,
+        [ATTR_TEST_SUITE_RUN_ID]: runId,
+        [ATTR_TEST_CASE_ID]: testCase.id,
+        [ATTR_TEST_CASE_NAME]: testCase.name,
+        [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_EVALUATION,
+    };
+    if (agentRunId) {
+        attributes[ATTR_AGENT_HEALTH_AGENT_RUN_ID] = agentRunId;
+        attributes[ATTR_GEN_AI_CONVERSATION_ID] = agentRunId;
+    }
+    attributes[ATTR_TEST_CASE_INPUT] = truncate(testCase.initialPrompt || '', MAX_ATTRIBUTE_LENGTH);
+    const expected = truncate(testCase.expectedOutcomes ? JSON.stringify(testCase.expectedOutcomes) : undefined, MAX_ATTRIBUTE_LENGTH);
+    if (expected)
+        attributes[ATTR_TEST_CASE_EXPECTED] = expected;
+    // If an agent traceId is provided, create the span within the same trace
+    // so it appears as a sibling of the agent's root span in the trace tree.
+    let parentCtx;
+    if (agentTraceId && /^[0-9a-f]{32}$/i.test(agentTraceId)) {
+        const remoteSpanContext = {
+            traceId: agentTraceId,
+            // Use a valid synthetic spanId — an all-zero spanId is considered invalid
+            // by the OTel SDK and would cause it to ignore this context entirely.
+            // This "remote" context carries the traceId forward so the eval span
+            // shares the same trace as the agent spans.
+            spanId: 'eeee000000000001',
+            traceFlags: TraceFlags.SAMPLED,
+            isRemote: true,
+        };
+        parentCtx = trace.setSpanContext(context.active(), remoteSpanContext);
+    }
+    const span = tracer.startSpan('test_case', {
+        kind: SpanKind.INTERNAL,
+        startTime,
+        attributes,
+    }, parentCtx);
+    addEvaluationResultEvents(span, report);
+    finalizeTestCaseSpan(span, report, endTime);
+    console.log(`[Telemetry] Emitted deferred test_case span for "${testCase.name}" (agentRunId=${agentRunId || 'none'}, traceId=${agentTraceId || 'new'}), flushing...`);
+    flushEvalTracer().catch(() => { });
+}
+//# sourceMappingURL=evalSpans.js.map

package/lib/dist/lib/telemetry/evalSpans.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evalSpans.js","sourceRoot":"","sources":["../../../telemetry/evalSpans.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;GAUG;AAEH,OAAO,EACL,OAAO,EACP,KAAK,EACL,UAAU,EAIV,QAAQ,EACR,cAAc,GAEf,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,aAAa,EAAE,sBAAsB,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACvF,OAAO;AACL,qBAAqB;AACrB,oBAAoB,EACpB,0BAA0B,EAC1B,mBAAmB,EACnB,4BAA4B,EAC5B,0BAA0B,EAC1B,2BAA2B,EAC3B,2BAA2B,EAC3B,kCAAkC,EAClC,kCAAkC,EAClC,kCAAkC,EAClC,8BAA8B,EAC9B,mCAAmC,EACnC,mCAAmC,EACnC,kCAAkC,EAClC,kCAAkC;AAElC,8BAA8B;AAC9B,sBAAsB,EACtB,iBAAiB;AAEjB,gBAAgB;AAChB,oBAAoB,EACpB,qBAAqB,EACrB,uBAAuB;AAEvB,0BAA0B;AAC1B,gCAAgC,EAChC,mCAAmC,EACnC,gCAAgC,EAChC,mCAAmC,EACnC,oCAAoC,EACpC,8BAA8B;AAE9B,YAAY;AACZ,sCAAsC,EACtC,oBAAoB,EACpB,sBAAsB,GACvB,MAAM,gBAAgB,CAAC;AAGxB;;GAEG;AACH,SAAS,QAAQ,CAAC,KAAyB,EAAE,SAAiB;IAC5D,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,OAAO,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;AAC9E,CAAC;AAED;;GAEG;AACH,SAAS,oBAAoB,CAAC,MAAmB;IAC/C,MAAM,YAAY,GAAG,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,OAAO,EAAE,CAAC,IAAI,CACxD,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,CACrD,CAAC;IACF,OAAO,YAAY,EAAE,OAAO,CAAC;AAC/B,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CACnC,SAAoB,EACpB,GAAiB;IAEjB,IAAI,CAAC,sBAAsB,EAAE;QAAE,OAAO,IAAI,CAAC;IAE3C,MAAM,MAAM,GAAG,aAAa,EAAE,CAAC;IAC/B,MAAM,IAAI,GAAG,MAAM,CAAC,SAAS,CAAC,kBAAkB,SAAS,CAAC,IAAI,EAAE,EAAE;QAChE,IAAI,EAAE,QAAQ,CAAC,QAAQ;QACvB,UAAU,EAAE;YACV,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,IAAI;YACtC,CAAC,sBAAsB,CAAC,EAAE,GAAG,CAAC,EAAE;YAChC,CAAC,0BAA0B,CAAC,EAAE,sCAAsC;SACrE;KACF,CAAC,CAAC;IAEH,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAC7B,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAE7C,OAAO,CAAC,GAAG,CAAC,gDAAgD,SAAS,CAAC,IAAI,YAAY,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;IACjG,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;AACxC,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,iBAAiB,CAC/B,SAAkB,EAClB,QAAkB,EAClB,SAAoB,EACpB,GAAiB,EACjB,UAAmB,EACnB,KAAc;IAEd,IAAI,CAAC,sBAAsB,EAAE;QAAE,OAAO,IAAI,CAAC;IAE3C,MAAM,MAAM,GAAG,aAAa,EAAE,CAAC;IAC/B,MAAM,UAAU,GAA2B;QACzC,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,IAAI;QACtC,CAAC,sBAAsB,CAAC,EAAE,GAAG,CAAC,EAAE;QAChC,CAAC,iBAAiB,CAAC,EAAE,QAAQ,CAAC,EAAE;QAChC,CAAC,mBAAmB,CAAC,EAAE,QAAQ,CAAC,IAAI;QACpC,CAAC,0BAA0B,CAAC,EAAE,sCAAsC;KACrE,CAAC;IACF,IAAI,UAAU,EAAE,CAAC;QACf,UAAU,CAAC,8BAA8B,CAAC,GAAG,UAAU,CAAC;QACxD,0EAA0E;QAC1E,0EAA0E;QAC1E,qDAAqD;QACrD,UAAU,CAAC,2BAA2B,CAAC,GAAG,UAAU,CAAC;IACvD,CAAC;IACD,UAAU,CAAC,oBAAoB,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,aAAa,IAAI,EAAE,EAAE,oBAAoB,CAAE,CAAC;IACjG,MAAM,QAAQ,GAAG,QAAQ,CACvB,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,SAAS,EACjF,oBAAoB,CACrB,CAAC;IACF,IAAI,QAAQ;QAAE,UAAU,CAAC,uBAAuB,CAAC,GAAG,QAAQ,CAAC;IAE7D,MAAM,IAAI,GAAG,MAAM,CAAC,SAAS,CAC3B,WAAW,EACX;QACE,IAAI,EAAE,QAAQ,CAAC,QAAQ;QACvB,UAAU;QACV,KAAK;KACN,EACD,SAAS,CACV,CAAC;IAEF,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAC3C,OAAO,CAAC,GAAG,CAAC,2CAA2C,QAAQ,CAAC,IAAI,iBAAiB,UAAU,IAAI,MAAM,GAAG,CAAC,CAAC;IAC9G,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;AAChC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CAAC,IAAU,EAAE,MAAmB;IACvE,IAAI,CAAC,sBAAsB,EAAE;QAAE,OAAO;IAEtC,2BAA2B;IAC3B,IAAI,MAAM,CAAC,OAAO,EAAE,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3C,IAAI,CAAC,QAAQ,CAAC,8BAA8B,EAAE;YAC5C,CAAC,2BAA2B,CAAC,EAAE,UAAU;YACzC,CAAC,kCAAkC,CAAC,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ;YAC7D,CAAC,kCAAkC,CAAC,EAAE,MAAM,CAAC,cAAc,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAC1F,CAAC,kCAAkC,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,iBAAiB,EAAE,sBAAsB,CAAC;SACjG,CAAC,CAAC;IACL,CAAC;IAED,+EAA+E;IAC/E,IAAI,MAAM,CAAC,OAAO,EAAE,YAAY,KAAK,SAAS,EAAE,CAAC;QAC/C,IAAI,CAAC,QAAQ,CAAC,8BAA8B,EAAE;YAC5C,CAAC,2BAA2B,CAAC,EAAE,cAAc;YAC7C,CAAC,kCAAkC,CAAC,EAAE,MAAM,CAAC,OAAO,CAAC,YAAY;SAClE,CAAC,CAAC;IACL,CAAC;IAED,IAAI,MAAM,CAAC,OAAO,EAAE,aAAa,KAAK,SAAS,EAAE,CAAC;QAChD,IAAI,CAAC,QAAQ,CAAC,8BAA8B,EAAE;YAC5C,CAAC,2BAA2B,CAAC,EAAE,eAAe;YAC9C,CAAC,kCAAkC,CAAC,EAAE,MAAM,CAAC,OAAO,CAAC,aAAa;SACnE,CAAC,CAAC;IACL,CAAC;IAED,IAAI,MAAM,CAAC,OAAO,EAAE,0BAA0B,KAAK,SAAS,EAAE,CAAC;QAC7D,IAAI,CAAC,QAAQ,CAAC,8BAA8B,EAAE;YAC5C,CAAC,2BAA2B,CAAC,EAAE,4BAA4B;YAC3D,CAAC,kCAAkC,CAAC,EAAE,MAAM,CAAC,OAAO,CAAC,0BAA0B;SAChF,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAU,EAAE,MAAmB,EAAE,OAAc;IAClF,IAAI,CAAC,sBAAsB,EAAE;QAAE,OAAO;IAEtC,oBAAoB;IACpB,MAAM,MAAM,GAAG,MAAM,CAAC,cAAc,KAAK,QAAQ;QAC/C,CAAC,CAAC,kCAAkC;QACpC,CAAC,CAAC,kCAAkC,CAAC;IACvC,IAAI,CAAC,YAAY,CAAC,4BAA4B,EAAE,MAAM,CAAC,CAAC;IAExD,0FAA0F;IAC1F,MAAM,MAAM,GAAG,oBAAoB,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IAClD,IAAI,CAAC,YAAY,CAAC,qBAAqB,EAAE,QAAQ,CAAC,MAAM,EAAE,oBAAoB,CAAE,CAAC,CAAC;IAElF,wCAAwC;IACxC,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;QAC7B,IAAI,CAAC,YAAY,CAAC,oCAAoC,EAAE,MAAM,CAAC,iBAAiB,CAAC,CAAC;IACpF,CAAC;IACD,IAAI,MAAM,CAAC,kBAAkB,EAAE,eAAe,KAAK,SAAS,EAAE,CAAC;QAC7D,IAAI,CAAC,YAAY,CAAC,mCAAmC,EAAE,MAAM,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;IACpG,CAAC;IACD,IAAI,MAAM,CAAC,kBAAkB,EAAE,eAAe,KAAK,SAAS,EAAE,CAAC;QAC7D,IAAI,CAAC,YAAY,CAAC,mCAAmC,EAAE,MAAM,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;IACpG,CAAC;IACD,IAAI,MAAM,CAAC,kBAAkB,EAAE,aAAa,KAAK,SAAS,EAAE,CAAC;QAC3D,IAAI,CAAC,YAAY,CAAC,gCAAgC,EAAE,MAAM,CAAC,kBAAkB,CAAC,aAAa,CAAC,CAAC;IAC/F,CAAC;IACD,IAAI,MAAM,CAAC,gBAAgB,EAAE,OAAO,EAAE,CAAC;QACrC,IAAI,CAAC,YAAY,CAAC,gCAAgC,EAAE,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;IACvF,CAAC;IAED,8CAA8C;IAC9C,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,cAAc,CAAC,KAAK,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC,CAAC;IAC/E,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAClB,OAAO,CAAC,GAAG,CAAC,gDAAgD,MAAM,oBAAoB,MAAM,CAAC,cAAc,GAAG,CAAC,CAAC;AAClH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,wBAAwB,CAAC,IAAU,EAAE,GAAiB;IACpE,IAAI,CAAC,sBAAsB,EAAE;QAAE,OAAO;IAEtC,sCAAsC;IACtC,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3C,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,WAAW;QACxB,CAAC,CAAC,mCAAmC;QACrC,CAAC,CAAC,mCAAmC,CAAC;IAExC,IAAI,CAAC,YAAY,CAAC,0BAA0B,EAAE,MAAM,CAAC,CAAC;IAEtD,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;QACpE,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,cAAc,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,SAAS,iBAAiB,EAAE,CAAC,CAAC;IACzF,CAAC;IAED,IAAI,CAAC,GAAG,EAAE,CAAC;IACX,OAAO,CAAC,GAAG,CAAC,qDAAqD,MAAM,gBAAgB,CAAC,CAAC;IACzF,eAAe,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;AACpC,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,wBAAwB,CACtC,QAAkB,EAClB,MAAmB,EACnB,SAA2B,EAC3B,KAAa,EACb,UAAmB,EACnB,SAAgB,EAChB,OAAc,EACd,YAAqB;IAErB,IAAI,CAAC,sBAAsB,EAAE;QAAE,OAAO;IAEtC,MAAM,MAAM,GAAG,aAAa,EAAE,CAAC;IAC/B,MAAM,UAAU,GAA2B;QACzC,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,IAAI;QACtC,CAAC,sBAAsB,CAAC,EAAE,KAAK;QAC/B,CAAC,iBAAiB,CAAC,EAAE,QAAQ,CAAC,EAAE;QAChC,CAAC,mBAAmB,CAAC,EAAE,QAAQ,CAAC,IAAI;QACpC,CAAC,0BAA0B,CAAC,EAAE,sCAAsC;KACrE,CAAC;IACF,IAAI,UAAU,EAAE,CAAC;QACf,UAAU,CAAC,8BAA8B,CAAC,GAAG,UAAU,CAAC;QACxD,UAAU,CAAC,2BAA2B,CAAC,GAAG,UAAU,CAAC;IACvD,CAAC;IACD,UAAU,CAAC,oBAAoB,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,aAAa,IAAI,EAAE,EAAE,oBAAoB,CAAE,CAAC;IACjG,MAAM,QAAQ,GAAG,QAAQ,CACvB,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,SAAS,EACjF,oBAAoB,CACrB,CAAC;IACF,IAAI,QAAQ;QAAE,UAAU,CAAC,uBAAuB,CAAC,GAAG,QAAQ,CAAC;IAE7D,yEAAyE;IACzE,yEAAyE;IACzE,IAAI,SAA8B,CAAC;IACnC,IAAI,YAAY,IAAI,iBAAiB,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;QACzD,MAAM,iBAAiB,GAAgB;YACrC,OAAO,EAAE,YAAY;YACrB,0EAA0E;YAC1E,sEAAsE;YACtE,qEAAqE;YACrE,4CAA4C;YAC5C,MAAM,EAAE,kBAAkB;YAC1B,UAAU,EAAE,UAAU,CAAC,OAAO;YAC9B,QAAQ,EAAE,IAAI;SACf,CAAC;QACF,SAAS,GAAG,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,iBAAiB,CAAC,CAAC;IACxE,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,CAAC,SAAS,CAAC,WAAW,EAAE;QACzC,IAAI,EAAE,QAAQ,CAAC,QAAQ;QACvB,SAAS;QACT,UAAU;KACX,EAAE,SAAS,CAAC,CAAC;IAEd,yBAAyB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACxC,oBAAoB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAC5C,OAAO,CAAC,GAAG,CAAC,oDAAoD,QAAQ,CAAC,IAAI,iBAAiB,UAAU,IAAI,MAAM,aAAa,YAAY,IAAI,KAAK,gBAAgB,CAAC,CAAC;IACtK,eAAe,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;AACpC,CAAC"}

package/lib/dist/lib/telemetry/index.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Evaluation Telemetry Module
+ *
+ * Emits OTel spans for benchmark evaluation data using semantic conventions.
+ * Disabled by default — enable via OTEL_EVAL_ENABLED=true or config file.
+ */
+export { initEvalTracerProvider, getEvalTracer, isEvalTelemetryEnabled, flushEvalTracer, shutdownEvalTracer, resolveEvalTelemetryConfig, type EvalTelemetryConfig, } from './provider.js';
+export { startTestSuiteRunSpan, startTestCaseSpan, addEvaluationResultEvents, finalizeTestCaseSpan, finalizeTestSuiteRunSpan, emitDeferredTestCaseSpan, } from './evalSpans.js';
+export * from './constants.js';
+export { OpenSearchSpanExporter, type OpenSearchExporterConfig, } from './opensearchExporter.js';
+//# sourceMappingURL=index.d.ts.map

package/lib/dist/lib/telemetry/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../telemetry/index.ts"],"names":[],"mappings":"AAKA;;;;;GAKG;AAEH,OAAO,EACL,sBAAsB,EACtB,aAAa,EACb,sBAAsB,EACtB,eAAe,EACf,kBAAkB,EAClB,0BAA0B,EAC1B,KAAK,mBAAmB,GACzB,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,qBAAqB,EACrB,iBAAiB,EACjB,yBAAyB,EACzB,oBAAoB,EACpB,wBAAwB,EACxB,wBAAwB,GACzB,MAAM,gBAAgB,CAAC;AAExB,cAAc,gBAAgB,CAAC;AAE/B,OAAO,EACL,sBAAsB,EACtB,KAAK,wBAAwB,GAC9B,MAAM,yBAAyB,CAAC"}

package/lib/dist/lib/telemetry/index.js

@@ -0,0 +1,15 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * Evaluation Telemetry Module
+ *
+ * Emits OTel spans for benchmark evaluation data using semantic conventions.
+ * Disabled by default — enable via OTEL_EVAL_ENABLED=true or config file.
+ */
+export { initEvalTracerProvider, getEvalTracer, isEvalTelemetryEnabled, flushEvalTracer, shutdownEvalTracer, resolveEvalTelemetryConfig, } from './provider.js';
+export { startTestSuiteRunSpan, startTestCaseSpan, addEvaluationResultEvents, finalizeTestCaseSpan, finalizeTestSuiteRunSpan, emitDeferredTestCaseSpan, } from './evalSpans.js';
+export * from './constants.js';
+export { OpenSearchSpanExporter, } from './opensearchExporter.js';
+//# sourceMappingURL=index.js.map

package/lib/dist/lib/telemetry/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../telemetry/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;GAKG;AAEH,OAAO,EACL,sBAAsB,EACtB,aAAa,EACb,sBAAsB,EACtB,eAAe,EACf,kBAAkB,EAClB,0BAA0B,GAE3B,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,qBAAqB,EACrB,iBAAiB,EACjB,yBAAyB,EACzB,oBAAoB,EACpB,wBAAwB,EACxB,wBAAwB,GACzB,MAAM,gBAAgB,CAAC;AAExB,cAAc,gBAAgB,CAAC;AAE/B,OAAO,EACL,sBAAsB,GAEvB,MAAM,yBAAyB,CAAC"}

package/lib/dist/lib/telemetry/opensearchExporter.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Custom OpenSearch Span Exporter
+ *
+ * Writes OTel spans directly to an OpenSearch cluster using the same
+ * observability data source config that the trace reader uses.
+ * Formats spans in the OTEL-faithful Data Prepper `trace-analytics-plain-raw`
+ * schema (a nested `attributes` object keyed by literal dotted OTel names)
+ * so they co-exist with agent spans already in the otel-v1-apm-span-* index.
+ */
+import { SpanExporter, ReadableSpan } from '@opentelemetry/sdk-trace-node';
+import { ExportResult } from '@opentelemetry/core';
+/**
+ * Configuration for the OpenSearch exporter
+ */
+export interface OpenSearchExporterConfig {
+    /** OpenSearch endpoint URL */
+    endpoint: string;
+    /** Basic auth username */
+    username?: string;
+    /** Basic auth password */
+    password?: string;
+    /** Index pattern to write to (default: otel-v1-apm-span-000001) */
+    indexName?: string;
+    /** Whether to skip TLS verification */
+    tlsSkipVerify?: boolean;
+}
+/**
+ * Custom SpanExporter that writes directly to OpenSearch.
+ *
+ * Formats spans in OSIS-compatible format so eval spans co-exist
+ * with agent spans in the same otel-v1-apm-span-* indices.
+ */
+export declare class OpenSearchSpanExporter implements SpanExporter {
+    private client;
+    private indexName;
+    private shutdownRequested;
+    constructor(config: OpenSearchExporterConfig);
+    export(spans: ReadableSpan[], resultCallback: (result: ExportResult) => void): void;
+    private _export;
+    shutdown(): Promise<void>;
+    forceFlush(): Promise<void>;
+}
+//# sourceMappingURL=opensearchExporter.d.ts.map

package/lib/dist/lib/telemetry/opensearchExporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"opensearchExporter.d.ts","sourceRoot":"","sources":["../../../telemetry/opensearchExporter.ts"],"names":[],"mappings":"AAKA;;;;;;;;GAQG;AAEH,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAC3E,OAAO,EAAE,YAAY,EAAoB,MAAM,qBAAqB,CAAC;AAIrE;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,8BAA8B;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,0BAA0B;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,0BAA0B;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,mEAAmE;IACnE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AA6JD;;;;;GAKG;AACH,qBAAa,sBAAuB,YAAW,YAAY;IACzD,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,iBAAiB,CAAS;gBAEtB,MAAM,EAAE,wBAAwB;IAiB5C,MAAM,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE,cAAc,EAAE,CAAC,MAAM,EAAE,YAAY,KAAK,IAAI,GAAG,IAAI;YAcrE,OAAO;IA0Bf,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAKzB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;CAGlC"}