@opensearch-project/agent-health 0.4.0 → 0.6.0

package/lib/dist/lib/telemetry/evalSpans.js

@@ -0,0 +1,254 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * OTel span helpers for evaluation telemetry.
+ *
+ * Creates and manages spans for benchmark execution following
+ * OpenTelemetry semantic conventions for test suites and GenAI evaluation.
+ *
+ * Span structure:
+ *   test_suite_run {benchmarkName}     ← root span (1 per benchmark run)
+ *     └── test_case                    ← child span (1 per test case)
+ *           └── Event: gen_ai.evaluation.result
+ */
+import { context, trace, TraceFlags, SpanKind, SpanStatusCode, } from '@opentelemetry/api';
+import { getEvalTracer, isEvalTelemetryEnabled, flushEvalTracer } from './provider.js';
+import { 
+// In-spec attributes
+ATTR_TEST_SUITE_NAME, ATTR_TEST_SUITE_RUN_STATUS, ATTR_TEST_CASE_NAME, ATTR_TEST_CASE_RESULT_STATUS, ATTR_GEN_AI_OPERATION_NAME, ATTR_GEN_AI_EVALUATION_NAME, ATTR_GEN_AI_CONVERSATION_ID, ATTR_GEN_AI_EVALUATION_SCORE_VALUE, ATTR_GEN_AI_EVALUATION_SCORE_LABEL, ATTR_GEN_AI_EVALUATION_EXPLANATION, EVENT_GEN_AI_EVALUATION_RESULT, TEST_SUITE_RUN_STATUS_VALUE_SUCCESS, TEST_SUITE_RUN_STATUS_VALUE_FAILURE, TEST_CASE_RESULT_STATUS_VALUE_PASS, TEST_CASE_RESULT_STATUS_VALUE_FAIL, 
+// Proposed attributes (#3398)
+ATTR_TEST_SUITE_RUN_ID, ATTR_TEST_CASE_ID, 
+// IO attributes
+ATTR_TEST_CASE_INPUT, ATTR_TEST_CASE_OUTPUT, ATTR_TEST_CASE_EXPECTED, 
+// Agent Health extensions
+ATTR_AGENT_HEALTH_JUDGE_MODEL_ID, ATTR_AGENT_HEALTH_JUDGE_DURATION_MS, ATTR_AGENT_HEALTH_JUDGE_ATTEMPTS, ATTR_AGENT_HEALTH_AGENT_DURATION_MS, ATTR_AGENT_HEALTH_CONNECTOR_PROTOCOL, ATTR_AGENT_HEALTH_AGENT_RUN_ID, 
+// Constants
+GEN_AI_OPERATION_NAME_VALUE_EVALUATION, MAX_ATTRIBUTE_LENGTH, MAX_EXPLANATION_LENGTH, } from './constants.js';
+/**
+ * Truncate a string to the given max length
+ */
+function truncate(value, maxLength) {
+    if (!value)
+        return undefined;
+    return value.length > maxLength ? value.slice(0, maxLength) + '...' : value;
+}
+/**
+ * Extract the agent's final response from a trajectory
+ */
+function extractAgentResponse(report) {
+    const responseStep = [...report.trajectory].reverse().find(s => s.type === 'response' || s.type === 'assistant');
+    return responseStep?.content;
+}
+/**
+ * Start a test_suite_run root span for a benchmark run.
+ *
+ * Returns the span and a context carrying it for child span creation.
+ * The caller MUST call finalizeTestSuiteRunSpan() when the run completes.
+ */
+export function startTestSuiteRunSpan(benchmark, run) {
+    if (!isEvalTelemetryEnabled())
+        return null;
+    const tracer = getEvalTracer();
+    const span = tracer.startSpan(`test_suite_run ${benchmark.name}`, {
+        kind: SpanKind.INTERNAL,
+        attributes: {
+            [ATTR_TEST_SUITE_NAME]: benchmark.name,
+            [ATTR_TEST_SUITE_RUN_ID]: run.id,
+            [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_EVALUATION,
+        },
+    });
+    const ctx = context.active();
+    const spanContext = trace.setSpan(ctx, span);
+    console.log(`[Telemetry] Started test_suite_run span for "${benchmark.name}" (runId=${run.id})`);
+    return { span, context: spanContext };
+}
+/**
+ * Start a test_case child span for a single test case evaluation.
+ *
+ * @param parentCtx - Context from the parent test_suite_run span
+ * @param testCase - The test case being evaluated
+ * @param benchmark - Parent benchmark (for name duplication)
+ * @param run - Parent benchmark run (for run ID duplication)
+ * @param agentRunId - Optional agent execution run ID (links eval span to agent traces)
+ * @param links - Optional span links (e.g., to agent execution trace)
+ */
+export function startTestCaseSpan(parentCtx, testCase, benchmark, run, agentRunId, links) {
+    if (!isEvalTelemetryEnabled())
+        return null;
+    const tracer = getEvalTracer();
+    const attributes = {
+        [ATTR_TEST_SUITE_NAME]: benchmark.name,
+        [ATTR_TEST_SUITE_RUN_ID]: run.id,
+        [ATTR_TEST_CASE_ID]: testCase.id,
+        [ATTR_TEST_CASE_NAME]: testCase.name,
+        [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_EVALUATION,
+    };
+    if (agentRunId) {
+        attributes[ATTR_AGENT_HEALTH_AGENT_RUN_ID] = agentRunId;
+        // OTEL-standard correlation id (incubating). Set to the same agent run id
+        // so trace queries correlating on `gen_ai.conversation.id` find this eval
+        // span alongside `agent_health.run.id` (Strategy B).
+        attributes[ATTR_GEN_AI_CONVERSATION_ID] = agentRunId;
+    }
+    attributes[ATTR_TEST_CASE_INPUT] = truncate(testCase.initialPrompt || '', MAX_ATTRIBUTE_LENGTH);
+    const expected = truncate(testCase.expectedOutcomes ? JSON.stringify(testCase.expectedOutcomes) : undefined, MAX_ATTRIBUTE_LENGTH);
+    if (expected)
+        attributes[ATTR_TEST_CASE_EXPECTED] = expected;
+    const span = tracer.startSpan('test_case', {
+        kind: SpanKind.INTERNAL,
+        attributes,
+        links,
+    }, parentCtx);
+    const ctx = trace.setSpan(parentCtx, span);
+    console.log(`[Telemetry] Started test_case span for "${testCase.name}" (agentRunId=${agentRunId || 'none'})`);
+    return { span, context: ctx };
+}
+/**
+ * Add gen_ai.evaluation.result events to a test case span.
+ *
+ * Emits one event per metric (currently: accuracy with pass/fail label).
+ */
+export function addEvaluationResultEvents(span, report) {
+    if (!isEvalTelemetryEnabled())
+        return;
+    // Primary metric: accuracy
+    if (report.metrics?.accuracy !== undefined) {
+        span.addEvent(EVENT_GEN_AI_EVALUATION_RESULT, {
+            [ATTR_GEN_AI_EVALUATION_NAME]: 'accuracy',
+            [ATTR_GEN_AI_EVALUATION_SCORE_VALUE]: report.metrics.accuracy,
+            [ATTR_GEN_AI_EVALUATION_SCORE_LABEL]: report.passFailStatus === 'passed' ? 'pass' : 'fail',
+            [ATTR_GEN_AI_EVALUATION_EXPLANATION]: truncate(report.llmJudgeReasoning, MAX_EXPLANATION_LENGTH),
+        });
+    }
+    // Additional metrics (faithfulness, latency_score, trajectory_alignment_score)
+    if (report.metrics?.faithfulness !== undefined) {
+        span.addEvent(EVENT_GEN_AI_EVALUATION_RESULT, {
+            [ATTR_GEN_AI_EVALUATION_NAME]: 'faithfulness',
+            [ATTR_GEN_AI_EVALUATION_SCORE_VALUE]: report.metrics.faithfulness,
+        });
+    }
+    if (report.metrics?.latency_score !== undefined) {
+        span.addEvent(EVENT_GEN_AI_EVALUATION_RESULT, {
+            [ATTR_GEN_AI_EVALUATION_NAME]: 'latency_score',
+            [ATTR_GEN_AI_EVALUATION_SCORE_VALUE]: report.metrics.latency_score,
+        });
+    }
+    if (report.metrics?.trajectory_alignment_score !== undefined) {
+        span.addEvent(EVENT_GEN_AI_EVALUATION_RESULT, {
+            [ATTR_GEN_AI_EVALUATION_NAME]: 'trajectory_alignment_score',
+            [ATTR_GEN_AI_EVALUATION_SCORE_VALUE]: report.metrics.trajectory_alignment_score,
+        });
+    }
+}
+/**
+ * Finalize a test case span with evaluation results and end it.
+ */
+export function finalizeTestCaseSpan(span, report, endTime) {
+    if (!isEvalTelemetryEnabled())
+        return;
+    // Set result status
+    const status = report.passFailStatus === 'passed'
+        ? TEST_CASE_RESULT_STATUS_VALUE_PASS
+        : TEST_CASE_RESULT_STATUS_VALUE_FAIL;
+    span.setAttribute(ATTR_TEST_CASE_RESULT_STATUS, status);
+    // Set output (agent's final response) — always set so UI can distinguish missing vs empty
+    const output = extractAgentResponse(report) || '';
+    span.setAttribute(ATTR_TEST_CASE_OUTPUT, truncate(output, MAX_ATTRIBUTE_LENGTH));
+    // Set Agent Health extension attributes
+    if (report.connectorProtocol) {
+        span.setAttribute(ATTR_AGENT_HEALTH_CONNECTOR_PROTOCOL, report.connectorProtocol);
+    }
+    if (report.performanceMetrics?.agentDurationMs !== undefined) {
+        span.setAttribute(ATTR_AGENT_HEALTH_AGENT_DURATION_MS, report.performanceMetrics.agentDurationMs);
+    }
+    if (report.performanceMetrics?.judgeDurationMs !== undefined) {
+        span.setAttribute(ATTR_AGENT_HEALTH_JUDGE_DURATION_MS, report.performanceMetrics.judgeDurationMs);
+    }
+    if (report.performanceMetrics?.judgeAttempts !== undefined) {
+        span.setAttribute(ATTR_AGENT_HEALTH_JUDGE_ATTEMPTS, report.performanceMetrics.judgeAttempts);
+    }
+    if (report.llmJudgeResponse?.modelId) {
+        span.setAttribute(ATTR_AGENT_HEALTH_JUDGE_MODEL_ID, report.llmJudgeResponse.modelId);
+    }
+    // Set span status based on evaluation outcome
+    if (report.status === 'failed') {
+        span.setStatus({ code: SpanStatusCode.ERROR, message: 'Evaluation failed' });
+    }
+    span.end(endTime);
+    console.log(`[Telemetry] Finalized test_case span (status=${status}, passFailStatus=${report.passFailStatus})`);
+}
+/**
+ * Finalize a test_suite_run span and end it.
+ */
+export function finalizeTestSuiteRunSpan(span, run) {
+    if (!isEvalTelemetryEnabled())
+        return;
+    // Compute overall status from results
+    const results = Object.values(run.results);
+    const hasFailures = results.some(r => r.status === 'failed');
+    const status = hasFailures
+        ? TEST_SUITE_RUN_STATUS_VALUE_FAILURE
+        : TEST_SUITE_RUN_STATUS_VALUE_SUCCESS;
+    span.setAttribute(ATTR_TEST_SUITE_RUN_STATUS, status);
+    if (hasFailures) {
+        const failCount = results.filter(r => r.status === 'failed').length;
+        span.setStatus({ code: SpanStatusCode.ERROR, message: `${failCount} case(s) failed` });
+    }
+    span.end();
+    console.log(`[Telemetry] Finalized test_suite_run span (status=${status}), flushing...`);
+    flushEvalTracer().catch(() => { });
+}
+/**
+ * Emit a complete test case span for a deferred (trace-mode) evaluation.
+ *
+ * Used when the judge runs asynchronously after trace polling completes.
+ * Creates a standalone span with explicit timestamps reflecting the original execution.
+ */
+export function emitDeferredTestCaseSpan(testCase, report, benchmark, runId, agentRunId, startTime, endTime, agentTraceId) {
+    if (!isEvalTelemetryEnabled())
+        return;
+    const tracer = getEvalTracer();
+    const attributes = {
+        [ATTR_TEST_SUITE_NAME]: benchmark.name,
+        [ATTR_TEST_SUITE_RUN_ID]: runId,
+        [ATTR_TEST_CASE_ID]: testCase.id,
+        [ATTR_TEST_CASE_NAME]: testCase.name,
+        [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_EVALUATION,
+    };
+    if (agentRunId) {
+        attributes[ATTR_AGENT_HEALTH_AGENT_RUN_ID] = agentRunId;
+        attributes[ATTR_GEN_AI_CONVERSATION_ID] = agentRunId;
+    }
+    attributes[ATTR_TEST_CASE_INPUT] = truncate(testCase.initialPrompt || '', MAX_ATTRIBUTE_LENGTH);
+    const expected = truncate(testCase.expectedOutcomes ? JSON.stringify(testCase.expectedOutcomes) : undefined, MAX_ATTRIBUTE_LENGTH);
+    if (expected)
+        attributes[ATTR_TEST_CASE_EXPECTED] = expected;
+    // If an agent traceId is provided, create the span within the same trace
+    // so it appears as a sibling of the agent's root span in the trace tree.
+    let parentCtx;
+    if (agentTraceId && /^[0-9a-f]{32}$/i.test(agentTraceId)) {
+        const remoteSpanContext = {
+            traceId: agentTraceId,
+            // Use a valid synthetic spanId — an all-zero spanId is considered invalid
+            // by the OTel SDK and would cause it to ignore this context entirely.
+            // This "remote" context carries the traceId forward so the eval span
+            // shares the same trace as the agent spans.
+            spanId: 'eeee000000000001',
+            traceFlags: TraceFlags.SAMPLED,
+            isRemote: true,
+        };
+        parentCtx = trace.setSpanContext(context.active(), remoteSpanContext);
+    }
+    const span = tracer.startSpan('test_case', {
+        kind: SpanKind.INTERNAL,
+        startTime,
+        attributes,
+    }, parentCtx);
+    addEvaluationResultEvents(span, report);
+    finalizeTestCaseSpan(span, report, endTime);
+    console.log(`[Telemetry] Emitted deferred test_case span for "${testCase.name}" (agentRunId=${agentRunId || 'none'}, traceId=${agentTraceId || 'new'}), flushing...`);
+    flushEvalTracer().catch(() => { });
+}
+//# sourceMappingURL=evalSpans.js.map

package/lib/dist/lib/telemetry/evalSpans.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evalSpans.js","sourceRoot":"","sources":["../../../telemetry/evalSpans.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;GAUG;AAEH,OAAO,EACL,OAAO,EACP,KAAK,EACL,UAAU,EAIV,QAAQ,EACR,cAAc,GAEf,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,aAAa,EAAE,sBAAsB,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACvF,OAAO;AACL,qBAAqB;AACrB,oBAAoB,EACpB,0BAA0B,EAC1B,mBAAmB,EACnB,4BAA4B,EAC5B,0BAA0B,EAC1B,2BAA2B,EAC3B,2BAA2B,EAC3B,kCAAkC,EAClC,kCAAkC,EAClC,kCAAkC,EAClC,8BAA8B,EAC9B,mCAAmC,EACnC,mCAAmC,EACnC,kCAAkC,EAClC,kCAAkC;AAElC,8BAA8B;AAC9B,sBAAsB,EACtB,iBAAiB;AAEjB,gBAAgB;AAChB,oBAAoB,EACpB,qBAAqB,EACrB,uBAAuB;AAEvB,0BAA0B;AAC1B,gCAAgC,EAChC,mCAAmC,EACnC,gCAAgC,EAChC,mCAAmC,EACnC,oCAAoC,EACpC,8BAA8B;AAE9B,YAAY;AACZ,sCAAsC,EACtC,oBAAoB,EACpB,sBAAsB,GACvB,MAAM,gBAAgB,CAAC;AAGxB;;GAEG;AACH,SAAS,QAAQ,CAAC,KAAyB,EAAE,SAAiB;IAC5D,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,OAAO,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;AAC9E,CAAC;AAED;;GAEG;AACH,SAAS,oBAAoB,CAAC,MAAmB;IAC/C,MAAM,YAAY,GAAG,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,OAAO,EAAE,CAAC,IAAI,CACxD,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,CACrD,CAAC;IACF,OAAO,YAAY,EAAE,OAAO,CAAC;AAC/B,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CACnC,SAAoB,EACpB,GAAiB;IAEjB,IAAI,CAAC,sBAAsB,EAAE;QAAE,OAAO,IAAI,CAAC;IAE3C,MAAM,MAAM,GAAG,aAAa,EAAE,CAAC;IAC/B,MAAM,IAAI,GAAG,MAAM,CAAC,SAAS,CAAC,kBAAkB,SAAS,CAAC,IAAI,EAAE,EAAE;QAChE,IAAI,EAAE,QAAQ,CAAC,QAAQ;QACvB,UAAU,EAAE;YACV,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,IAAI;YACtC,CAAC,sBAAsB,CAAC,EAAE,GAAG,CAAC,EAAE;YAChC,CAAC,0BAA0B,CAAC,EAAE,sCAAsC;SACrE;KACF,CAAC,CAAC;IAEH,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAC7B,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAE7C,OAAO,CAAC,GAAG,CAAC,gDAAgD,SAAS,CAAC,IAAI,YAAY,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;IACjG,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;AACxC,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,iBAAiB,CAC/B,SAAkB,EAClB,QAAkB,EAClB,SAAoB,EACpB,GAAiB,EACjB,UAAmB,EACnB,KAAc;IAEd,IAAI,CAAC,sBAAsB,EAAE;QAAE,OAAO,IAAI,CAAC;IAE3C,MAAM,MAAM,GAAG,aAAa,EAAE,CAAC;IAC/B,MAAM,UAAU,GAA2B;QACzC,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,IAAI;QACtC,CAAC,sBAAsB,CAAC,EAAE,GAAG,CAAC,EAAE;QAChC,CAAC,iBAAiB,CAAC,EAAE,QAAQ,CAAC,EAAE;QAChC,CAAC,mBAAmB,CAAC,EAAE,QAAQ,CAAC,IAAI;QACpC,CAAC,0BAA0B,CAAC,EAAE,sCAAsC;KACrE,CAAC;IACF,IAAI,UAAU,EAAE,CAAC;QACf,UAAU,CAAC,8BAA8B,CAAC,GAAG,UAAU,CAAC;QACxD,0EAA0E;QAC1E,0EAA0E;QAC1E,qDAAqD;QACrD,UAAU,CAAC,2BAA2B,CAAC,GAAG,UAAU,CAAC;IACvD,CAAC;IACD,UAAU,CAAC,oBAAoB,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,aAAa,IAAI,EAAE,EAAE,oBAAoB,CAAE,CAAC;IACjG,MAAM,QAAQ,GAAG,QAAQ,CACvB,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,SAAS,EACjF,oBAAoB,CACrB,CAAC;IACF,IAAI,QAAQ;QAAE,UAAU,CAAC,uBAAuB,CAAC,GAAG,QAAQ,CAAC;IAE7D,MAAM,IAAI,GAAG,MAAM,CAAC,SAAS,CAC3B,WAAW,EACX;QACE,IAAI,EAAE,QAAQ,CAAC,QAAQ;QACvB,UAAU;QACV,KAAK;KACN,EACD,SAAS,CACV,CAAC;IAEF,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAC3C,OAAO,CAAC,GAAG,CAAC,2CAA2C,QAAQ,CAAC,IAAI,iBAAiB,UAAU,IAAI,MAAM,GAAG,CAAC,CAAC;IAC9G,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;AAChC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CAAC,IAAU,EAAE,MAAmB;IACvE,IAAI,CAAC,sBAAsB,EAAE;QAAE,OAAO;IAEtC,2BAA2B;IAC3B,IAAI,MAAM,CAAC,OAAO,EAAE,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3C,IAAI,CAAC,QAAQ,CAAC,8BAA8B,EAAE;YAC5C,CAAC,2BAA2B,CAAC,EAAE,UAAU;YACzC,CAAC,kCAAkC,CAAC,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ;YAC7D,CAAC,kCAAkC,CAAC,EAAE,MAAM,CAAC,cAAc,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAC1F,CAAC,kCAAkC,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,iBAAiB,EAAE,sBAAsB,CAAC;SACjG,CAAC,CAAC;IACL,CAAC;IAED,+EAA+E;IAC/E,IAAI,MAAM,CAAC,OAAO,EAAE,YAAY,KAAK,SAAS,EAAE,CAAC;QAC/C,IAAI,CAAC,QAAQ,CAAC,8BAA8B,EAAE;YAC5C,CAAC,2BAA2B,CAAC,EAAE,cAAc;YAC7C,CAAC,kCAAkC,CAAC,EAAE,MAAM,CAAC,OAAO,CAAC,YAAY;SAClE,CAAC,CAAC;IACL,CAAC;IAED,IAAI,MAAM,CAAC,OAAO,EAAE,aAAa,KAAK,SAAS,EAAE,CAAC;QAChD,IAAI,CAAC,QAAQ,CAAC,8BAA8B,EAAE;YAC5C,CAAC,2BAA2B,CAAC,EAAE,eAAe;YAC9C,CAAC,kCAAkC,CAAC,EAAE,MAAM,CAAC,OAAO,CAAC,aAAa;SACnE,CAAC,CAAC;IACL,CAAC;IAED,IAAI,MAAM,CAAC,OAAO,EAAE,0BAA0B,KAAK,SAAS,EAAE,CAAC;QAC7D,IAAI,CAAC,QAAQ,CAAC,8BAA8B,EAAE;YAC5C,CAAC,2BAA2B,CAAC,EAAE,4BAA4B;YAC3D,CAAC,kCAAkC,CAAC,EAAE,MAAM,CAAC,OAAO,CAAC,0BAA0B;SAChF,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAU,EAAE,MAAmB,EAAE,OAAc;IAClF,IAAI,CAAC,sBAAsB,EAAE;QAAE,OAAO;IAEtC,oBAAoB;IACpB,MAAM,MAAM,GAAG,MAAM,CAAC,cAAc,KAAK,QAAQ;QAC/C,CAAC,CAAC,kCAAkC;QACpC,CAAC,CAAC,kCAAkC,CAAC;IACvC,IAAI,CAAC,YAAY,CAAC,4BAA4B,EAAE,MAAM,CAAC,CAAC;IAExD,0FAA0F;IAC1F,MAAM,MAAM,GAAG,oBAAoB,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IAClD,IAAI,CAAC,YAAY,CAAC,qBAAqB,EAAE,QAAQ,CAAC,MAAM,EAAE,oBAAoB,CAAE,CAAC,CAAC;IAElF,wCAAwC;IACxC,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;QAC7B,IAAI,CAAC,YAAY,CAAC,oCAAoC,EAAE,MAAM,CAAC,iBAAiB,CAAC,CAAC;IACpF,CAAC;IACD,IAAI,MAAM,CAAC,kBAAkB,EAAE,eAAe,KAAK,SAAS,EAAE,CAAC;QAC7D,IAAI,CAAC,YAAY,CAAC,mCAAmC,EAAE,MAAM,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;IACpG,CAAC;IACD,IAAI,MAAM,CAAC,kBAAkB,EAAE,eAAe,KAAK,SAAS,EAAE,CAAC;QAC7D,IAAI,CAAC,YAAY,CAAC,mCAAmC,EAAE,MAAM,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;IACpG,CAAC;IACD,IAAI,MAAM,CAAC,kBAAkB,EAAE,aAAa,KAAK,SAAS,EAAE,CAAC;QAC3D,IAAI,CAAC,YAAY,CAAC,gCAAgC,EAAE,MAAM,CAAC,kBAAkB,CAAC,aAAa,CAAC,CAAC;IAC/F,CAAC;IACD,IAAI,MAAM,CAAC,gBAAgB,EAAE,OAAO,EAAE,CAAC;QACrC,IAAI,CAAC,YAAY,CAAC,gCAAgC,EAAE,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;IACvF,CAAC;IAED,8CAA8C;IAC9C,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,cAAc,CAAC,KAAK,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC,CAAC;IAC/E,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAClB,OAAO,CAAC,GAAG,CAAC,gDAAgD,MAAM,oBAAoB,MAAM,CAAC,cAAc,GAAG,CAAC,CAAC;AAClH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,wBAAwB,CAAC,IAAU,EAAE,GAAiB;IACpE,IAAI,CAAC,sBAAsB,EAAE;QAAE,OAAO;IAEtC,sCAAsC;IACtC,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3C,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,WAAW;QACxB,CAAC,CAAC,mCAAmC;QACrC,CAAC,CAAC,mCAAmC,CAAC;IAExC,IAAI,CAAC,YAAY,CAAC,0BAA0B,EAAE,MAAM,CAAC,CAAC;IAEtD,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;QACpE,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,cAAc,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,SAAS,iBAAiB,EAAE,CAAC,CAAC;IACzF,CAAC;IAED,IAAI,CAAC,GAAG,EAAE,CAAC;IACX,OAAO,CAAC,GAAG,CAAC,qDAAqD,MAAM,gBAAgB,CAAC,CAAC;IACzF,eAAe,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;AACpC,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,wBAAwB,CACtC,QAAkB,EAClB,MAAmB,EACnB,SAA2B,EAC3B,KAAa,EACb,UAAmB,EACnB,SAAgB,EAChB,OAAc,EACd,YAAqB;IAErB,IAAI,CAAC,sBAAsB,EAAE;QAAE,OAAO;IAEtC,MAAM,MAAM,GAAG,aAAa,EAAE,CAAC;IAC/B,MAAM,UAAU,GAA2B;QACzC,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,IAAI;QACtC,CAAC,sBAAsB,CAAC,EAAE,KAAK;QAC/B,CAAC,iBAAiB,CAAC,EAAE,QAAQ,CAAC,EAAE;QAChC,CAAC,mBAAmB,CAAC,EAAE,QAAQ,CAAC,IAAI;QACpC,CAAC,0BAA0B,CAAC,EAAE,sCAAsC;KACrE,CAAC;IACF,IAAI,UAAU,EAAE,CAAC;QACf,UAAU,CAAC,8BAA8B,CAAC,GAAG,UAAU,CAAC;QACxD,UAAU,CAAC,2BAA2B,CAAC,GAAG,UAAU,CAAC;IACvD,CAAC;IACD,UAAU,CAAC,oBAAoB,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,aAAa,IAAI,EAAE,EAAE,oBAAoB,CAAE,CAAC;IACjG,MAAM,QAAQ,GAAG,QAAQ,CACvB,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,SAAS,EACjF,oBAAoB,CACrB,CAAC;IACF,IAAI,QAAQ;QAAE,UAAU,CAAC,uBAAuB,CAAC,GAAG,QAAQ,CAAC;IAE7D,yEAAyE;IACzE,yEAAyE;IACzE,IAAI,SAA8B,CAAC;IACnC,IAAI,YAAY,IAAI,iBAAiB,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;QACzD,MAAM,iBAAiB,GAAgB;YACrC,OAAO,EAAE,YAAY;YACrB,0EAA0E;YAC1E,sEAAsE;YACtE,qEAAqE;YACrE,4CAA4C;YAC5C,MAAM,EAAE,kBAAkB;YAC1B,UAAU,EAAE,UAAU,CAAC,OAAO;YAC9B,QAAQ,EAAE,IAAI;SACf,CAAC;QACF,SAAS,GAAG,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,iBAAiB,CAAC,CAAC;IACxE,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,CAAC,SAAS,CAAC,WAAW,EAAE;QACzC,IAAI,EAAE,QAAQ,CAAC,QAAQ;QACvB,SAAS;QACT,UAAU;KACX,EAAE,SAAS,CAAC,CAAC;IAEd,yBAAyB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACxC,oBAAoB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAC5C,OAAO,CAAC,GAAG,CAAC,oDAAoD,QAAQ,CAAC,IAAI,iBAAiB,UAAU,IAAI,MAAM,aAAa,YAAY,IAAI,KAAK,gBAAgB,CAAC,CAAC;IACtK,eAAe,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;AACpC,CAAC"}

package/lib/dist/lib/telemetry/index.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Evaluation Telemetry Module
+ *
+ * Emits OTel spans for benchmark evaluation data using semantic conventions.
+ * Disabled by default — enable via OTEL_EVAL_ENABLED=true or config file.
+ */
+export { initEvalTracerProvider, getEvalTracer, isEvalTelemetryEnabled, flushEvalTracer, shutdownEvalTracer, resolveEvalTelemetryConfig, type EvalTelemetryConfig, } from './provider.js';
+export { startTestSuiteRunSpan, startTestCaseSpan, addEvaluationResultEvents, finalizeTestCaseSpan, finalizeTestSuiteRunSpan, emitDeferredTestCaseSpan, } from './evalSpans.js';
+export * from './constants.js';
+export { OpenSearchSpanExporter, type OpenSearchExporterConfig, } from './opensearchExporter.js';
+//# sourceMappingURL=index.d.ts.map

package/lib/dist/lib/telemetry/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../telemetry/index.ts"],"names":[],"mappings":"AAKA;;;;;GAKG;AAEH,OAAO,EACL,sBAAsB,EACtB,aAAa,EACb,sBAAsB,EACtB,eAAe,EACf,kBAAkB,EAClB,0BAA0B,EAC1B,KAAK,mBAAmB,GACzB,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,qBAAqB,EACrB,iBAAiB,EACjB,yBAAyB,EACzB,oBAAoB,EACpB,wBAAwB,EACxB,wBAAwB,GACzB,MAAM,gBAAgB,CAAC;AAExB,cAAc,gBAAgB,CAAC;AAE/B,OAAO,EACL,sBAAsB,EACtB,KAAK,wBAAwB,GAC9B,MAAM,yBAAyB,CAAC"}

package/lib/dist/lib/telemetry/index.js

@@ -0,0 +1,15 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * Evaluation Telemetry Module
+ *
+ * Emits OTel spans for benchmark evaluation data using semantic conventions.
+ * Disabled by default — enable via OTEL_EVAL_ENABLED=true or config file.
+ */
+export { initEvalTracerProvider, getEvalTracer, isEvalTelemetryEnabled, flushEvalTracer, shutdownEvalTracer, resolveEvalTelemetryConfig, } from './provider.js';
+export { startTestSuiteRunSpan, startTestCaseSpan, addEvaluationResultEvents, finalizeTestCaseSpan, finalizeTestSuiteRunSpan, emitDeferredTestCaseSpan, } from './evalSpans.js';
+export * from './constants.js';
+export { OpenSearchSpanExporter, } from './opensearchExporter.js';
+//# sourceMappingURL=index.js.map

package/lib/dist/lib/telemetry/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../telemetry/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;GAKG;AAEH,OAAO,EACL,sBAAsB,EACtB,aAAa,EACb,sBAAsB,EACtB,eAAe,EACf,kBAAkB,EAClB,0BAA0B,GAE3B,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,qBAAqB,EACrB,iBAAiB,EACjB,yBAAyB,EACzB,oBAAoB,EACpB,wBAAwB,EACxB,wBAAwB,GACzB,MAAM,gBAAgB,CAAC;AAExB,cAAc,gBAAgB,CAAC;AAE/B,OAAO,EACL,sBAAsB,GAEvB,MAAM,yBAAyB,CAAC"}

package/lib/dist/lib/telemetry/opensearchExporter.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Custom OpenSearch Span Exporter
+ *
+ * Writes OTel spans directly to an OpenSearch cluster using the same
+ * observability data source config that the trace reader uses.
+ * Formats spans in the OTEL-faithful Data Prepper `trace-analytics-plain-raw`
+ * schema (a nested `attributes` object keyed by literal dotted OTel names)
+ * so they co-exist with agent spans already in the otel-v1-apm-span-* index.
+ */
+import { SpanExporter, ReadableSpan } from '@opentelemetry/sdk-trace-node';
+import { ExportResult } from '@opentelemetry/core';
+/**
+ * Configuration for the OpenSearch exporter
+ */
+export interface OpenSearchExporterConfig {
+    /** OpenSearch endpoint URL */
+    endpoint: string;
+    /** Basic auth username */
+    username?: string;
+    /** Basic auth password */
+    password?: string;
+    /** Index pattern to write to (default: otel-v1-apm-span-000001) */
+    indexName?: string;
+    /** Whether to skip TLS verification */
+    tlsSkipVerify?: boolean;
+}
+/**
+ * Custom SpanExporter that writes directly to OpenSearch.
+ *
+ * Formats spans in OSIS-compatible format so eval spans co-exist
+ * with agent spans in the same otel-v1-apm-span-* indices.
+ */
+export declare class OpenSearchSpanExporter implements SpanExporter {
+    private client;
+    private indexName;
+    private shutdownRequested;
+    constructor(config: OpenSearchExporterConfig);
+    export(spans: ReadableSpan[], resultCallback: (result: ExportResult) => void): void;
+    private _export;
+    shutdown(): Promise<void>;
+    forceFlush(): Promise<void>;
+}
+//# sourceMappingURL=opensearchExporter.d.ts.map

package/lib/dist/lib/telemetry/opensearchExporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"opensearchExporter.d.ts","sourceRoot":"","sources":["../../../telemetry/opensearchExporter.ts"],"names":[],"mappings":"AAKA;;;;;;;;GAQG;AAEH,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAC3E,OAAO,EAAE,YAAY,EAAoB,MAAM,qBAAqB,CAAC;AAIrE;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,8BAA8B;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,0BAA0B;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,0BAA0B;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,mEAAmE;IACnE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AA6JD;;;;;GAKG;AACH,qBAAa,sBAAuB,YAAW,YAAY;IACzD,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,iBAAiB,CAAS;gBAEtB,MAAM,EAAE,wBAAwB;IAiB5C,MAAM,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE,cAAc,EAAE,CAAC,MAAM,EAAE,YAAY,KAAK,IAAI,GAAG,IAAI;YAcrE,OAAO;IA0Bf,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAKzB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;CAGlC"}

package/lib/dist/lib/telemetry/opensearchExporter.js

@@ -0,0 +1,217 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { ExportResultCode } from '@opentelemetry/core';
+import { SpanKind, SpanStatusCode } from '@opentelemetry/api';
+import { Client } from '@opensearch-project/opensearch';
+/**
+ * Convert OTel HrTime [seconds, nanoseconds] to ISO date string
+ */
+function hrTimeToDate(hrTime) {
+    const ms = hrTime[0] * 1000 + hrTime[1] / 1_000_000;
+    return new Date(ms).toISOString();
+}
+/**
+ * Convert OTel HrTime to nanoseconds
+ */
+function hrTimeToNanos(hrTime) {
+    return hrTime[0] * 1_000_000_000 + hrTime[1];
+}
+/**
+ * Convert duration between two HrTimes to nanoseconds
+ */
+function durationNanos(start, end) {
+    return hrTimeToNanos(end) - hrTimeToNanos(start);
+}
+/**
+ * Map OTel SpanKind to OSIS string format
+ */
+function spanKindToString(kind) {
+    switch (kind) {
+        case SpanKind.INTERNAL: return 'SPAN_KIND_INTERNAL';
+        case SpanKind.SERVER: return 'SPAN_KIND_SERVER';
+        case SpanKind.CLIENT: return 'SPAN_KIND_CLIENT';
+        case SpanKind.PRODUCER: return 'SPAN_KIND_PRODUCER';
+        case SpanKind.CONSUMER: return 'SPAN_KIND_CONSUMER';
+        default: return 'SPAN_KIND_INTERNAL';
+    }
+}
+/**
+ * Map OTel SpanStatusCode to OSIS integer
+ */
+function statusCodeToInt(code) {
+    switch (code) {
+        case SpanStatusCode.UNSET: return 0;
+        case SpanStatusCode.OK: return 1;
+        case SpanStatusCode.ERROR: return 2;
+        default: return 0;
+    }
+}
+/**
+ * Convert a ReadableSpan to an OSIS-compatible OpenSearch document
+ */
+function spanToDocument(span) {
+    const doc = {
+        traceId: span.spanContext().traceId,
+        spanId: span.spanContext().spanId,
+        parentSpanId: (span.parentSpanContext?.spanId && span.parentSpanContext.spanId !== '0000000000000000')
+            ? span.parentSpanContext.spanId : '',
+        name: span.name,
+        kind: spanKindToString(span.kind),
+        startTime: hrTimeToDate(span.startTime),
+        endTime: hrTimeToDate(span.endTime),
+        durationInNanos: durationNanos(span.startTime, span.endTime),
+        status: {
+            code: statusCodeToInt(span.status.code),
+            message: span.status.message || '',
+        },
+        traceState: '',
+        droppedAttributesCount: span.droppedAttributesCount,
+        droppedEventsCount: span.droppedEventsCount,
+        droppedLinksCount: span.droppedLinksCount,
+    };
+    // Span attributes: nested object keyed by literal dotted OTel names
+    // (Data Prepper trace-analytics-plain-raw / OTEL-faithful schema).
+    const spanAttributes = {};
+    for (const [key, value] of Object.entries(span.attributes)) {
+        if (value !== undefined) {
+            spanAttributes[key] = value;
+        }
+    }
+    doc.attributes = spanAttributes;
+    // Resource attributes: nested resource.attributes object, literal dotted keys.
+    const resource = span.resource;
+    const resourceAttributes = {};
+    if (resource?.attributes) {
+        for (const [key, value] of Object.entries(resource.attributes)) {
+            if (value !== undefined) {
+                resourceAttributes[key] = value;
+            }
+        }
+    }
+    doc.resource = { attributes: resourceAttributes };
+    // Set serviceName from resource
+    const serviceName = resource?.attributes?.['service.name'] || 'unknown';
+    doc.serviceName = serviceName;
+    // Instrumentation scope — nested object (runtime property is instrumentationScope).
+    const scope = span.instrumentationScope ?? span.instrumentationLibrary ?? {};
+    doc.instrumentationScope = {
+        name: scope.name || '',
+        ...(scope.version ? { version: scope.version } : {}),
+    };
+    // Events (nested array; attribute keys are literal dotted OTel names)
+    if (span.events.length > 0) {
+        doc.events = span.events.map(event => {
+            const eventDoc = {
+                name: event.name,
+                time: hrTimeToDate(event.time),
+                droppedAttributesCount: event.droppedAttributesCount || 0,
+            };
+            if (event.attributes) {
+                const attrs = {};
+                for (const [key, value] of Object.entries(event.attributes)) {
+                    if (value !== undefined) {
+                        attrs[key] = value;
+                    }
+                }
+                eventDoc.attributes = attrs;
+            }
+            return eventDoc;
+        });
+    }
+    else {
+        doc.events = [];
+    }
+    // Links
+    if (span.links.length > 0) {
+        doc.links = span.links.map(link => ({
+            traceId: link.context.traceId,
+            spanId: link.context.spanId,
+            traceState: link.context.traceState?.serialize() || '',
+            attributes: link.attributes || {},
+            droppedAttributesCount: link.droppedAttributesCount || 0,
+        }));
+    }
+    else {
+        doc.links = [];
+    }
+    // traceGroupFields (populated for root spans)
+    const hasRealParent = span.parentSpanContext?.spanId && span.parentSpanContext.spanId !== '0000000000000000';
+    if (!hasRealParent) {
+        doc.traceGroup = span.name;
+        doc.traceGroupFields = {
+            endTime: hrTimeToDate(span.endTime),
+            durationInNanos: durationNanos(span.startTime, span.endTime),
+            statusCode: statusCodeToInt(span.status.code),
+        };
+    }
+    return doc;
+}
+/**
+ * Custom SpanExporter that writes directly to OpenSearch.
+ *
+ * Formats spans in OSIS-compatible format so eval spans co-exist
+ * with agent spans in the same otel-v1-apm-span-* indices.
+ */
+export class OpenSearchSpanExporter {
+    constructor(config) {
+        this.shutdownRequested = false;
+        const auth = config.username && config.password
+            ? `${config.username}:${config.password}`
+            : undefined;
+        this.client = new Client({
+            node: config.endpoint,
+            auth: auth ? { username: config.username, password: config.password } : undefined,
+            ssl: {
+                rejectUnauthorized: config.tlsSkipVerify !== true,
+            },
+        });
+        this.indexName = config.indexName || 'otel-v1-apm-span-000001';
+        console.log(`[Telemetry] OpenSearch exporter initialized → ${config.endpoint} (index: ${this.indexName})`);
+    }
+    export(spans, resultCallback) {
+        if (this.shutdownRequested) {
+            resultCallback({ code: ExportResultCode.FAILED });
+            return;
+        }
+        this._export(spans)
+            .then(() => resultCallback({ code: ExportResultCode.SUCCESS }))
+            .catch((err) => {
+            console.error('[Telemetry] OpenSearch export failed:', err instanceof Error ? err.message : err);
+            resultCallback({ code: ExportResultCode.FAILED });
+        });
+    }
+    async _export(spans) {
+        if (spans.length === 0)
+            return;
+        // Use bulk API for efficiency
+        const body = [];
+        for (const span of spans) {
+            const doc = spanToDocument(span);
+            const docId = `${doc.traceId}/${doc.spanId}`;
+            body.push({ index: { _index: this.indexName, _id: docId } });
+            body.push(doc);
+        }
+        const response = await this.client.bulk({ body });
+        if (response.body.errors) {
+            const errors = response.body.items
+                .filter((item) => item.index?.error)
+                .map((item) => `${item.index.error.type}: ${item.index.error.reason}`);
+            if (errors.length > 0) {
+                console.warn(`[Telemetry] OpenSearch bulk write had ${errors.length} error(s):`, errors[0]);
+            }
+        }
+        else {
+            console.log(`[Telemetry] Exported ${spans.length} span(s) to OpenSearch`);
+        }
+    }
+    async shutdown() {
+        this.shutdownRequested = true;
+        await this.client.close();
+    }
+    async forceFlush() {
+        // No buffering — exports are immediate via bulk API
+    }
+}
+//# sourceMappingURL=opensearchExporter.js.map

package/lib/dist/lib/telemetry/opensearchExporter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"opensearchExporter.js","sourceRoot":"","sources":["../../../telemetry/opensearchExporter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAaH,OAAO,EAAgB,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAU,MAAM,oBAAoB,CAAC;AACtE,OAAO,EAAE,MAAM,EAAE,MAAM,gCAAgC,CAAC;AAkBxD;;GAEG;AACH,SAAS,YAAY,CAAC,MAAc;IAClC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC;IACpD,OAAO,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;AACpC,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,MAAc;IACnC,OAAO,MAAM,CAAC,CAAC,CAAC,GAAG,aAAa,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;AAC/C,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,KAAa,EAAE,GAAW;IAC/C,OAAO,aAAa,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;AACnD,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,IAAc;IACtC,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,oBAAoB,CAAC;QACpD,KAAK,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,kBAAkB,CAAC;QAChD,KAAK,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,kBAAkB,CAAC;QAChD,KAAK,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,oBAAoB,CAAC;QACpD,KAAK,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,oBAAoB,CAAC;QACpD,OAAO,CAAC,CAAC,OAAO,oBAAoB,CAAC;IACvC,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,IAAoB;IAC3C,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,cAAc,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;QACpC,KAAK,cAAc,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;QACjC,KAAK,cAAc,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;QACpC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC;IACpB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,IAAkB;IACxC,MAAM,GAAG,GAA4B;QACnC,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO;QACnC,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,MAAM;QACjC,YAAY,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,MAAM,IAAI,IAAI,CAAC,iBAAiB,CAAC,MAAM,KAAK,kBAAkB,CAAC;YACpG,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;QACtC,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,IAAI,EAAE,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC;QACjC,SAAS,EAAE,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC;QACvC,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC;QACnC,eAAe,EAAE,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC;QAC5D,MAAM,EAAE;YACN,IAAI,EAAE,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;YACvC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE;SACnC;QACD,UAAU,EAAE,EAAE;QACd,sBAAsB,EAAE,IAAI,CAAC,sBAAsB;QACnD,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;QAC3C,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;KAC1C,CAAC;IAEF,oEAAoE;IACpE,mEAAmE;IACnE,MAAM,cAAc,GAA4B,EAAE,CAAC;IACnD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3D,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,cAAc,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QAC9B,CAAC;IACH,CAAC;IACD,GAAG,CAAC,UAAU,GAAG,cAAc,CAAC;IAEhC,+EAA+E;IAC/E,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;IAC/B,MAAM,kBAAkB,GAA4B,EAAE,CAAC;IACvD,IAAI,QAAQ,EAAE,UAAU,EAAE,CAAC;QACzB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;YAC/D,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACxB,kBAAkB,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;YAClC,CAAC;QACH,CAAC;IACH,CAAC;IACD,GAAG,CAAC,QAAQ,GAAG,EAAE,UAAU,EAAE,kBAAkB,EAAE,CAAC;IAElD,gCAAgC;IAChC,MAAM,WAAW,GAAG,QAAQ,EAAE,UAAU,EAAE,CAAC,cAAc,CAAC,IAAI,SAAS,CAAC;IACxE,GAAG,CAAC,WAAW,GAAG,WAAW,CAAC;IAE9B,oFAAoF;IACpF,MAAM,KAAK,GAAI,IAAY,CAAC,oBAAoB,IAAK,IAAY,CAAC,sBAAsB,IAAI,EAAE,CAAC;IAC/F,GAAG,CAAC,oBAAoB,GAAG;QACzB,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,EAAE;QACtB,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACrD,CAAC;IAEF,sEAAsE;IACtE,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;YACnC,MAAM,QAAQ,GAA4B;gBACxC,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,IAAI,EAAE,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC;gBAC9B,sBAAsB,EAAE,KAAK,CAAC,sBAAsB,IAAI,CAAC;aAC1D,CAAC;YACF,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;gBACrB,MAAM,KAAK,GAA4B,EAAE,CAAC;gBAC1C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC5D,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;wBACxB,KAAK,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;oBACrB,CAAC;gBACH,CAAC;gBACD,QAAQ,CAAC,UAAU,GAAG,KAAK,CAAC;YAC9B,CAAC;YACD,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC;IAClB,CAAC;IAED,QAAQ;IACR,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAClC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;YAC7B,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;YAC3B,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,SAAS,EAAE,IAAI,EAAE;YACtD,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,EAAE;YACjC,sBAAsB,EAAE,IAAI,CAAC,sBAAsB,IAAI,CAAC;SACzD,CAAC,CAAC,CAAC;IACN,CAAC;SAAM,CAAC;QACN,GAAG,CAAC,KAAK,GAAG,EAAE,CAAC;IACjB,CAAC;IAED,8CAA8C;IAC9C,MAAM,aAAa,GAAG,IAAI,CAAC,iBAAiB,EAAE,MAAM,IAAI,IAAI,CAAC,iBAAiB,CAAC,MAAM,KAAK,kBAAkB,CAAC;IAC7G,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC;QAC3B,GAAG,CAAC,gBAAgB,GAAG;YACrB,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC;YACnC,eAAe,EAAE,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC;YAC5D,UAAU,EAAE,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;SAC9C,CAAC;IACJ,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;GAKG;AACH,MAAM,OAAO,sBAAsB;IAKjC,YAAY,MAAgC;QAFpC,sBAAiB,GAAG,KAAK,CAAC;QAGhC,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ;YAC7C,CAAC,CAAC,GAAG,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,EAAE;YACzC,CAAC,CAAC,SAAS,CAAC;QAEd,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC;YACvB,IAAI,EAAE,MAAM,CAAC,QAAQ;YACrB,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAS,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAS,EAAE,CAAC,CAAC,CAAC,SAAS;YACnF,GAAG,EAAE;gBACH,kBAAkB,EAAE,MAAM,CAAC,aAAa,KAAK,IAAI;aAClD;SACF,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,yBAAyB,CAAC;QAC/D,OAAO,CAAC,GAAG,CAAC,iDAAiD,MAAM,CAAC,QAAQ,YAAY,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;IAC7G,CAAC;IAED,MAAM,CAAC,KAAqB,EAAE,cAA8C;QAC1E,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3B,cAAc,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,MAAM,EAAE,CAAC,CAAC;YAClD,OAAO;QACT,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC;aAChB,IAAI,CAAC,GAAG,EAAE,CAAC,cAAc,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,OAAO,EAAE,CAAC,CAAC;aAC9D,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACb,OAAO,CAAC,KAAK,CAAC,uCAAuC,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YACjG,cAAc,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,MAAM,EAAE,CAAC,CAAC;QACpD,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,KAAqB;QACzC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAE/B,8BAA8B;QAC9B,MAAM,IAAI,GAAmC,EAAE,CAAC;QAChD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,GAAG,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;YACjC,MAAM,KAAK,GAAG,GAAG,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;YAC7C,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;YAC7D,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjB,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;QAElD,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK;iBAC/B,MAAM,CAAC,CAAC,IAAS,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC;iBACxC,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YAC9E,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,OAAO,CAAC,IAAI,CAAC,yCAAyC,MAAM,CAAC,MAAM,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9F,CAAC;QACH,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,wBAAwB,KAAK,CAAC,MAAM,wBAAwB,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAC9B,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,UAAU;QACd,oDAAoD;IACtD,CAAC;CACF"}

package/lib/dist/lib/telemetry/provider.d.ts

@@ -0,0 +1,55 @@
+/**
+ * OTel TracerProvider management for evaluation telemetry.
+ *
+ * Manages a singleton TracerProvider that exports evaluation spans
+ * via OTLP/HTTP to a collector, or directly to OpenSearch when an
+ * observability data source is configured.
+ */
+import { type Tracer } from '@opentelemetry/api';
+import { type OpenSearchExporterConfig } from './opensearchExporter.js';
+/**
+ * Configuration for evaluation telemetry
+ */
+export interface EvalTelemetryConfig {
+    /** Whether evaluation telemetry is enabled */
+    enabled: boolean;
+    /** OTLP exporter endpoint (e.g., http://localhost:4318/v1/traces) */
+    exporterEndpoint: string;
+    /** Optional headers for the OTLP exporter (e.g., auth) */
+    exporterHeaders?: Record<string, string>;
+    /** Service name for resource attributes */
+    serviceName?: string;
+    /** OpenSearch direct export config (derived from observability data source) */
+    opensearch?: OpenSearchExporterConfig;
+}
+/**
+ * Resolve telemetry config from environment variables and optional user config
+ */
+export declare function resolveEvalTelemetryConfig(userConfig?: Partial<EvalTelemetryConfig>): EvalTelemetryConfig;
+/**
+ * Initialize the evaluation TracerProvider.
+ * Safe to call multiple times — subsequent calls are no-ops.
+ *
+ * When an OpenSearch config is provided (from observability data source),
+ * spans are written directly to OpenSearch in OSIS-compatible format.
+ * Otherwise, falls back to OTLP/HTTP exporter.
+ */
+export declare function initEvalTracerProvider(config: EvalTelemetryConfig): void;
+/**
+ * Get the evaluation tracer. Returns a no-op tracer if telemetry is disabled.
+ */
+export declare function getEvalTracer(): Tracer;
+/**
+ * Check if evaluation telemetry is enabled
+ */
+export declare function isEvalTelemetryEnabled(): boolean;
+/**
+ * Force-flush any buffered spans to the exporter.
+ * Call after emitting spans to ensure they are exported promptly.
+ */
+export declare function flushEvalTracer(): Promise<void>;
+/**
+ * Flush pending spans and shut down the TracerProvider.
+ */
+export declare function shutdownEvalTracer(): Promise<void>;
+//# sourceMappingURL=provider.d.ts.map

package/lib/dist/lib/telemetry/provider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../../../telemetry/provider.ts"],"names":[],"mappings":"AAKA;;;;;;GAMG;AAEH,OAAO,EAAS,KAAK,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAKxD,OAAO,EAA0B,KAAK,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AAEhG;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,8CAA8C;IAC9C,OAAO,EAAE,OAAO,CAAC;IACjB,qEAAqE;IACrE,gBAAgB,EAAE,MAAM,CAAC;IACzB,0DAA0D;IAC1D,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACzC,2CAA2C;IAC3C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+EAA+E;IAC/E,UAAU,CAAC,EAAE,wBAAwB,CAAC;CACvC;AAKD;;GAEG;AACH,wBAAgB,0BAA0B,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,mBAAmB,CAwBzG;AAED;;;;;;;GAOG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI,CAqDxE;AAED;;GAEG;AACH,wBAAgB,aAAa,IAAI,MAAM,CAEtC;AAED;;GAEG;AACH,wBAAgB,sBAAsB,IAAI,OAAO,CAEhD;AAED;;;GAGG;AACH,wBAAsB,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC,CAQrD;AAED;;GAEG;AACH,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC,CAOxD"}