@arnilo/prism-observability-opentelemetry 0.0.4 → 0.0.6

package/CHANGELOG.md

@@ -1,6 +1,12 @@
 # Changelog
 
-## [Unreleased]
+## [0.0.6] - 2026-07-19
+
+- Released with the exact 0.0.6 first-party package graph.
+
+## [0.0.5] - 2026-07-16
+
+- Added safe run-feedback/evaluation span events or ended-run spans plus low-cardinality counters; comments, tag values, scorer/evaluation IDs, and arbitrary metadata are not accepted by telemetry handlers.
 
 ## [0.0.4] - 2026-07-14
 

package/README.md

@@ -36,7 +36,10 @@ Set `enabled: false` or omit both `tracer` and `meter` for a no-op adapter.
 
 - Metadata-only spans: no prompts, tool arguments, or credentials.
 - High-cardinality IDs (`sessionId`, `runId`, `requestId`, `toolCallId`) are span attributes, not metric labels.
-- Metric labels are limited to `provider_id`, `outcome`, `status`, and token `kind`.
+- Metric labels are limited to `provider_id`, `outcome`, `status`, token `kind`, and feedback rating bucket/link presence; comments, tags, and linked IDs never become labels.
+- `handleRunFeedback()` / `handleEvaluation()` add safe scalar metadata to the active run span or a short post-run span.
+- `prism.provider.tokens` records turns and `prism.run.tokens` records aggregates.
+- Run errors and detach close attributable outstanding spans exactly once.
 - Exporter failures are isolated: instrumentation catches errors and forwards them to `onExporterError` without affecting the run.
 
 ## Profile inclusion

package/dist/index.d.ts

@@ -1,3 +1,3 @@
 export { createInMemoryTelemetry, createOpenTelemetryInstrumentation, wrapOpenTelemetryApi, } from "./instrumentation.js";
-export type { InMemoryTelemetry, OpenTelemetryInstrumentation, OpenTelemetryInstrumentationOptions, PrismCounter, PrismHistogram, PrismMeter, PrismSpan, PrismSpanStatus, PrismTracer, RecordedMetric, RecordedSpan, } from "./instrumentation.js";
+export type { EvaluationTelemetry, InMemoryTelemetry, OpenTelemetryInstrumentation, OpenTelemetryInstrumentationOptions, PrismCounter, PrismHistogram, PrismMeter, PrismSpan, PrismSpanStatus, PrismTracer, RunFeedbackTelemetry, RecordedMetric, RecordedSpan, } from "./instrumentation.js";
 export declare const packageName = "@arnilo/prism-observability-opentelemetry";

package/dist/instrumentation.d.ts

@@ -3,6 +3,7 @@ export type PrismSpanStatus = "ok" | "error";
 export interface PrismSpan {
     setAttribute(key: string, value: string | number | boolean): void;
     setStatus(code: PrismSpanStatus, message?: string): void;
+    addEvent?(name: string, attributes?: Readonly<Record<string, string | number | boolean>>): void;
     end(): void;
 }
 export interface PrismTracer {
@@ -31,10 +32,26 @@ export interface OpenTelemetryInstrumentationOptions {
     readonly meter?: PrismMeter;
     readonly onExporterError?: (error: unknown) => void;
 }
+export interface RunFeedbackTelemetry {
+    readonly runId: string;
+    readonly rating?: number;
+    readonly hasComment: boolean;
+    readonly tagCount: number;
+    readonly scorerCount: number;
+    readonly evaluationCount: number;
+}
+export interface EvaluationTelemetry {
+    readonly runId: string;
+    readonly status: "scored" | "skipped" | "failed";
+    readonly score?: number;
+    readonly hasReason: boolean;
+}
 export interface OpenTelemetryInstrumentation {
     readonly enabled: boolean;
     handleAgentEvent(event: AgentEvent): void;
-    attachSession(session: Pick<AgentSession, "subscribe">): () => void;
+    handleRunFeedback(feedback: RunFeedbackTelemetry): void;
+    handleEvaluation(evaluation: EvaluationTelemetry): void;
+    attachSession(session: Pick<AgentSession, "id" | "subscribe">): () => void;
 }
 export interface RecordedSpan {
     readonly name: string;
@@ -44,6 +61,10 @@ export interface RecordedSpan {
         readonly message?: string;
     };
     readonly ended: boolean;
+    readonly events: readonly {
+        readonly name: string;
+        readonly attributes: Record<string, string | number | boolean>;
+    }[];
 }
 export interface RecordedMetric {
     readonly name: string;
@@ -66,6 +87,7 @@ interface OpenTelemetrySpan {
         code: number;
         message?: string;
     }): void;
+    addEvent?(name: string, attributes?: Record<string, string | number | boolean>): void;
     end(): void;
 }
 interface OpenTelemetryTracer {

package/dist/instrumentation.js

@@ -9,7 +9,7 @@ export function createInMemoryTelemetry() {
     const metrics = [];
     const tracer = {
         startSpan(name, options) {
-            const record = { name, attributes: { ...(options?.attributes ?? {}) }, ended: false };
+            const record = { name, attributes: { ...(options?.attributes ?? {}) }, ended: false, events: [] };
             spans.push(record);
             return {
                 setAttribute(key, value) {
@@ -18,6 +18,9 @@ export function createInMemoryTelemetry() {
                 setStatus(code, message) {
                     record.status = { code, message };
                 },
+                addEvent(name, attributes) {
+                    record.events.push({ name, attributes: { ...(attributes ?? {}) } });
+                },
                 end() {
                     record.ended = true;
                 },
@@ -68,7 +71,10 @@ export function createOpenTelemetryInstrumentation(options = {}) {
         description: "Tool execution latency",
         unit: "ms",
     });
-    const tokenCounter = meter?.createCounter("prism.provider.tokens", { description: "Provider token usage" });
+    const providerTokenCounter = meter?.createCounter("prism.provider.tokens", { description: "Provider-turn token usage" });
+    const runTokenCounter = meter?.createCounter("prism.run.tokens", { description: "Aggregate agent-run token usage" });
+    const feedbackCounter = meter?.createCounter("prism.run.feedback", { description: "Run feedback count" });
+    const evaluationCounter = meter?.createCounter("prism.run.evaluation", { description: "Run evaluation count" });
     const activeSpans = new Map();
     const safe = (fn) => {
         if (!enabled)
@@ -81,12 +87,22 @@ export function createOpenTelemetryInstrumentation(options = {}) {
         }
     };
     const endSpan = (key, status, message) => {
-        const span = activeSpans.get(key);
-        if (!span)
+        const active = activeSpans.get(key);
+        if (!active)
             return;
-        span.setStatus(status, message);
-        span.end();
         activeSpans.delete(key);
+        try {
+            active.span.setStatus(status, message);
+        }
+        finally {
+            active.span.end();
+        }
+    };
+    const endMatchingSpans = (predicate, message) => {
+        for (const [key, active] of activeSpans) {
+            if (predicate(active))
+                endSpan(key, "error", message);
+        }
     };
     const handleAgentEvent = (event) => {
         if (!enabled)
@@ -97,21 +113,28 @@ export function createOpenTelemetryInstrumentation(options = {}) {
                     if (!tracer)
                         return;
                     const key = spanKey("agent", event.runId);
-                    activeSpans.set(key, tracer.startSpan("prism.agent.run", {
-                        attributes: {
-                            "prism.session_id": event.sessionId,
-                            "prism.run_id": event.runId,
-                        },
-                    }));
+                    endSpan(key, "error", "Duplicate agent start");
+                    activeSpans.set(key, {
+                        sessionId: event.sessionId,
+                        runId: event.runId,
+                        span: tracer.startSpan("prism.agent.run", {
+                            attributes: {
+                                "prism.session_id": event.sessionId,
+                                "prism.run_id": event.runId,
+                            },
+                        }),
+                    });
                 });
                 break;
             case "agent_finished":
                 safe(() => {
                     endSpan(spanKey("agent", event.runId), "ok");
-                    if (!event.usage || !tokenCounter)
+                    if (!event.usage || !runTokenCounter)
                         return;
-                    tokenCounter.add(event.usage.inputTokens ?? 0, metricAttrs({ kind: "input", scope: "agent" }));
-                    tokenCounter.add(event.usage.outputTokens ?? 0, metricAttrs({ kind: "output", scope: "agent" }));
+                    if (event.usage.inputTokens !== undefined)
+                        runTokenCounter.add(event.usage.inputTokens, metricAttrs({ kind: "input" }));
+                    if (event.usage.outputTokens !== undefined)
+                        runTokenCounter.add(event.usage.outputTokens, metricAttrs({ kind: "output" }));
                 });
                 break;
             case "provider_turn_started":
@@ -120,49 +143,57 @@ export function createOpenTelemetryInstrumentation(options = {}) {
                         return;
                     const attempt = event.metadata.attempt ?? 1;
                     const key = spanKey("provider", event.runId, event.turn, attempt);
-                    activeSpans.set(key, tracer.startSpan("prism.provider.turn", {
-                        attributes: {
-                            "prism.session_id": event.sessionId,
-                            "prism.run_id": event.runId,
-                            "prism.turn": event.turn,
-                            "prism.provider_id": event.metadata.providerId,
-                            "prism.model": event.metadata.model.model,
-                            ...(event.metadata.requestId ? { "prism.request_id": event.metadata.requestId } : {}),
-                            ...(event.metadata.attempt ? { "prism.attempt": event.metadata.attempt } : {}),
-                        },
-                    }));
+                    endSpan(key, "error", "Duplicate provider turn start");
+                    activeSpans.set(key, {
+                        sessionId: event.sessionId,
+                        runId: event.runId,
+                        span: tracer.startSpan("prism.provider.turn", {
+                            attributes: {
+                                "prism.session_id": event.sessionId,
+                                "prism.run_id": event.runId,
+                                "prism.turn": event.turn,
+                                "prism.provider_id": event.metadata.providerId,
+                                "prism.model": event.metadata.model.model,
+                                ...(event.metadata.requestId ? { "prism.request_id": event.metadata.requestId } : {}),
+                                ...(event.metadata.attempt ? { "prism.attempt": event.metadata.attempt } : {}),
+                            },
+                        }),
+                    });
                 });
                 break;
             case "provider_turn_finished":
                 safe(() => {
                     const attempt = event.metadata.attempt ?? 1;
                     const key = spanKey("provider", event.runId, event.turn, attempt);
-                    const span = activeSpans.get(key);
-                    if (span) {
-                        if (event.metadata.latencyMs !== undefined)
-                            span.setAttribute("prism.latency_ms", event.metadata.latencyMs);
-                        if (event.metadata.httpStatus !== undefined)
-                            span.setAttribute("http.status_code", event.metadata.httpStatus);
-                        if (event.error)
-                            span.setStatus("error", event.error.message);
-                        else
-                            span.setStatus("ok");
-                        span.end();
-                        activeSpans.delete(key);
+                    const active = activeSpans.get(key);
+                    if (active) {
+                        try {
+                            if (event.metadata.latencyMs !== undefined)
+                                active.span.setAttribute("prism.latency_ms", event.metadata.latencyMs);
+                            if (event.metadata.httpStatus !== undefined)
+                                active.span.setAttribute("http.status_code", event.metadata.httpStatus);
+                        }
+                        finally {
+                            endSpan(key, event.error ? "error" : "ok", event.error?.message);
+                        }
                     }
                     providerDuration?.record(event.metadata.latencyMs ?? 0, metricAttrs({
                         provider_id: event.metadata.providerId,
                         outcome: event.error ? "error" : "success",
                     }));
-                    if (!event.usage || !tokenCounter)
+                    if (!event.usage || !providerTokenCounter)
                         return;
-                    tokenCounter.add(event.usage.inputTokens ?? 0, metricAttrs({ provider_id: event.metadata.providerId, kind: "input" }));
-                    tokenCounter.add(event.usage.outputTokens ?? 0, metricAttrs({ provider_id: event.metadata.providerId, kind: "output" }));
+                    if (event.usage.inputTokens !== undefined) {
+                        providerTokenCounter.add(event.usage.inputTokens, metricAttrs({ provider_id: event.metadata.providerId, kind: "input" }));
+                    }
+                    if (event.usage.outputTokens !== undefined) {
+                        providerTokenCounter.add(event.usage.outputTokens, metricAttrs({ provider_id: event.metadata.providerId, kind: "output" }));
+                    }
                     if (event.usage.cacheReadTokens) {
-                        tokenCounter.add(event.usage.cacheReadTokens, metricAttrs({ provider_id: event.metadata.providerId, kind: "cache_read" }));
+                        providerTokenCounter.add(event.usage.cacheReadTokens, metricAttrs({ provider_id: event.metadata.providerId, kind: "cache_read" }));
                     }
                     if (event.usage.cacheWriteTokens) {
-                        tokenCounter.add(event.usage.cacheWriteTokens, metricAttrs({ provider_id: event.metadata.providerId, kind: "cache_write" }));
+                        providerTokenCounter.add(event.usage.cacheWriteTokens, metricAttrs({ provider_id: event.metadata.providerId, kind: "cache_write" }));
                     }
                 });
                 break;
@@ -170,15 +201,20 @@ export function createOpenTelemetryInstrumentation(options = {}) {
                 safe(() => {
                     if (!tracer)
                         return;
-                    const key = spanKey("tool", event.call.id);
-                    activeSpans.set(key, tracer.startSpan("prism.tool.execute", {
-                        attributes: {
-                            "prism.session_id": event.sessionId,
-                            "prism.run_id": event.runId,
-                            "prism.tool_call_id": event.call.id,
-                            "prism.tool_name": event.call.name,
-                        },
-                    }));
+                    const key = spanKey("tool", event.runId, event.call.id);
+                    endSpan(key, "error", "Duplicate tool start");
+                    activeSpans.set(key, {
+                        sessionId: event.sessionId,
+                        runId: event.runId,
+                        span: tracer.startSpan("prism.tool.execute", {
+                            attributes: {
+                                "prism.session_id": event.sessionId,
+                                "prism.run_id": event.runId,
+                                "prism.tool_call_id": event.call.id,
+                                "prism.tool_name": event.call.name,
+                            },
+                        }),
+                    });
                 });
                 break;
             case "tool_execution_finished":
@@ -190,28 +226,78 @@ export function createOpenTelemetryInstrumentation(options = {}) {
                         : event.type === "tool_execution_error"
                             ? event.call.id
                             : event.toolCallId;
-                    const key = spanKey("tool", toolCallId);
-                    const span = activeSpans.get(key);
-                    if (span) {
-                        span.setAttribute("prism.tool_status", event.metadata.status);
-                        span.setAttribute("prism.duration_ms", event.metadata.durationMs);
-                        if (event.type === "tool_execution_finished")
-                            span.setStatus("ok");
-                        else
-                            span.setStatus("error", event.error.message);
-                        span.end();
-                        activeSpans.delete(key);
+                    const key = spanKey("tool", event.runId, toolCallId);
+                    const active = activeSpans.get(key);
+                    if (active) {
+                        try {
+                            active.span.setAttribute("prism.tool_status", event.metadata.status);
+                            active.span.setAttribute("prism.duration_ms", event.metadata.durationMs);
+                        }
+                        finally {
+                            endSpan(key, event.type === "tool_execution_finished" ? "ok" : "error", event.type === "tool_execution_finished" ? undefined : event.error.message);
+                        }
                     }
                     toolDuration?.record(event.metadata.durationMs, metricAttrs({ status: event.metadata.status }));
                 });
                 break;
+            case "error":
+                safe(() => {
+                    if (event.runId)
+                        endMatchingSpans((active) => active.runId === event.runId, event.error.message);
+                    else if (event.sessionId)
+                        endMatchingSpans((active) => active.sessionId === event.sessionId, event.error.message);
+                });
+                break;
             default:
                 break;
         }
     };
+    const recordRunMetadata = (runId, eventName, spanName, attributes) => safe(() => {
+        const active = activeSpans.get(spanKey("agent", runId));
+        if (active?.span.addEvent) {
+            active.span.addEvent(eventName, attributes);
+            return;
+        }
+        if (!tracer)
+            return;
+        const span = tracer.startSpan(spanName, { attributes: { "prism.run_id": runId, ...attributes } });
+        span.setStatus("ok");
+        span.end();
+    });
     return {
         enabled,
         handleAgentEvent,
+        handleRunFeedback(feedback) {
+            const rating = feedback.rating !== undefined && Number.isFinite(feedback.rating) && feedback.rating >= -1 && feedback.rating <= 1
+                ? feedback.rating
+                : undefined;
+            const tagCount = boundedCount(feedback.tagCount);
+            const scorerCount = boundedCount(feedback.scorerCount);
+            const evaluationCount = boundedCount(feedback.evaluationCount);
+            const attributes = {
+                "prism.feedback.rating": rating ?? 0,
+                "prism.feedback.has_rating": rating !== undefined,
+                "prism.feedback.has_comment": Boolean(feedback.hasComment),
+                "prism.feedback.tag_count": tagCount,
+                "prism.feedback.scorer_count": scorerCount,
+                "prism.feedback.evaluation_count": evaluationCount,
+            };
+            recordRunMetadata(feedback.runId, "prism.run.feedback", "prism.run.feedback", attributes);
+            safe(() => feedbackCounter?.add(1, metricAttrs({
+                rating: rating === undefined ? "none" : rating > 0 ? "positive" : rating < 0 ? "negative" : "neutral",
+                linked_evaluation: evaluationCount > 0 ? "true" : "false",
+            })));
+        },
+        handleEvaluation(evaluation) {
+            const attributes = {
+                "prism.evaluation.status": evaluation.status,
+                "prism.evaluation.has_score": evaluation.score !== undefined && Number.isFinite(evaluation.score),
+                "prism.evaluation.has_reason": Boolean(evaluation.hasReason),
+                ...(evaluation.score !== undefined && Number.isFinite(evaluation.score) ? { "prism.evaluation.score": evaluation.score } : {}),
+            };
+            recordRunMetadata(evaluation.runId, "prism.run.evaluation", "prism.run.evaluation", attributes);
+            safe(() => evaluationCounter?.add(1, metricAttrs({ status: evaluation.status })));
+        },
         attachSession(session) {
             if (!enabled)
                 return () => { };
@@ -222,7 +308,7 @@ export function createOpenTelemetryInstrumentation(options = {}) {
                 try {
                     while (active) {
                         const next = await iterator.next();
-                        if (next.done)
+                        if (!active || next.done)
                             break;
                         handleAgentEvent(next.value);
                     }
@@ -234,10 +320,14 @@ export function createOpenTelemetryInstrumentation(options = {}) {
             return () => {
                 active = false;
                 void iterator.return?.();
+                safe(() => endMatchingSpans((span) => span.sessionId === session.id, "Instrumentation detached"));
             };
         },
     };
 }
+function boundedCount(value) {
+    return Number.isSafeInteger(value) && value > 0 ? Math.min(value, 64) : 0;
+}
 const OTEL_STATUS_OK = 1;
 const OTEL_STATUS_ERROR = 2;
 export function wrapOpenTelemetryApi(tracer, meter) {
@@ -252,6 +342,9 @@ export function wrapOpenTelemetryApi(tracer, meter) {
                     setStatus(code, message) {
                         span.setStatus({ code: code === "ok" ? OTEL_STATUS_OK : OTEL_STATUS_ERROR, message });
                     },
+                    addEvent(name, attributes) {
+                        span.addEvent?.(name, attributes ? { ...attributes } : undefined);
+                    },
                     end() {
                         span.end();
                     },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arnilo/prism-observability-opentelemetry",
-  "version": "0.0.4",
+  "version": "0.0.6",
   "description": "Optional OpenTelemetry adapter for Prism agent events.",
   "type": "module",
   "main": "./dist/index.js",
@@ -25,7 +25,7 @@
     "pack:dry-run": "npm pack --dry-run"
   },
   "peerDependencies": {
-    "@arnilo/prism": "0.0.4"
+    "@arnilo/prism": "0.0.6"
   },
   "devDependencies": {
     "@arnilo/prism": "file:../..",