@zhivex-ai/core 1.4.0 → 1.6.0

package/dist/observability.js

@@ -1,137 +1,445 @@
 import { createTelemetryMiddleware } from "./middleware.js";
+/** Zhivex's stable, privacy-first mapping contract for GenAI telemetry. */
+export const OTEL_GENAI_CONTRACT_VERSION = 1;
+/** Upstream Development revision used to audit the v1 mapping. */
+export const OTEL_GENAI_SEMCONV_REVISION = "a685613a207a580163353b8e48a7ad88967e7b42";
 const OTEL_API_MODULE = "@opentelemetry/api";
-// Keep OpenTelemetry optional for consumers that never use the OTEL helpers.
-const loadOptionalOtelApi = async () => {
-    try {
-        return await import(OTEL_API_MODULE);
-    }
-    catch {
-        return undefined;
-    }
+const GEN_AI = {
+    agentName: "gen_ai.agent.name",
+    operationName: "gen_ai.operation.name",
+    providerName: "gen_ai.provider.name",
+    requestMaxTokens: "gen_ai.request.max_tokens",
+    requestModel: "gen_ai.request.model",
+    requestReasoningLevel: "gen_ai.request.reasoning.level",
+    requestStream: "gen_ai.request.stream",
+    requestTemperature: "gen_ai.request.temperature",
+    responseFinishReasons: "gen_ai.response.finish_reasons",
+    responseTimeToFirstChunk: "gen_ai.response.time_to_first_chunk",
+    tokenType: "gen_ai.token.type",
+    toolCallId: "gen_ai.tool.call.id",
+    toolName: "gen_ai.tool.name",
+    toolType: "gen_ai.tool.type",
+    usageCacheCreationInputTokens: "gen_ai.usage.cache_creation.input_tokens",
+    usageCacheReadInputTokens: "gen_ai.usage.cache_read.input_tokens",
+    usageInputTokens: "gen_ai.usage.input_tokens",
+    usageOutputTokens: "gen_ai.usage.output_tokens",
+    usageReasoningOutputTokens: "gen_ai.usage.reasoning.output_tokens"
+};
+const GEN_AI_OPERATION = {
+    chat: "chat",
+    executeTool: "execute_tool",
+    generateContent: "generate_content",
+    invokeAgent: "invoke_agent"
+};
+const PROVIDER_NAMES = {
+    "azure-openai": "azure.ai.openai",
+    bedrock: "aws.bedrock",
+    gemini: "gcp.gemini",
+    google: "gcp.gen_ai",
+    kimi: "moonshot_ai",
+    vertex: "gcp.vertex_ai",
+    xai: "x_ai"
 };
-const toSpanAttributeValue = (value) => {
-    if (value === null || typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
+const DURATION_BUCKETS = [
+    0.01, 0.02, 0.04, 0.08, 0.16, 0.32, 0.64, 1.28, 2.56, 5.12, 10.24, 20.48, 40.96, 81.92
+];
+const TOKEN_BUCKETS = [
+    1, 4, 16, 64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216, 67108864
+];
+const AGENT_DURATION_BUCKETS = [0.1, 0.2, 0.4, 0.8, 1.6, 3.2, 6.4, 12.8, 25.6, 51.2, 102.4, 204.8, 409.6];
+const CALL_COUNT_BUCKETS = [1, 2, 4, 8, 16, 32, 64, 128];
+let optionalOtelApiPromise;
+// OpenTelemetry stays optional for consumers that never instantiate these helpers.
+const loadOptionalOtelApi = () => {
+    optionalOtelApiPromise ??= import(OTEL_API_MODULE).catch(() => undefined);
+    return optionalOtelApiPromise;
+};
+const toProviderName = (provider) => PROVIDER_NAMES[provider] ?? provider;
+const toModelOperationName = (provider) => provider === "gemini" || provider === "google" || provider === "vertex"
+    ? GEN_AI_OPERATION.generateContent
+    : GEN_AI_OPERATION.chat;
+const toAttributeValue = (value) => {
+    if (typeof value === "string" || typeof value === "number" || typeof value === "boolean")
         return value;
-    }
     if (Array.isArray(value)) {
-        const normalized = value
-            .map((item) => (item === null || typeof item === "string" || typeof item === "number" || typeof item === "boolean" ? item : JSON.stringify(item)))
-            .filter((item) => item !== undefined);
-        return normalized.length ? normalized : undefined;
+        const normalized = value.filter((item) => item !== null && item !== undefined);
+        if (!normalized.length)
+            return undefined;
+        if (normalized.every((item) => typeof item === "string"))
+            return normalized;
+        if (normalized.every((item) => typeof item === "number"))
+            return normalized;
+        if (normalized.every((item) => typeof item === "boolean"))
+            return normalized;
+        return normalized.map((item) => {
+            if (typeof item === "string")
+                return item;
+            try {
+                return JSON.stringify(item) ?? String(item);
+            }
+            catch {
+                return String(item);
+            }
+        });
     }
-    if (value === undefined) {
+    if (value === null || value === undefined)
         return undefined;
-    }
     try {
-        return JSON.stringify(value);
+        return JSON.stringify(value) ?? String(value);
     }
     catch {
         return String(value);
     }
 };
-const toSpanAttributes = (attributes) => {
-    const entries = Object.entries(attributes ?? {})
-        .map(([key, value]) => [key, toSpanAttributeValue(value)])
-        .filter((entry) => entry[1] !== undefined);
-    return Object.fromEntries(entries);
-};
+const toAttributes = (attributes) => Object.fromEntries(Object.entries(attributes ?? {})
+    .map(([key, value]) => [key, toAttributeValue(value)])
+    .filter((entry) => entry[1] !== undefined));
 const safeSetAttributes = (span, attributes) => {
-    for (const [key, value] of Object.entries(toSpanAttributes(attributes))) {
+    for (const [key, value] of Object.entries(toAttributes(attributes)))
         span.setAttribute?.(key, value);
-    }
 };
 const safeAddEvent = (span, name, attributes) => {
-    if (!span?.addEvent) {
+    span?.addEvent?.(name, toAttributes(attributes));
+};
+const GUARDRAIL_METADATA_ALLOWLIST = {
+    actual: "actual",
+    budgetLimit: "budget_limit",
+    includeChildRuns: "include_child_runs",
+    limit: "limit",
+    operation: "operation",
+    remaining: "remaining",
+    required: "required"
+};
+const truncateTelemetryString = (value, maxLength) => value.length <= maxLength ? value : `${value.slice(0, maxLength)}…`;
+const safeGuardrailMetadataAttributes = (metadata) => {
+    const attributes = {};
+    for (const [sourceKey, attributeKey] of Object.entries(GUARDRAIL_METADATA_ALLOWLIST)) {
+        const value = metadata?.[sourceKey];
+        if (typeof value === "string") {
+            attributes[`zhivex.guardrail.metadata.${attributeKey}`] = truncateTelemetryString(value, 128);
+        }
+        else if (typeof value === "number" && Number.isFinite(value)) {
+            attributes[`zhivex.guardrail.metadata.${attributeKey}`] = value;
+        }
+        else if (typeof value === "boolean") {
+            attributes[`zhivex.guardrail.metadata.${attributeKey}`] = value;
+        }
+    }
+    return attributes;
+};
+const errorType = (error) => error.name || error.constructor?.name || "_OTHER";
+const seconds = (milliseconds) => Math.max(0, milliseconds) / 1_000;
+const usageAttributes = (usage) => ({
+    [GEN_AI.usageInputTokens]: usage?.inputTokens,
+    [GEN_AI.usageOutputTokens]: usage?.outputTokens,
+    [GEN_AI.usageCacheReadInputTokens]: usage?.cachedInputTokens,
+    [GEN_AI.usageCacheCreationInputTokens]: usage?.cacheWriteTokens,
+    [GEN_AI.usageReasoningOutputTokens]: usage?.reasoningTokens
+});
+const isStreamTelemetry = (event) => event.type.startsWith("stream-");
+const requestAttributes = (event) => ({
+    [GEN_AI.operationName]: toModelOperationName(event.model.provider),
+    [GEN_AI.providerName]: toProviderName(event.model.provider),
+    [GEN_AI.requestModel]: event.model.modelId,
+    [GEN_AI.requestStream]: isStreamTelemetry(event) ? true : undefined,
+    [GEN_AI.requestTemperature]: event.input.temperature,
+    [GEN_AI.requestMaxTokens]: event.input.maxTokens,
+    [GEN_AI.requestReasoningLevel]: event.input.reasoning?.effort,
+    "zhivex.provider": event.model.provider,
+    "zhivex.model_id": event.model.modelId
+});
+const clientMetricAttributes = (event, error) => ({
+    [GEN_AI.operationName]: toModelOperationName(event.model.provider),
+    [GEN_AI.providerName]: toProviderName(event.model.provider),
+    [GEN_AI.requestModel]: event.model.modelId,
+    ...(error ? { "error.type": errorType(error) } : {})
+});
+const responseAttributes = (options) => ({
+    [GEN_AI.responseFinishReasons]: options.finishReason ? [options.finishReason] : undefined,
+    ...usageAttributes(options.usage),
+    "zhivex.finish_reason": options.finishReason,
+    "zhivex.provider_finish_reason": options.providerFinishReason,
+    "zhivex.output_chunk_count": options.outputChunkCount,
+    "zhivex.latency_ms": options.latencyMs
+});
+const histogram = (meter, name, unit, description, explicitBucketBoundaries) => meter.createHistogram(name, {
+    unit,
+    description,
+    advice: { explicitBucketBoundaries: [...explicitBucketBoundaries] }
+});
+const createGenAiMetrics = (meter) => ({
+    clientTokenUsage: histogram(meter, "gen_ai.client.token.usage", "{token}", "Number of input and output tokens used.", TOKEN_BUCKETS),
+    clientOperationDuration: histogram(meter, "gen_ai.client.operation.duration", "s", "GenAI operation duration.", DURATION_BUCKETS),
+    clientTimeToFirstChunk: histogram(meter, "gen_ai.client.operation.time_to_first_chunk", "s", "Time to receive the first chunk, measured from when the client issues the generation request to when the first chunk is received in the response stream.", DURATION_BUCKETS),
+    clientTimePerOutputChunk: histogram(meter, "gen_ai.client.operation.time_per_output_chunk", "s", "Time per output chunk, recorded for each chunk received after the first one, measured as the time elapsed from the end of the previous chunk to the end of the current chunk.", DURATION_BUCKETS),
+    invokeAgentDuration: histogram(meter, "gen_ai.invoke_agent.duration", "s", "The end-to-end duration of a single in-process agent invocation, from the moment the invocation starts until the agent emits the last chunk of its final response or terminates with an error.", AGENT_DURATION_BUCKETS),
+    invokeAgentInferenceCalls: histogram(meter, "gen_ai.invoke_agent.inference_calls", "{inference_call}", "The number of inference (model) calls a GenAI agent makes during a single invocation.", CALL_COUNT_BUCKETS),
+    invokeAgentToolCalls: histogram(meter, "gen_ai.invoke_agent.tool_calls", "{tool_call}", "The number of tool calls an agent makes during a single invocation.", CALL_COUNT_BUCKETS),
+    executeToolDuration: histogram(meter, "gen_ai.execute_tool.duration", "s", "The duration of a single tool execution.", DURATION_BUCKETS)
+});
+const safeRecord = (instrument, value, attributes) => {
+    if (value === undefined || !Number.isFinite(value) || value < 0)
         return;
+    try {
+        instrument?.record(value, toAttributes(attributes));
+    }
+    catch {
+        // Exporters are observational and cannot own the instrumented operation outcome.
     }
-    span.addEvent(name, toSpanAttributes(attributes));
+};
+const recordTokenUsage = (metrics, usage, attributes) => {
+    safeRecord(metrics?.clientTokenUsage, usage?.inputTokens, { ...attributes, [GEN_AI.tokenType]: "input" });
+    safeRecord(metrics?.clientTokenUsage, usage?.outputTokens, { ...attributes, [GEN_AI.tokenType]: "output" });
+};
+const resolveMeter = async (options) => {
+    if (options.meter)
+        return options.meter;
+    if (options.meterProvider) {
+        return options.meterProvider.getMeter(options.meterName ?? options.tracerName ?? "zhivex-ai", options.version);
+    }
+    const otel = await loadOptionalOtelApi();
+    return otel?.metrics.getMeter(options.meterName ?? options.tracerName ?? "zhivex-ai", options.version);
 };
 export class OTelSpanHandle {
     span;
     statusSetter;
+    endPromise;
     constructor(span, statusSetter) {
         this.span = span;
         this.statusSetter = statusSetter;
     }
-    async end(options = {}) {
-        safeSetAttributes(this.span, options.attributes);
-        if (options.error) {
-            this.span.recordException?.(options.error);
-            await this.statusSetter?.(this.span, options.error);
+    get ended() {
+        return this.endPromise !== undefined;
+    }
+    end(options = {}) {
+        this.endPromise ??= this.finish(options);
+        return this.endPromise;
+    }
+    async finish(options) {
+        try {
+            safeSetAttributes(this.span, {
+                ...(options.error ? { "error.type": errorType(options.error) } : {}),
+                ...options.attributes
+            });
+            if (options.error) {
+                this.span.recordException?.(options.error);
+                await this.statusSetter?.(this.span, options.error);
+            }
+        }
+        finally {
+            this.span.end(options.endTime);
         }
-        this.span.end();
     }
 }
 export class OTelObserver {
     tracer;
     statusSetter;
-    constructor(tracer, statusSetter) {
+    contextFor;
+    spanKindFor;
+    runWithSpan;
+    constructor(tracer, statusSetter, contextFor, spanKindFor, runWithSpan) {
         this.tracer = tracer;
         this.statusSetter = statusSetter;
+        this.contextFor = contextFor;
+        this.spanKindFor = spanKindFor;
+        this.runWithSpan = runWithSpan;
     }
-    startSpan(name, attributes) {
+    startSpan(name, attributes, options = {}) {
         const span = this.tracer.startSpan(name, {
-            attributes: toSpanAttributes(attributes)
-        });
+            attributes: toAttributes(attributes),
+            kind: options.kind ? this.spanKindFor?.(options.kind) : undefined,
+            startTime: options.startTime
+        }, this.contextFor?.(options.parent?.span));
         return new OTelSpanHandle(span, this.statusSetter);
     }
+    async withSpan(handle, callback) {
+        if (!this.runWithSpan)
+            return callback();
+        return this.runWithSpan(handle.span, callback);
+    }
 }
-const createDefaultStatusSetter = async () => {
-    const otel = await loadOptionalOtelApi();
-    if (!otel) {
+const createDefaultStatusSetter = (otel) => {
+    if (!otel)
         return undefined;
-    }
     return (span, error) => {
-        span.setStatus?.({
-            code: otel.SpanStatusCode.ERROR,
-            message: error.message
-        });
+        span.setStatus?.({ code: otel.SpanStatusCode.ERROR, message: error.message });
     };
 };
 export const createOtelObserver = async (options = {}) => {
-    const statusSetter = await createDefaultStatusSetter();
-    if (options.tracer) {
-        return new OTelObserver(options.tracer, statusSetter);
-    }
     const otel = await loadOptionalOtelApi();
-    if (!otel) {
+    if (!options.tracer && !options.tracerProvider && !otel) {
         throw new Error('OpenTelemetry is not installed. Install "@opentelemetry/api" to use OTEL observability helpers.', { cause: new Error(`Missing optional dependency: ${OTEL_API_MODULE}`) });
     }
-    return new OTelObserver(otel.trace.getTracer(options.tracerName ?? "zhivex-ai", options.version), statusSetter);
+    const tracer = options.tracer
+        ?? options.tracerProvider?.getTracer(options.tracerName ?? "zhivex-ai", options.version)
+        ?? otel.trace.getTracer(options.tracerName ?? "zhivex-ai", options.version);
+    return new OTelObserver(tracer, createDefaultStatusSetter(otel), otel ? (parent) => parent ? otel.trace.setSpan(otel.context.active(), parent) : otel.context.active() : undefined, otel ? (kind) => kind === "client" ? otel.SpanKind.CLIENT : otel.SpanKind.INTERNAL : undefined, otel
+        ? (span, callback) => otel.context.with(otel.trace.setSpan(otel.context.active(), span), callback)
+        : undefined);
+};
+const settleSpanEnds = async (handles, attributes, endTime) => {
+    const results = await Promise.allSettled(Array.from(handles, (handle) => handle.end({ attributes, endTime })));
+    const rejection = results.find((result) => result.status === "rejected");
+    if (rejection)
+        throw rejection.reason;
+};
+const agentStatusError = (event) => {
+    if (event.status === "failed")
+        return new Error(event.state.error?.message ?? "Agent run failed.");
+    if (event.status === "timed_out") {
+        return Object.assign(new Error("Agent run timed out."), { name: "TimeoutError" });
+    }
+    return undefined;
 };
 export const createOtelAgentObserver = async (options = {}) => {
-    const observer = options.observer ?? (await createOtelObserver(options));
+    const [observer, meter] = await Promise.all([
+        options.observer ? Promise.resolve(options.observer) : createOtelObserver(options),
+        resolveMeter(options)
+    ]);
+    const metrics = meter ? createGenAiMetrics(meter) : undefined;
     const runSpans = new Map();
     const stepSpans = new Map();
-    const prefix = options.spanNamePrefix ?? "zhivex.agent";
-    return async (event) => {
-        if (event.type === "run-start") {
-            runSpans.set(event.runId, observer.startSpan(`${prefix}.run`, {
-                "zhivex.run_id": event.runId,
+    const metricStates = new Map();
+    const queues = new Map();
+    const prefix = options.spanNamePrefix?.trim();
+    const runSpanName = (agentName) => prefix ? `${prefix}.run` : `${GEN_AI_OPERATION.invokeAgent}${agentName ? ` ${agentName}` : ""}`;
+    const stepSpanName = () => prefix ? `${prefix}.step` : "agent_step";
+    const closeSteps = async (runId, attributes, endTime) => {
+        const spans = stepSpans.get(runId);
+        stepSpans.delete(runId);
+        if (spans)
+            await settleSpanEnds(spans.values(), attributes, endTime);
+    };
+    const recordAgentMetrics = (runId, endTime, error) => {
+        const state = metricStates.get(runId);
+        metricStates.delete(runId);
+        if (!state)
+            return;
+        const nameAttributes = { [GEN_AI.agentName]: state.agentName };
+        safeRecord(metrics?.invokeAgentDuration, seconds(endTime - state.startedAt), {
+            ...nameAttributes,
+            [GEN_AI.requestModel]: state.modelId,
+            ...(error ? { "error.type": errorType(error) } : {})
+        });
+        safeRecord(metrics?.invokeAgentInferenceCalls, state.inferenceCalls, nameAttributes);
+        safeRecord(metrics?.invokeAgentToolCalls, state.toolCalls, nameAttributes);
+    };
+    const closeRun = async (runId, attributes, error, endTime = Date.now()) => {
+        const run = runSpans.get(runId);
+        runSpans.delete(runId);
+        let firstError;
+        try {
+            await closeSteps(runId, {
+                "zhivex.step_status": "interrupted",
+                "zhivex.run_status": attributes["zhivex.status"]
+            }, endTime);
+        }
+        catch (caught) {
+            firstError = caught;
+        }
+        try {
+            await run?.end({ attributes, error, endTime });
+        }
+        catch (caught) {
+            firstError ??= caught;
+        }
+        try {
+            recordAgentMetrics(runId, endTime, error);
+        }
+        catch (caught) {
+            firstError ??= caught;
+        }
+        if (firstError)
+            throw firstError;
+    };
+    const startRun = async (event) => {
+        const existing = runSpans.get(event.runId);
+        if (existing) {
+            const metricState = metricStates.get(event.runId);
+            if (metricState) {
+                metricState.agentName = event.agentName ?? metricState.agentName;
+                metricState.modelId = event.modelId;
+            }
+            safeSetAttributes(existing.span, {
+                [GEN_AI.agentName]: event.agentName,
+                [GEN_AI.requestModel]: event.modelId,
                 "zhivex.agent_id": event.agentId,
                 "zhivex.provider": event.provider,
                 "zhivex.model_id": event.modelId,
                 "zhivex.max_steps": event.maxSteps
-            }));
+            });
+            return;
+        }
+        const startedAt = event.startedAt ?? Date.now();
+        await closeRun(event.runId, { "zhivex.status": "replaced" }, undefined, startedAt);
+        metricStates.set(event.runId, {
+            startedAt,
+            agentName: event.agentName,
+            modelId: event.modelId,
+            inferenceCalls: 0,
+            toolCalls: 0
+        });
+        runSpans.set(event.runId, observer.startSpan(runSpanName(event.agentName), {
+            [GEN_AI.operationName]: GEN_AI_OPERATION.invokeAgent,
+            [GEN_AI.agentName]: event.agentName,
+            [GEN_AI.requestModel]: event.modelId,
+            "zhivex.run_id": event.runId,
+            "zhivex.agent_id": event.agentId,
+            "zhivex.provider": event.provider,
+            "zhivex.model_id": event.modelId,
+            "zhivex.max_steps": event.maxSteps
+        }, { kind: "internal", startTime: startedAt }));
+    };
+    const processEvent = async (event) => {
+        if (event.type === "run-start") {
+            await startRun(event);
             return;
         }
         if (event.type === "step-start") {
-            stepSpans.set(`${event.runId}:${event.stepIndex}`, observer.startSpan(`${prefix}.step`, {
+            const metricState = metricStates.get(event.runId);
+            if (metricState)
+                metricState.inferenceCalls += 1;
+            const spans = stepSpans.get(event.runId) ?? new Map();
+            const previous = spans.get(event.stepIndex);
+            spans.delete(event.stepIndex);
+            await previous?.end({ attributes: { "zhivex.step_status": "replaced" }, endTime: event.startedAt });
+            spans.set(event.stepIndex, observer.startSpan(stepSpanName(), {
+                [GEN_AI.agentName]: event.agentName ?? metricState?.agentName,
                 "zhivex.run_id": event.runId,
                 "zhivex.agent_id": event.agentId,
                 "zhivex.step_index": event.stepIndex
+            }, {
+                kind: "internal",
+                parent: runSpans.get(event.runId),
+                startTime: event.startedAt
             }));
+            stepSpans.set(event.runId, spans);
             return;
         }
         if (event.type === "step-finish") {
-            const key = `${event.runId}:${event.step.index}`;
-            const handle = stepSpans.get(key);
-            stepSpans.delete(key);
+            const spans = stepSpans.get(event.runId);
+            const handle = spans?.get(event.step.index);
+            spans?.delete(event.step.index);
+            if (spans?.size === 0)
+                stepSpans.delete(event.runId);
             await handle?.end({
                 attributes: {
                     "zhivex.step_status": event.step.status,
                     "zhivex.tool_results": event.step.toolResults.length
-                }
+                },
+                error: event.step.status === "failed" ? new Error(event.step.error?.message ?? "Agent step failed.") : undefined,
+                endTime: event.step.finishedAt
+            });
+            return;
+        }
+        if (event.type === "tool-start") {
+            const metricState = metricStates.get(event.runId);
+            if (metricState)
+                metricState.toolCalls += 1;
+            safeAddEvent(runSpans.get(event.runId)?.span, "tool-start", {
+                [GEN_AI.toolName]: event.toolCall.name,
+                [GEN_AI.toolCallId]: event.toolCall.id,
+                "zhivex.step_index": event.stepIndex
             });
             return;
         }
@@ -154,6 +462,8 @@ export const createOtelAgentObserver = async (options = {}) => {
         }
         if (event.type === "tool-approval") {
             safeAddEvent(runSpans.get(event.runId)?.span, "tool-approval", {
+                [GEN_AI.toolName]: event.toolCall.name,
+                [GEN_AI.toolCallId]: event.toolCall.id,
                 "zhivex.tool_name": event.toolCall.name,
                 "zhivex.tool_call_id": event.toolCall.id,
                 "zhivex.approved": event.approved,
@@ -162,16 +472,14 @@ export const createOtelAgentObserver = async (options = {}) => {
             return;
         }
         if (event.type === "memory-loaded") {
-            safeAddEvent(runSpans.get(event.runId)?.span, "memory-loaded", {
-                "zhivex.message_count": event.messageCount
-            });
+            safeAddEvent(runSpans.get(event.runId)?.span, "memory-loaded", { "zhivex.message_count": event.messageCount });
             return;
         }
         if (event.type === "guardrail-triggered") {
             safeAddEvent(runSpans.get(event.runId)?.span, "guardrail-triggered", {
                 "zhivex.stage": event.stage,
-                "zhivex.reason": event.reason,
-                ...event.metadata
+                "zhivex.reason": event.reason ? truncateTelemetryString(event.reason, 256) : undefined,
+                ...safeGuardrailMetadataAttributes(event.metadata)
             });
             return;
         }
@@ -183,97 +491,210 @@ export const createOtelAgentObserver = async (options = {}) => {
             });
             return;
         }
-        if (event.type === "state-saved") {
-            safeAddEvent(runSpans.get(event.runId)?.span, "state-saved", {
-                "zhivex.status": event.status
+        if (event.type === "subagent-start") {
+            safeAddEvent(runSpans.get(event.runId)?.span, "subagent-start", {
+                "zhivex.child_agent_id": event.childAgentId,
+                "zhivex.tool_name": event.toolName
             });
             return;
         }
-        if (event.type === "run-finish") {
-            const handle = runSpans.get(event.runId);
-            runSpans.delete(event.runId);
-            await handle?.end({
-                attributes: {
-                    "zhivex.status": event.status,
-                    "zhivex.current_step": event.state.currentStep,
-                    "zhivex.pending_approvals": event.state.pendingApprovals.length,
-                    "zhivex.finish_reason": event.state.finishReason,
-                    "zhivex.provider_finish_reason": event.state.providerFinishReason
-                },
-                error: event.status === "failed" && event.state.error?.message
-                    ? new Error(event.state.error.message)
-                    : undefined
+        if (event.type === "subagent-finish") {
+            safeAddEvent(runSpans.get(event.runId)?.span, "subagent-finish", {
+                "zhivex.child_run_id": event.childRun.runId,
+                "zhivex.child_agent_id": event.childRun.agentId,
+                "zhivex.status": event.childRun.status
             });
+            return;
+        }
+        if (event.type === "state-saved") {
+            safeAddEvent(runSpans.get(event.runId)?.span, "state-saved", { "zhivex.status": event.status });
+            return;
         }
+        if (event.type === "run-finish") {
+            const error = agentStatusError(event);
+            const finishedAt = event.finishedAt ?? event.state.updatedAt ?? Date.now();
+            await closeRun(event.runId, {
+                "zhivex.status": event.status,
+                "zhivex.current_step": event.state.currentStep,
+                "zhivex.pending_approvals": event.state.pendingApprovals.length,
+                "zhivex.finish_reason": event.state.finishReason,
+                "zhivex.provider_finish_reason": event.state.providerFinishReason,
+                "zhivex.usage_input_tokens": event.state.usage?.inputTokens,
+                "zhivex.usage_output_tokens": event.state.usage?.outputTokens,
+                "zhivex.usage_total_tokens": event.state.usage?.totalTokens
+            }, error, finishedAt);
+        }
+    };
+    const enqueue = (runId, operation) => {
+        const previous = queues.get(runId) ?? Promise.resolve();
+        const task = previous.catch(() => undefined).then(operation);
+        queues.set(runId, task);
+        return task.finally(() => {
+            if (queues.get(runId) === task)
+                queues.delete(runId);
+        });
     };
+    const telemetryObserver = (event) => enqueue(event.runId, () => processEvent(event));
+    telemetryObserver.startInvocation = (event) => enqueue(event.runId, () => startRun(event));
+    telemetryObserver.finishInvocation = (event) => enqueue(event.runId, () => closeRun(event.runId, { "zhivex.status": event.status }, event.error, event.finishedAt));
+    telemetryObserver.withRunContext = async (runId, callback) => {
+        const handle = runSpans.get(runId);
+        return handle ? observer.withSpan(handle, callback) : callback();
+    };
+    telemetryObserver.withStepContext = async (runId, stepIndex, callback) => {
+        const handle = stepSpans.get(runId)?.get(stepIndex) ?? runSpans.get(runId);
+        return handle ? observer.withSpan(handle, callback) : callback();
+    };
+    return telemetryObserver;
 };
 export const createOtelTelemetryMiddleware = async (options = {}) => {
-    const observer = options.observer ?? (await createOtelObserver(options));
+    const [observer, meter] = await Promise.all([
+        options.observer ? Promise.resolve(options.observer) : createOtelObserver(options),
+        resolveMeter(options)
+    ]);
+    const metrics = meter ? createGenAiMetrics(meter) : undefined;
     const spans = new Map();
-    const prefix = options.spanNamePrefix ?? "zhivex.model";
+    const inputIds = new WeakMap();
+    let nextInputId = 1;
+    const prefix = options.spanNamePrefix?.trim();
+    const inputId = (input) => {
+        const existing = inputIds.get(input);
+        if (existing !== undefined)
+            return existing;
+        const id = nextInputId;
+        nextInputId += 1;
+        inputIds.set(input, id);
+        return id;
+    };
     const keyFor = (event) => {
         if (event.type === "generate-start" || event.type === "generate-finish" || event.type === "generate-error") {
-            return `generate:${event.model.provider}:${event.model.modelId}:${event.startedAt}`;
+            return event.generateId === undefined
+                ? `generate:${event.model.provider}:${event.model.modelId}:${inputId(event.input)}:${event.startedAt}`
+                : `generate:${event.generateId}`;
         }
-        if (event.type === "stream-start" || event.type === "stream-finish" || event.type === "stream-error") {
-            return `stream:${event.model.provider}:${event.model.modelId}:${event.startedAt}`;
+        if (event.type === "stream-start" || event.type === "stream-chunk" || event.type === "stream-finish" || event.type === "stream-error") {
+            return event.streamId === undefined
+                ? `stream:${event.model.provider}:${event.model.modelId}:${inputId(event.input)}:${event.startedAt}`
+                : `stream:${event.streamId}`;
         }
-        return `tool:${event.model.provider}:${event.model.modelId}:${event.step}:${event.toolCall.id}:${event.startedAt}`;
+        return `tool:${event.model.provider}:${event.model.modelId}:${inputId(event.input)}:${event.step}:${event.toolCall.id}:${event.startedAt}`;
     };
+    const pushSpan = (key, handle) => {
+        const queued = spans.get(key) ?? [];
+        queued.push(handle);
+        spans.set(key, queued);
+    };
+    const peekSpan = (key) => spans.get(key)?.[0];
+    const shiftSpan = (key) => {
+        const queued = spans.get(key);
+        const handle = queued?.shift();
+        if (queued?.length === 0)
+            spans.delete(key);
+        return handle;
+    };
+    const modelSpanName = (modelId, provider, operation) => prefix ? `${prefix}.${operation}` : `${toModelOperationName(provider)} ${modelId}`;
+    const toolSpanName = (toolName) => prefix ? `${prefix}.tool` : `${GEN_AI_OPERATION.executeTool} ${toolName}`;
     return createTelemetryMiddleware({
         onEvent: async (event) => {
             if (event.type === "generate-start" || event.type === "stream-start") {
-                spans.set(keyFor(event), observer.startSpan(`${prefix}.${event.type === "generate-start" ? "generate" : "stream"}`, {
-                    "zhivex.provider": event.model.provider,
-                    "zhivex.model_id": event.model.modelId
-                }));
+                pushSpan(keyFor(event), observer.startSpan(modelSpanName(event.model.modelId, event.model.provider, event.type === "generate-start" ? "generate" : "stream"), {
+                    ...requestAttributes(event),
+                    "zhivex.operation_type": event.type === "generate-start" ? "generate" : "stream",
+                    "zhivex.generate_id": event.type === "generate-start" ? event.generateId : undefined,
+                    "zhivex.stream_id": event.type === "stream-start" ? event.streamId : undefined
+                }, { kind: "client", startTime: event.startedAt }));
+                return;
+            }
+            if (event.type === "stream-chunk") {
+                const attributes = clientMetricAttributes(event);
+                if (event.chunkIndex === 1 && event.timeToFirstChunkMs !== undefined) {
+                    const duration = seconds(event.timeToFirstChunkMs);
+                    safeRecord(metrics?.clientTimeToFirstChunk, duration, attributes);
+                    const span = peekSpan(keyFor(event))?.span;
+                    if (span)
+                        safeSetAttributes(span, { [GEN_AI.responseTimeToFirstChunk]: duration });
+                }
+                if (event.timeSincePreviousChunkMs !== undefined) {
+                    safeRecord(metrics?.clientTimePerOutputChunk, seconds(event.timeSincePreviousChunkMs), attributes);
+                }
                 return;
             }
             if (event.type === "tool-execution-start") {
-                spans.set(keyFor(event), observer.startSpan(`${prefix}.tool`, {
+                pushSpan(keyFor(event), observer.startSpan(toolSpanName(event.toolCall.name), {
+                    [GEN_AI.operationName]: GEN_AI_OPERATION.executeTool,
+                    [GEN_AI.toolName]: event.toolCall.name,
+                    [GEN_AI.toolCallId]: event.toolCall.id,
+                    [GEN_AI.toolType]: "function",
                     "zhivex.provider": event.model.provider,
                     "zhivex.model_id": event.model.modelId,
                     "zhivex.step": event.step,
                     "zhivex.tool_name": event.toolCall.name,
                     "zhivex.tool_call_id": event.toolCall.id
-                }));
+                }, { kind: "internal", startTime: event.startedAt }));
                 return;
             }
-            const handle = spans.get(keyFor(event));
-            spans.delete(keyFor(event));
+            const handle = shiftSpan(keyFor(event));
             if (event.type === "generate-finish") {
+                const attributes = clientMetricAttributes(event);
+                safeRecord(metrics?.clientOperationDuration, seconds(event.latencyMs), attributes);
+                recordTokenUsage(metrics, event.output.usage, attributes);
                 await handle?.end({
-                    attributes: {
-                        "zhivex.latency_ms": event.latencyMs
-                    }
+                    attributes: responseAttributes({
+                        finishReason: event.output.finishReason,
+                        providerFinishReason: event.output.providerFinishReason,
+                        usage: event.output.usage,
+                        latencyMs: event.latencyMs
+                    }),
+                    endTime: event.finishedAt
                 });
                 return;
             }
             if (event.type === "stream-finish") {
-                await handle?.end({
-                    attributes: {
-                        "zhivex.finish_reason": event.finishReason,
-                        "zhivex.provider_finish_reason": event.providerFinishReason,
-                        "zhivex.latency_ms": event.latencyMs
-                    }
-                });
+                const attributes = clientMetricAttributes(event);
+                safeRecord(metrics?.clientOperationDuration, seconds(event.latencyMs), attributes);
+                recordTokenUsage(metrics, event.usage, attributes);
+                await handle?.end({ attributes: responseAttributes(event), endTime: event.finishedAt });
                 return;
             }
             if (event.type === "tool-execution-finish") {
+                const error = event.toolResult.isError ? new Error(event.toolResult.error?.message ?? "Tool execution failed.") : undefined;
+                safeRecord(metrics?.executeToolDuration, seconds(event.latencyMs), {
+                    [GEN_AI.toolName]: event.toolCall.name,
+                    [GEN_AI.toolType]: "function",
+                    ...(error ? { "error.type": errorType(error) } : {})
+                });
                 await handle?.end({
                     attributes: {
                         "zhivex.latency_ms": event.latencyMs,
                         "zhivex.tool_error": event.toolResult.isError
-                    }
+                    },
+                    error,
+                    endTime: event.finishedAt
                 });
                 return;
             }
-            if (event.type === "generate-error" || event.type === "stream-error" || event.type === "tool-execution-error") {
+            if (event.type === "generate-error" || event.type === "stream-error") {
+                safeRecord(metrics?.clientOperationDuration, seconds(event.latencyMs), clientMetricAttributes(event, event.error));
                 await handle?.end({
                     attributes: {
-                        "zhivex.latency_ms": event.latencyMs
+                        "zhivex.latency_ms": event.latencyMs,
+                        ...(event.type === "stream-error" ? { "zhivex.output_chunk_count": event.outputChunkCount } : {})
                     },
-                    error: event.error
+                    error: event.error,
+                    endTime: event.finishedAt
+                });
+                return;
+            }
+            if (event.type === "tool-execution-error") {
+                safeRecord(metrics?.executeToolDuration, seconds(event.latencyMs), {
+                    [GEN_AI.toolName]: event.toolCall.name,
+                    [GEN_AI.toolType]: "function",
+                    "error.type": errorType(event.error)
+                });
+                await handle?.end({
+                    attributes: { "zhivex.latency_ms": event.latencyMs },
+                    error: event.error,
+                    endTime: event.finishedAt
                 });
             }
         }