@senpi/runtime 3.0.6-dev.runtime-phase-2.20260630134327 → 3.0.6-dev.runtime-phase-2.20260630134410

package/dist/tracking/spans/span-hooks.js

@@ -1,13 +1,42 @@
+/**
+ * registerSpanHooks: wires the nine OpenClaw plugin hooks (llm_input/llm_output,
+ * before/after_tool_call, the four subagent_* hooks, agent_end) to real OTel gen_ai
+ * spans via the shared `getTracer()`:
+ *   - llm_input → llm_output  → `chat` (CLIENT)
+ *   - tool_call               → `execute_tool` (INTERNAL), parented to the model span
+ *   - subagent_* lifecycle    → `invoke_agent` (INTERNAL), parented to the model span
+ *   - agent_end               → ends any still-open spans for the run
+ *
+ * Open and finalize are separate callbacks, so we can't use `withSpan`; we `startSpan`
+ * and hold the handle in a session-keyed registry until the finalize hook, recording
+ * errors directly on the held span (it's never the active span, so `logger.error` won't).
+ *
+ * Identity rides as `senpi.*` attributes from the `X-Senpi-*` headers (ALS doesn't cross
+ * the loopback fetch). The trading trace is joined by the `traceparent` header when present
+ * (nesting under `signal.process`), else by a `senpi.correlation_id` attribute. Handlers
+ * never throw.
+ */
+import { SpanKind, context as otelContext, propagation, trace, } from "@opentelemetry/api";
+import { getTracer } from "../../telemetry/tracing.js";
 import { HEADER_ADDRESS, HEADER_CORRELATION_ID, HEADER_RECIPE_NAME, HEADER_RUNTIME_ID, HEADER_RUNTIME_VERSION, } from "./constants.js";
+import { GEN_AI, errorTypeOf, firstNumber, normalizeGenAiMessages, recordSpanError, setDefinedAttributes, setGenAiContentAttribute, toFinishReasons, } from "./gen-ai.js";
+import { elapsedSeconds, recordExecuteToolDuration, recordInvokeAgentDuration, recordOperationDuration, recordTokenUsage, } from "../../telemetry/metrics.js";
 import { createLogger } from "../../utils/logger.js";
-import { sanitizePayloadValue } from "./sanitize-payload.js";
 const logger = createLogger("[senpi-core][span-hooks]");
 export function registerSpanHooks(options) {
-    const { api, tracker } = options;
-    // Tool hooks often arrive with a ctx that lacks sessionKey/runId — OpenClaw
-    // doesn't always thread the host agent's identity through to tool calls.
-    // Mirror opik's fallback strategy: remember (agentId → sessionKey/runId)
-    // from llm_input, and the last seen pair, so tool hooks can resolve them.
+    const { api } = options;
+    const tracer = getTracer();
+    // Held model spans, keyed by (sessionKey, runId). The tool/subagent spans parent
+    // off the model span for the same session/run.
+    const modelSpans = new Map();
+    // Held tool spans, keyed by (sessionKey, runId, toolCallId|toolName seq).
+    const toolSpans = new Map();
+    // Held subagent spans, keyed by childSessionKey (sessionKey/runId = the host).
+    const subagentSpans = new Map();
+    let toolSeq = 0;
+    // Tool hooks often arrive with a ctx that lacks sessionKey/runId. Remember
+    // (agentId → sessionKey/runId) from llm_input, plus the last-seen pair, so tool
+    // hooks can resolve their host session.
     const sessionByAgentId = new Map();
     let lastActive = null;
     const rememberSession = (agentCtx, sessionKey, runId) => {
@@ -32,10 +61,15 @@ export function registerSpanHooks(options) {
         }
         return lastActive;
     };
-    // ─── LLM ───────────────────────────────────────────────────────────────
+    /** Build a parent Context for tool/subagent spans from the host model span (if any). */
+    const parentContextForModel = (sessionKey, runId) => {
+        const model = modelSpans.get(sessionLlmKey(sessionKey, runId));
+        return model ? trace.setSpan(otelContext.active(), model.span) : otelContext.active();
+    };
+    // ─── LLM (`chat` span) ───────────────────────────────────────────────────
     api.on("llm_input", (event, agentCtx) => {
         try {
-            const ctx = readContext(agentCtx);
+            const identity = readIdentity(agentCtx);
             const sessionKey = readSessionKey(agentCtx, event);
             const runId = readRunId(agentCtx, event);
             if (!sessionKey || !runId) {
@@ -43,13 +77,43 @@ export function registerSpanHooks(options) {
                 return;
             }
             rememberSession(agentCtx, sessionKey, runId);
-            tracker.openLlmSpan({
+            const key = sessionLlmKey(sessionKey, runId);
+            // A duplicate llm_input for the same (session, run) means a new turn opened
+            // without the previous one closing — end the prior span before replacing it.
+            const prior = modelSpans.get(key);
+            if (prior)
+                endSpan(prior.span);
+            const ev = asRecord(event) ?? {};
+            const requestModel = nonEmptyString(ev.model);
+            const provider = nonEmptyString(ev.provider);
+            const span = tracer.startSpan(`chat ${requestModel ?? "unknown"}`, { kind: SpanKind.CLIENT }, 
+            // Parent off the trading trace when the traceparent header threaded through.
+            parentContextFromHeaders(agentCtx));
+            const held = {
+                span,
                 sessionKey,
                 runId,
-                emissionSource: "hook",
-                input: extractLlmInput(event),
-                ...ctx,
+                startMs: Date.now(),
+                provider,
+                model: requestModel,
+            };
+            modelSpans.set(key, held);
+            setDefinedAttributes(span, {
+                [GEN_AI.OPERATION_NAME]: "chat",
+                [GEN_AI.PROVIDER_NAME]: provider,
+                [GEN_AI.REQUEST_MODEL]: requestModel,
+                [GEN_AI.REQUEST_TEMPERATURE]: firstNumber(ev.temperature),
+                [GEN_AI.REQUEST_MAX_TOKENS]: firstNumber(ev.maxTokens),
+                ...identityAttributes(identity),
             });
+            // Content (redacted + capped). Stamping is async and the span ends in llm_output,
+            // so track the promise for the finalize to await — else a fast round-trip could end
+            // the span before the input content lands.
+            held.pending = Promise.allSettled([
+                setGenAiContentAttribute(span, GEN_AI.SYSTEM_INSTRUCTIONS, buildSystemInstructions(ev)),
+                setGenAiContentAttribute(span, GEN_AI.INPUT_MESSAGES, buildInputMessages(ev)),
+                setGenAiContentAttribute(span, GEN_AI.TOOL_DEFINITIONS, ev.tools),
+            ]);
         }
         catch (err) {
             logger.warn("llm_input handler failed", { err: errMsg(err) });
@@ -57,7 +121,6 @@ export function registerSpanHooks(options) {
     });
     api.on("llm_output", (event, agentCtx) => {
         try {
-            const ctx = readContext(agentCtx);
             const sessionKey = readSessionKey(agentCtx, event);
             const runId = readRunId(agentCtx, event);
             if (!sessionKey || !runId) {
@@ -65,47 +128,88 @@ export function registerSpanHooks(options) {
                 return;
             }
             rememberSession(agentCtx, sessionKey, runId);
-            const out = extractLlmOutput(event);
-            tracker.finalizeLlmSpan({
-                sessionKey,
-                runId,
-                output: out.output,
-                model: out.model,
-                tokensIn: out.tokensIn,
-                tokensOut: out.tokensOut,
-                ...ctx,
+            const held = modelSpans.get(sessionLlmKey(sessionKey, runId));
+            if (!held) {
+                logger.debug("llm_output: no open chat span", { sessionKey, runId });
+                return;
+            }
+            const span = held.span;
+            modelSpans.delete(sessionLlmKey(sessionKey, runId));
+            const ev = asRecord(event) ?? {};
+            const usage = asRecord(ev.usage);
+            // Token field naming varies by provider/wrapper; try every known spelling.
+            const inputTokens = firstNumber(usage?.input_tokens, usage?.prompt_tokens, usage?.promptTokens, usage?.input);
+            const outputTokens = firstNumber(usage?.output_tokens, usage?.completion_tokens, usage?.completionTokens, usage?.output);
+            setDefinedAttributes(span, {
+                [GEN_AI.RESPONSE_MODEL]: nonEmptyString(ev.model),
+                [GEN_AI.RESPONSE_FINISH_REASONS]: toFinishReasons(ev.finishReason ?? ev.stopReason),
+                [GEN_AI.USAGE_INPUT_TOKENS]: inputTokens,
+                [GEN_AI.USAGE_OUTPUT_TOKENS]: outputTokens,
             });
+            const error = ev.error;
+            const errored = error !== undefined && error !== null;
+            const dims = { provider: held.provider, model: held.model };
+            recordTokenUsage(dims, { input: inputTokens, output: outputTokens });
+            recordOperationDuration(elapsedSeconds(held.startMs), {
+                ...dims,
+                errorType: errored ? errorTypeOf(error) : undefined,
+            });
+            const outputMessages = buildOutputMessages(ev);
+            void (async () => {
+                try {
+                    if (held.pending)
+                        await held.pending;
+                    await setGenAiContentAttribute(span, GEN_AI.OUTPUT_MESSAGES, outputMessages);
+                    if (errored)
+                        recordSpanError(span, error);
+                }
+                finally {
+                    try {
+                        span.end();
+                    }
+                    catch {
+                        /* best effort */
+                    }
+                }
+            })();
         }
         catch (err) {
             logger.warn("llm_output handler failed", { err: errMsg(err) });
         }
     });
-    // ─── Tool ──────────────────────────────────────────────────────────────
-    // OpenClaw's tool-hook context often omits sessionKey/runId. Fall back to
-    // (agentId → session/run) learned from the most recent llm_input, then to
-    // the last-active pair, matching opik's strategy.
+    // ─── Tool (`execute_tool` span) ────────────────────────────────────────────
     api.on("before_tool_call", (event, toolCtx) => {
         try {
-            const ctx = readContext(toolCtx);
+            const identity = readIdentity(toolCtx);
             const resolved = resolveToolSession(toolCtx);
             if (!resolved) {
-                logger.debug("before_tool_call sessionKey/runId unresolved (no llm_input seen yet); skipping");
+                logger.debug("before_tool_call session unresolved (no llm_input seen yet); skipping");
                 return;
             }
-            const evObj = asRecord(event);
-            const toolName = nonEmptyString(evObj?.toolName) ?? nonEmptyString(evObj?.name) ?? "unknown";
-            const toolCallId = nonEmptyString(evObj?.toolCallId) ??
+            const ev = asRecord(event);
+            const toolName = nonEmptyString(ev?.toolName) ?? nonEmptyString(ev?.name) ?? "unknown";
+            const toolCallId = nonEmptyString(ev?.toolCallId) ??
                 nonEmptyString(asRecord(toolCtx)?.toolCallId) ??
                 null;
-            tracker.openToolSpan({
+            const key = toolKey(resolved.sessionKey, resolved.runId, toolCallId, toolName, ++toolSeq);
+            const span = tracer.startSpan(`execute_tool ${toolName}`, { kind: SpanKind.INTERNAL }, parentContextForModel(resolved.sessionKey, resolved.runId));
+            const held = {
+                span,
                 sessionKey: resolved.sessionKey,
                 runId: resolved.runId,
-                emissionSource: "hook",
-                toolCallId,
+                startMs: Date.now(),
                 toolName,
-                toolInput: asRecord(evObj?.params) ?? asRecord(evObj?.input),
-                ...ctx,
+            };
+            toolSpans.set(key, held);
+            setDefinedAttributes(span, {
+                [GEN_AI.OPERATION_NAME]: "execute_tool",
+                [GEN_AI.TOOL_NAME]: toolName,
+                [GEN_AI.TOOL_CALL_ID]: toolCallId ?? undefined,
+                ...identityAttributes(identity),
             });
+            // Args stamping is async; the finalize (after_tool_call) awaits this so a fast
+            // tool doesn't end the span before the args land.
+            held.pending = setGenAiContentAttribute(span, GEN_AI.TOOL_CALL_ARGUMENTS, ev?.params ?? ev?.input);
         }
         catch (err) {
             logger.warn("before_tool_call handler failed", { err: errMsg(err) });
@@ -113,69 +217,93 @@ export function registerSpanHooks(options) {
     });
     api.on("after_tool_call", (event, toolCtx) => {
         try {
-            const ctx = readContext(toolCtx);
             const resolved = resolveToolSession(toolCtx);
             if (!resolved) {
-                logger.debug("after_tool_call sessionKey/runId unresolved (no llm_input seen yet); skipping");
+                logger.debug("after_tool_call session unresolved; skipping");
                 return;
             }
-            const evObj = asRecord(event);
-            const toolName = nonEmptyString(evObj?.toolName) ?? nonEmptyString(evObj?.name) ?? "unknown";
-            const toolCallId = nonEmptyString(evObj?.toolCallId) ??
+            const ev = asRecord(event);
+            const toolName = nonEmptyString(ev?.toolName) ?? nonEmptyString(ev?.name) ?? "unknown";
+            const toolCallId = nonEmptyString(ev?.toolCallId) ??
                 nonEmptyString(asRecord(toolCtx)?.toolCallId) ??
                 null;
-            const error = evObj?.error;
-            const result = evObj?.result;
-            const durationMs = typeof evObj?.durationMs === "number" ? evObj.durationMs : undefined;
-            tracker.finalizeToolSpan({
-                sessionKey: resolved.sessionKey,
-                runId: resolved.runId,
-                toolCallId,
-                toolName,
-                output: error !== undefined ? undefined : result,
-                error: error !== undefined ? error : undefined,
-                durationMs,
-                ...ctx,
+            const foundKey = findToolKey(toolSpans, resolved.sessionKey, resolved.runId, toolCallId, toolName);
+            if (!foundKey) {
+                logger.debug("after_tool_call: no open tool span", { toolName, toolCallId });
+                return;
+            }
+            const held = toolSpans.get(foundKey);
+            const span = held.span;
+            toolSpans.delete(foundKey);
+            const error = ev?.error;
+            const errored = error !== undefined && error !== null;
+            const result = ev?.result;
+            recordExecuteToolDuration(elapsedSeconds(held.startMs), {
+                toolName: held.toolName,
+                errorType: errored ? errorTypeOf(error) : undefined,
             });
+            void (async () => {
+                try {
+                    if (held.pending)
+                        await held.pending;
+                    if (errored) {
+                        recordSpanError(span, error);
+                    }
+                    else {
+                        await setGenAiContentAttribute(span, GEN_AI.TOOL_CALL_RESULT, result);
+                    }
+                }
+                finally {
+                    try {
+                        span.end();
+                    }
+                    catch {
+                        /* best effort */
+                    }
+                }
+            })();
         }
         catch (err) {
             logger.warn("after_tool_call handler failed", { err: errMsg(err) });
         }
     });
-    // ─── Subagent (four hooks) ─────────────────────────────────────────────
+    // ─── Subagent (`invoke_agent` span, four-hook lifecycle) ───────────────────
     api.on("subagent_spawning", (event, subagentCtx) => {
         try {
-            const ctx = readContext(subagentCtx);
-            const evObj = asRecord(event);
+            const identity = readIdentity(subagentCtx);
+            const ev = asRecord(event);
             const ctxObj = asRecord(subagentCtx);
             const hostSessionKey = nonEmptyString(ctxObj?.requesterSessionKey) ??
                 nonEmptyString(ctxObj?.sessionKey) ??
                 "";
-            const hostRunId = nonEmptyString(ctxObj?.runId) ??
-                nonEmptyString(ctxObj?.requesterRunId) ??
-                "";
-            const childSessionKey = nonEmptyString(evObj?.childSessionKey) ??
-                nonEmptyString(ctxObj?.childSessionKey);
+            const hostRunId = nonEmptyString(ctxObj?.runId) ?? nonEmptyString(ctxObj?.requesterRunId) ?? "";
+            const childSessionKey = nonEmptyString(ev?.childSessionKey) ?? nonEmptyString(ctxObj?.childSessionKey);
             if (!childSessionKey) {
                 logger.warn("subagent_spawning missing childSessionKey; skipping span");
                 return;
             }
-            const parentSpanId = hostSessionKey && hostRunId
-                ? tracker.getActiveLlmSpanId(hostSessionKey, hostRunId)
-                : null;
-            tracker.openSubagentSpan({
-                hostSessionKey,
-                hostRunId,
-                childSessionKey,
-                parentSpanId,
-                input: {
-                    agentId: evObj?.agentId,
-                    label: evObj?.label,
-                    mode: evObj?.mode,
-                    requester: evObj?.requester,
-                    threadRequested: evObj?.threadRequested,
-                },
-                ...ctx,
+            const prior = subagentSpans.get(childSessionKey);
+            if (prior)
+                endSpan(prior.span);
+            const agentName = nonEmptyString(ev?.label) ?? nonEmptyString(ev?.agentId) ?? "unknown";
+            const span = tracer.startSpan(`invoke_agent ${agentName}`, { kind: SpanKind.INTERNAL }, hostSessionKey && hostRunId
+                ? parentContextForModel(hostSessionKey, hostRunId)
+                : otelContext.active());
+            // sessionKey/runId carry the HOST run so the agent_end sweep can reach this span.
+            subagentSpans.set(childSessionKey, {
+                span,
+                sessionKey: hostSessionKey,
+                runId: hostRunId,
+                startMs: Date.now(),
+                agentName,
+            });
+            setDefinedAttributes(span, {
+                [GEN_AI.OPERATION_NAME]: "invoke_agent",
+                [GEN_AI.AGENT_NAME]: agentName,
+                "senpi.subagent.mode": nonEmptyString(ev?.mode),
+                "senpi.subagent.requester": nonEmptyString(ev?.requester),
+                "senpi.subagent.thread_requested": typeof ev?.threadRequested === "boolean" ? ev.threadRequested : undefined,
+                ...identityAttributes(identity),
             });
         }
         catch (err) {
@@ -184,23 +312,20 @@ export function registerSpanHooks(options) {
     });
     api.on("subagent_spawned", (event, subagentCtx) => {
         try {
-            const ctx = readContext(subagentCtx);
-            const evObj = asRecord(event);
+            const ev = asRecord(event);
             const ctxObj = asRecord(subagentCtx);
-            const childSessionKey = nonEmptyString(evObj?.childSessionKey) ??
-                nonEmptyString(ctxObj?.childSessionKey);
+            const childSessionKey = nonEmptyString(ev?.childSessionKey) ?? nonEmptyString(ctxObj?.childSessionKey);
             if (!childSessionKey)
                 return;
-            tracker.mergeSubagentSpan({
-                childSessionKey,
-                status: "spawned",
-                patch: {
-                    runId: nonEmptyString(evObj?.runId) ?? nonEmptyString(ctxObj?.runId),
-                    agentId: evObj?.agentId,
-                    mode: evObj?.mode,
-                    threadRequested: evObj?.threadRequested,
-                },
-            }, ctx);
+            const held = subagentSpans.get(childSessionKey);
+            if (!held)
+                return;
+            const span = held.span;
+            setDefinedAttributes(span, {
+                "senpi.subagent.run_id": nonEmptyString(ev?.runId) ?? nonEmptyString(ctxObj?.runId),
+                "senpi.subagent.mode": nonEmptyString(ev?.mode),
+                "senpi.subagent.thread_requested": typeof ev?.threadRequested === "boolean" ? ev.threadRequested : undefined,
+            });
         }
         catch (err) {
             logger.warn("subagent_spawned handler failed", { err: errMsg(err) });
@@ -208,31 +333,23 @@ export function registerSpanHooks(options) {
     });
     api.on("subagent_delivery_target", (event, subagentCtx) => {
         try {
-            const ctx = readContext(subagentCtx);
-            const evObj = asRecord(event);
+            const ev = asRecord(event);
             const ctxObj = asRecord(subagentCtx);
-            const childSessionKey = nonEmptyString(evObj?.childSessionKey) ??
-                nonEmptyString(ctxObj?.childSessionKey);
+            const childSessionKey = nonEmptyString(ev?.childSessionKey) ?? nonEmptyString(ctxObj?.childSessionKey);
             if (!childSessionKey)
                 return;
-            const requesterOrigin = asRecord(evObj?.requesterOrigin) ?? {};
-            tracker.mergeSubagentSpan({
-                childSessionKey,
-                status: "delivery_target",
-                patch: {
-                    delivery: {
-                        channel: nonEmptyString(requesterOrigin.channel),
-                        accountId: nonEmptyString(requesterOrigin.accountId),
-                        to: nonEmptyString(requesterOrigin.to),
-                        threadId: requesterOrigin.threadId,
-                        childRunId: nonEmptyString(evObj?.childRunId),
-                        spawnMode: nonEmptyString(evObj?.spawnMode),
-                        expectsCompletionMessage: typeof evObj?.expectsCompletionMessage === "boolean"
-                            ? evObj.expectsCompletionMessage
-                            : undefined,
-                    },
-                },
-            }, ctx);
+            const held = subagentSpans.get(childSessionKey);
+            if (!held)
+                return;
+            const span = held.span;
+            const origin = asRecord(ev?.requesterOrigin) ?? {};
+            setDefinedAttributes(span, {
+                "senpi.subagent.delivery.channel": nonEmptyString(origin.channel),
+                "senpi.subagent.delivery.account_id": nonEmptyString(origin.accountId),
+                "senpi.subagent.delivery.to": nonEmptyString(origin.to),
+                "senpi.subagent.delivery.child_run_id": nonEmptyString(ev?.childRunId),
+                "senpi.subagent.delivery.spawn_mode": nonEmptyString(ev?.spawnMode),
+            });
         }
         catch (err) {
             logger.warn("subagent_delivery_target handler failed", { err: errMsg(err) });
@@ -240,79 +357,192 @@ export function registerSpanHooks(options) {
     });
     api.on("subagent_ended", (event, subagentCtx) => {
         try {
-            const ctx = readContext(subagentCtx);
-            const evObj = asRecord(event);
+            const ev = asRecord(event);
             const ctxObj = asRecord(subagentCtx);
-            const childSessionKey = nonEmptyString(evObj?.targetSessionKey) ??
+            const childSessionKey = nonEmptyString(ev?.targetSessionKey) ??
                 nonEmptyString(ctxObj?.childSessionKey) ??
-                nonEmptyString(evObj?.childSessionKey);
+                nonEmptyString(ev?.childSessionKey);
             if (!childSessionKey)
                 return;
-            const error = evObj?.error;
-            tracker.finalizeSubagentSpan({
-                childSessionKey,
-                status: "ended",
-                output: error === undefined
-                    ? {
-                        outcome: evObj?.outcome,
-                        reason: evObj?.reason,
-                    }
-                    : undefined,
-                error: error !== undefined ? error : undefined,
-            }, ctx);
+            const held = subagentSpans.get(childSessionKey);
+            if (!held)
+                return;
+            const span = held.span;
+            subagentSpans.delete(childSessionKey);
+            const error = ev?.error;
+            const errored = error !== undefined && error !== null;
+            recordInvokeAgentDuration(elapsedSeconds(held.startMs), {
+                agentName: held.agentName,
+                errorType: errored ? errorTypeOf(error) : undefined,
+            });
+            if (errored) {
+                recordSpanError(span, error);
+            }
+            else {
+                setDefinedAttributes(span, {
+                    "senpi.subagent.outcome": nonEmptyString(ev?.outcome),
+                    "senpi.subagent.reason": nonEmptyString(ev?.reason),
+                });
+            }
+            try {
+                span.end();
+            }
+            catch {
+                /* best effort */
+            }
         }
         catch (err) {
             logger.warn("subagent_ended handler failed", { err: errMsg(err) });
         }
     });
-    // ─── Agent end (sweep orphans + emit marker) ───────────────────────────
+    // ─── Agent end (orphan sweep) ──────────────────────────────────────────────
     api.on("agent_end", (event, agentCtx) => {
         try {
-            const ctx = readContext(agentCtx);
             const sessionKey = readSessionKey(agentCtx, event);
             const runId = readRunId(agentCtx, event);
             if (!sessionKey || !runId) {
-                logger.warn("agent_end missing sessionKey/runId; skipping marker");
+                logger.warn("agent_end missing sessionKey/runId; skipping sweep");
                 return;
             }
-            const evObj = asRecord(event);
-            tracker.captureAgentEnd({
-                sessionKey,
-                runId,
-                outcome: evObj?.outcome,
-                error: evObj?.error,
-                ...ctx,
-            });
+            const ev = asRecord(event);
+            const outcomeError = ev?.error;
+            // End any span still open under this (session, run): the model span, its tool
+            // children, and any subagents it hosted. Matching is by the stored host
+            // session/run, so nothing leaks past the run's end (incl. subagents, which have
+            // no other guaranteed terminator).
+            const llmKey = sessionLlmKey(sessionKey, runId);
+            const model = modelSpans.get(llmKey);
+            if (model) {
+                modelSpans.delete(llmKey);
+                const errored = outcomeError !== undefined && outcomeError !== null;
+                recordOperationDuration(elapsedSeconds(model.startMs), {
+                    provider: model.provider,
+                    model: model.model,
+                    errorType: errored ? errorTypeOf(outcomeError) : undefined,
+                });
+                if (errored)
+                    recordSpanError(model.span, outcomeError);
+                endSpan(model.span);
+            }
+            for (const [k, held] of toolSpans) {
+                if (held.sessionKey === sessionKey && held.runId === runId) {
+                    toolSpans.delete(k);
+                    endSpan(held.span);
+                }
+            }
+            for (const [k, held] of subagentSpans) {
+                if (held.sessionKey === sessionKey && held.runId === runId) {
+                    subagentSpans.delete(k);
+                    endSpan(held.span);
+                }
+            }
         }
         catch (err) {
             logger.warn("agent_end handler failed", { err: errMsg(err) });
         }
     });
 }
-// ─── Helpers ──────────────────────────────────────────────────────────────
-function asRecord(v) {
-    if (v && typeof v === "object" && !Array.isArray(v)) {
-        return v;
+// ─── Span key helpers ───────────────────────────────────────────────────────
+const SEP = " ";
+function sessionLlmKey(sessionKey, runId) {
+    return `${sessionKey}${SEP}${runId}${SEP}llm`;
+}
+function toolKey(sessionKey, runId, toolCallId, toolName, seq) {
+    return toolCallId
+        ? `${sessionKey}${SEP}${runId}${SEP}toolcall${SEP}${toolCallId}`
+        : `${sessionKey}${SEP}${runId}${SEP}tool${SEP}${toolName}${SEP}${seq}`;
+}
+/** Find the open tool span key within one (session, run): by toolCallId first, else by tool name. */
+function findToolKey(toolSpans, sessionKey, runId, toolCallId, toolName) {
+    const prefix = `${sessionKey}${SEP}${runId}${SEP}`;
+    if (toolCallId) {
+        const suffix = `${SEP}toolcall${SEP}${toolCallId}`;
+        for (const key of toolSpans.keys()) {
+            if (key.startsWith(prefix) && key.endsWith(suffix))
+                return key;
+        }
+    }
+    const namePart = `${SEP}tool${SEP}${toolName}${SEP}`;
+    for (const key of toolSpans.keys()) {
+        if (key.startsWith(prefix) && key.includes(namePart))
+            return key;
     }
     return undefined;
 }
-function nonEmptyString(v) {
-    return typeof v === "string" && v.length > 0 ? v : undefined;
+function endSpan(span) {
+    try {
+        span.end();
+    }
+    catch {
+        /* best effort */
+    }
+}
+// ─── Identity + trace correlation ─────────────────────────────────────────────
+const ADDRESS_RE = /^0x[a-f0-9]{40}$/i;
+const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+/** Cap on a free-form header value stamped as an attribute (defence vs oversized header). */
+const MAX_HEADER_ATTR_CHARS = 256;
+function clampHeader(v) {
+    return v.length > MAX_HEADER_ATTR_CHARS ? v.slice(0, MAX_HEADER_ATTR_CHARS) : v;
 }
 /**
- * Pull senpi-context headers out of agentCtx. Several common locations are
- * checked because the exact shape depends on OpenClaw's runtime extension SDK.
+ * Map the senpi identity headers to `senpi.*` span attributes (keys match the on-site
+ * identity bag; the headers carry the runtime id as `senpi.runtime.id`, vs on-site
+ * `senpi.runtime`). Headers are caller-influenceable, so join keys are validated — a
+ * malformed address / correlation id is dropped, not stamped as garbage.
  */
-function readContext(agentCtx) {
+function identityAttributes(identity) {
+    const attrs = {};
+    if (identity.address) {
+        const lowered = identity.address.toLowerCase();
+        if (ADDRESS_RE.test(lowered))
+            attrs["senpi.strategy.address"] = lowered;
+    }
+    if (identity.recipeName)
+        attrs["senpi.recipe"] = clampHeader(identity.recipeName);
+    if (identity.runtimeVersion)
+        attrs["senpi.recipe.version"] = clampHeader(identity.runtimeVersion);
+    if (identity.runtimeId)
+        attrs["senpi.runtime.id"] = clampHeader(identity.runtimeId);
+    // Fallback join key to the trading trace when traceparent didn't thread through.
+    if (identity.correlationId && UUID_RE.test(identity.correlationId)) {
+        attrs["senpi.correlation_id"] = identity.correlationId;
+    }
+    return attrs;
+}
+function readIdentity(agentCtx) {
     const headers = readHeaders(agentCtx);
     return {
-        correlationId: headers[HEADER_CORRELATION_ID] ?? null,
-        address: headers[HEADER_ADDRESS] ?? null,
-        recipeName: headers[HEADER_RECIPE_NAME] ?? null,
-        runtimeId: headers[HEADER_RUNTIME_ID] ?? null,
-        runtimeVersion: headers[HEADER_RUNTIME_VERSION] ?? null,
+        correlationId: headers[HEADER_CORRELATION_ID],
+        address: headers[HEADER_ADDRESS],
+        recipeName: headers[HEADER_RECIPE_NAME],
+        runtimeId: headers[HEADER_RUNTIME_ID],
+        runtimeVersion: headers[HEADER_RUNTIME_VERSION],
     };
 }
+/** Extract a W3C traceparent from the request headers → a parent Context (else active). */
+function parentContextFromHeaders(agentCtx) {
+    try {
+        const headers = readHeaders(agentCtx);
+        if (headers.traceparent === undefined)
+            return otelContext.active();
+        return propagation.extract(otelContext.active(), headers);
+    }
+    catch {
+        return otelContext.active();
+    }
+}
+// ─── Field readers ────────────────────────────────────────────────────────────
+function asRecord(v) {
+    if (v && typeof v === "object" && !Array.isArray(v)) {
+        return v;
+    }
+    return undefined;
+}
+function nonEmptyString(v) {
+    return typeof v === "string" && v.length > 0 ? v : undefined;
+}
+/** Pull request headers out of agentCtx; several common locations are checked. */
 function readHeaders(agentCtx) {
     const ctxObj = asRecord(agentCtx);
     if (!ctxObj)
@@ -328,7 +558,6 @@ function readHeaders(agentCtx) {
         if (!candidate)
             continue;
         for (const [k, v] of Object.entries(candidate)) {
-            // HTTP headers should be case-insensitive; normalize to lowercase.
             const lk = k.toLowerCase();
             if (typeof v === "string" && v.length > 0 && out[lk] === undefined) {
                 out[lk] = v;
@@ -347,61 +576,32 @@ function readSessionKey(ctx, event) {
         nonEmptyString(asRecord(event)?.sessionId));
 }
 function readRunId(ctx, event) {
-    return (nonEmptyString(asRecord(ctx)?.runId) ??
-        nonEmptyString(asRecord(event)?.runId));
+    return nonEmptyString(asRecord(ctx)?.runId) ?? nonEmptyString(asRecord(event)?.runId);
 }
-function extractLlmInput(event) {
-    const ev = asRecord(event);
-    if (!ev)
-        return {};
-    // Strip OpenClaw framework metadata blocks (untrusted-context, sender info,
-    // reply_to markers, media refs) before stamping the payload onto the span.
-    // Without this, every llm_input on a chat session balloons past 1MB on the
-    // back of injected context and the real user prompt gets buried.
-    return {
-        system: sanitizePayloadValue(ev.systemPrompt ?? ev.system),
-        prompt: sanitizePayloadValue(ev.prompt),
-        messages: sanitizePayloadValue(ev.historyMessages ?? ev.messages),
-        model: ev.model,
-        provider: ev.provider,
-        imagesCount: ev.imagesCount,
-        temperature: ev.temperature,
-        maxTokens: ev.maxTokens,
-        tools: ev.tools,
-    };
+// ─── gen_ai content builders ──────────────────────────────────────────────────
+/** System instructions as `[{type:"text", content}]` per the gen_ai shape. */
+function buildSystemInstructions(ev) {
+    const system = ev.systemPrompt ?? ev.system;
+    if (typeof system !== "string" || system.length === 0)
+        return undefined;
+    return [{ type: "text", content: system }];
 }
-function extractLlmOutput(event) {
-    const ev = asRecord(event);
-    if (!ev)
-        return { model: null, tokensIn: null, tokensOut: null };
-    const usage = asRecord(ev.usage);
-    return {
-        output: {
-            assistantTexts: sanitizePayloadValue(ev.assistantTexts),
-            lastAssistant: sanitizePayloadValue(ev.lastAssistant),
-            content: sanitizePayloadValue(ev.content),
-            finishReason: ev.finishReason ?? ev.stopReason,
-        },
-        model: nonEmptyString(ev.model) ?? null,
-        // Token field naming varies by provider and by how OpenClaw shapes the
-        // usage object. OpenClaw normalizes most hook-path traffic to `input` /
-        // `output` (see normalizeUsage in session-cost-usage-*.js), but raw
-        // SDK shapes leak through on edge paths and direct-SDK callers. We try
-        // every known spelling so the count is captured regardless:
-        //   Anthropic SDK      → input_tokens / output_tokens
-        //   OpenAI SDK         → prompt_tokens / completion_tokens   (snake_case is canonical)
-        //   Wrapper layers     → promptTokens / completionTokens     (some abstractions)
-        //   OpenClaw / Gemini  → input / output                       (normalized)
-        tokensIn: firstNumber(usage?.input_tokens, usage?.prompt_tokens, usage?.promptTokens, usage?.input),
-        tokensOut: firstNumber(usage?.output_tokens, usage?.completion_tokens, usage?.completionTokens, usage?.output),
-    };
-}
-function firstNumber(...values) {
-    for (const v of values) {
-        if (typeof v === "number" && Number.isFinite(v))
-            return v;
+/** Input messages: the history (or messages), with any standalone `prompt` as a user turn. */
+function buildInputMessages(ev) {
+    const history = ev.historyMessages ?? ev.messages;
+    const messages = Array.isArray(history) ? [...history] : history ? [history] : [];
+    if (typeof ev.prompt === "string" && ev.prompt.length > 0) {
+        messages.push({ role: "user", content: ev.prompt });
     }
-    return null;
+    return normalizeGenAiMessages(messages, "user");
+}
+/** Output messages: the assistant text(s) / content, with the finish reason on the message. */
+function buildOutputMessages(ev) {
+    const content = ev.assistantTexts ?? ev.lastAssistant ?? ev.content;
+    if (content === undefined || content === null)
+        return undefined;
+    const finish = ev.finishReason ?? ev.stopReason;
+    return normalizeGenAiMessages([{ role: "assistant", content, ...(typeof finish === "string" ? { finish_reason: finish } : {}) }], "assistant");
 }
 function errMsg(err) {
     return err instanceof Error ? err.message : String(err);