@senpi/runtime 3.0.6-dev.skill-manager-v2.20260629194505 → 3.0.9-dev.runtime-phase-2.20260707070820

package/dist/telemetry/spans/span-hooks.js

@@ -0,0 +1,764 @@
+/**
+ * registerSpanHooks: wires the OpenClaw plugin hooks (llm_input/llm_output,
+ * model_call_ended, before/after_tool_call, the four subagent_* hooks, agent_end) to real
+ * OTel gen_ai spans via the shared `getTracer()`:
+ *   - llm_input → llm_output  → `chat` (CLIENT)
+ *   - model_call_ended        → outcome/failure on the `chat` span (the only failed-turn signal)
+ *   - tool_call               → `execute_tool` (INTERNAL), parented to the model span
+ *   - subagent_* lifecycle    → `invoke_agent` (INTERNAL), parented to the model span
+ *   - agent_end               → ends any still-open spans for the run
+ *
+ * Open and finalize are separate callbacks, so we can't use `withSpan`; we `startSpan`
+ * and hold the handle in a session-keyed registry until the finalize hook, recording
+ * errors directly on the held span (it's never the active span, so `logger.error` won't).
+ *
+ * Identity rides as `senpi.*` attributes from the `X-Senpi-*` headers (ALS doesn't cross
+ * the loopback fetch). The trading trace is joined by the `traceparent` header when present
+ * (nesting under `signal.process`), else by a `senpi.run.id` attribute. Handlers
+ * never throw.
+ */
+import { SpanKind, context as otelContext, propagation, trace, } from "@opentelemetry/api";
+import { getTracer } from "../../telemetry/tracing.js";
+import { HEADER_ADDRESS, HEADER_CORRELATION_ID, HEADER_RECIPE_NAME, HEADER_RUNTIME_ID, HEADER_RUNTIME_VERSION, } from "./constants.js";
+import { GEN_AI, errorTypeOf, firstNumber, normalizeGenAiMessages, recordSpanError, setDefinedAttributes, setGenAiContentAttribute, toFinishReasons, } from "./gen-ai.js";
+import { elapsedSeconds, recordToolCall, recordInvokeAgentDuration, recordOperationDuration, recordTokenUsage, } from "../../telemetry/metrics.js";
+import { emitToolCalled, offeredToolNames, requestedToolNames, } from "../../telemetry/tool-plane.js";
+import { createLogger } from "../../utils/logger.js";
+const logger = createLogger("[senpi-core][span-hooks]");
+/**
+ * Grace window the `agent_end` sweep holds a finished model span before ending it, so a
+ * trailing `llm_output` (observed ~1 ms later in the codex/harness flow, but timed generously)
+ * can finalize it with token usage + output messages first. Unref'd, so it never holds the
+ * process open; an orphan run with no `llm_output` lets the timer end the span.
+ */
+export const MODEL_SPAN_END_GRACE_MS = 2000;
+export function registerSpanHooks(options) {
+    const { api } = options;
+    const tracer = getTracer();
+    // Cap on drift events per turn so a flood of invented tool calls can't balloon the log.
+    const MAX_DRIFT_EVENTS_PER_TURN = 8;
+    // Held model spans, keyed by (sessionKey, runId). The tool/subagent spans parent
+    // off the model span for the same session/run.
+    const modelSpans = new Map();
+    // Held tool spans, keyed by (sessionKey, runId, toolCallId|toolName seq).
+    const toolSpans = new Map();
+    // Held subagent spans, keyed by childSessionKey (sessionKey/runId = the host).
+    const subagentSpans = new Map();
+    let toolSeq = 0;
+    // Tool hooks often arrive with a ctx that lacks sessionKey/runId. Remember
+    // (agentId → sessionKey/runId) from llm_input, plus the last-seen pair, so tool
+    // hooks can resolve their host session.
+    const sessionByAgentId = new Map();
+    let lastActive = null;
+    const rememberSession = (agentCtx, sessionKey, runId) => {
+        const agentId = nonEmptyString(asRecord(agentCtx)?.agentId);
+        if (agentId)
+            sessionByAgentId.set(agentId, { sessionKey, runId });
+        lastActive = { sessionKey, runId };
+    };
+    const resolveToolSession = (ctx) => {
+        const direct = {
+            sessionKey: readSessionKey(ctx, undefined),
+            runId: readRunId(ctx, undefined),
+        };
+        if (direct.sessionKey && direct.runId) {
+            return { sessionKey: direct.sessionKey, runId: direct.runId };
+        }
+        const agentId = nonEmptyString(asRecord(ctx)?.agentId);
+        if (agentId) {
+            const hit = sessionByAgentId.get(agentId);
+            if (hit)
+                return hit;
+        }
+        // Last resort: the most-recent session. Only safe when unambiguous — two or more
+        // model spans open at once means concurrent runs share this process and `lastActive`
+        // could belong to either, so drop the span rather than misattribute it across runs.
+        return modelSpans.size <= 1 ? lastActive : null;
+    };
+    /** Build a parent Context for tool/subagent spans from the host model span (if any). */
+    const parentContextForModel = (sessionKey, runId) => {
+        const model = modelSpans.get(sessionLlmKey(sessionKey, runId));
+        return model ? trace.setSpan(otelContext.active(), model.span) : otelContext.active();
+    };
+    // ─── LLM (`chat` span) ───────────────────────────────────────────────────
+    api.on("llm_input", (event, agentCtx) => {
+        try {
+            const identity = readIdentity(agentCtx);
+            const sessionKey = readSessionKey(agentCtx, event);
+            const runId = readRunId(agentCtx, event);
+            if (!sessionKey || !runId) {
+                logger.warn("llm_input missing sessionKey/runId; skipping span");
+                return;
+            }
+            rememberSession(agentCtx, sessionKey, runId);
+            const key = sessionLlmKey(sessionKey, runId);
+            // A duplicate llm_input for the same (session, run) means a new turn opened
+            // without the previous one closing — end the prior span before replacing it.
+            const prior = modelSpans.get(key);
+            if (prior)
+                endSpan(prior.span);
+            const ev = asRecord(event) ?? {};
+            const requestModel = nonEmptyString(ev.model);
+            const provider = nonEmptyString(ev.provider);
+            const span = tracer.startSpan(`chat ${requestModel ?? "unknown"}`, { kind: SpanKind.CLIENT }, 
+            // Parent off the trading trace when the traceparent header threaded through.
+            parentContextFromHeaders(agentCtx));
+            const held = {
+                span,
+                sessionKey,
+                runId,
+                startMs: Date.now(),
+                provider,
+                model: requestModel,
+                // Offered tools — the baseline for the llm_output drift diff. Set only when definitions
+                // were delivered (undefined → no diff, i.e. conversation access off); an empty set is
+                // authoritative (zero tools offered, so any requested tool is drift).
+                offeredTools: Array.isArray(ev.tools) ? offeredToolNames(ev.tools) : undefined,
+            };
+            modelSpans.set(key, held);
+            setDefinedAttributes(span, {
+                [GEN_AI.OPERATION_NAME]: "chat",
+                // Session key as the conversation id, so consecutive turns of one session correlate (a
+                // single per-turn trace otherwise can't be tied to its neighbours — e.g. a runaway loop).
+                [GEN_AI.CONVERSATION_ID]: sessionKey,
+                [GEN_AI.PROVIDER_NAME]: provider,
+                [GEN_AI.REQUEST_MODEL]: requestModel,
+                [GEN_AI.REQUEST_TEMPERATURE]: firstNumber(ev.temperature),
+                [GEN_AI.REQUEST_MAX_TOKENS]: firstNumber(ev.maxTokens),
+                ...identityAttributes(identity),
+            });
+            // Content (redacted + capped). Stamping is async and the span ends in llm_output,
+            // so track the promise for the finalize to await — else a fast round-trip could end
+            // the span before the input content lands.
+            held.pending = Promise.allSettled([
+                setGenAiContentAttribute(span, GEN_AI.SYSTEM_INSTRUCTIONS, buildSystemInstructions(ev)),
+                setGenAiContentAttribute(span, GEN_AI.INPUT_MESSAGES, buildInputMessages(ev)),
+                setGenAiContentAttribute(span, GEN_AI.TOOL_DEFINITIONS, ev.tools),
+            ]);
+        }
+        catch (err) {
+            logger.warn("llm_input handler failed", { err: errMsg(err) });
+        }
+    });
+    api.on("llm_output", (event, agentCtx) => {
+        try {
+            const sessionKey = readSessionKey(agentCtx, event);
+            const runId = readRunId(agentCtx, event);
+            if (!sessionKey || !runId) {
+                logger.warn("llm_output missing sessionKey/runId; skipping span");
+                return;
+            }
+            rememberSession(agentCtx, sessionKey, runId);
+            const held = modelSpans.get(sessionLlmKey(sessionKey, runId));
+            if (!held) {
+                logger.debug("llm_output: no open chat span", { sessionKey, runId });
+                return;
+            }
+            const span = held.span;
+            modelSpans.delete(sessionLlmKey(sessionKey, runId));
+            // Cancel any deferred teardown the agent_end sweep scheduled — we finalize the span here.
+            if (held.endTimer)
+                clearTimeout(held.endTimer);
+            const ev = asRecord(event) ?? {};
+            const usage = asRecord(ev.usage);
+            // Token field naming varies by provider/wrapper; try every known spelling.
+            const inputTokens = firstNumber(usage?.input_tokens, usage?.prompt_tokens, usage?.promptTokens, usage?.input);
+            const outputTokens = firstNumber(usage?.output_tokens, usage?.completion_tokens, usage?.completionTokens, usage?.output);
+            // Cache + total tokens quantify context growth — the signal a silent compaction otherwise hides.
+            const cacheReadTokens = firstNumber(usage?.cacheRead, usage?.cache_read_input_tokens);
+            const cacheWriteTokens = firstNumber(usage?.cacheWrite, usage?.cache_creation_input_tokens);
+            const totalTokens = firstNumber(usage?.total, usage?.total_tokens);
+            setDefinedAttributes(span, {
+                [GEN_AI.RESPONSE_MODEL]: nonEmptyString(ev.model),
+                [GEN_AI.USAGE_INPUT_TOKENS]: inputTokens,
+                [GEN_AI.USAGE_OUTPUT_TOKENS]: outputTokens,
+                [GEN_AI.USAGE_CACHE_READ_TOKENS]: cacheReadTokens,
+                [GEN_AI.USAGE_CACHE_WRITE_TOKENS]: cacheWriteTokens,
+                [GEN_AI.USAGE_TOTAL_TOKENS]: totalTokens,
+            });
+            // A failed turn never reaches llm_output; outcome/finish is sourced from
+            // model_call_ended, so this hook only carries the success-path content + usage.
+            const dims = { provider: held.provider, model: held.model };
+            recordTokenUsage(dims, {
+                input: inputTokens,
+                output: outputTokens,
+                cacheRead: cacheReadTokens,
+                cacheWrite: cacheWriteTokens,
+            });
+            recordOperationDuration(elapsedSeconds(held.startMs), dims);
+            const outputMessages = buildOutputMessages(ev);
+            // Drift alarm: a tool the model requested but was never offered (requested − offered) is a
+            // coverage/contract hole — surface each invented name as an `unknown`-outcome event for the
+            // agent to read. Runs only when the offered set was captured (i.e. conversation access is on);
+            // deduped + capped so a flood of invented calls can't balloon the log.
+            if (held.offeredTools) {
+                const offered = held.offeredTools;
+                const seen = new Set();
+                for (const name of requestedToolNames(outputMessages)) {
+                    if (offered.has(name) || seen.has(name))
+                        continue;
+                    if (seen.size >= MAX_DRIFT_EVENTS_PER_TURN)
+                        break;
+                    seen.add(name);
+                    emitToolCalled({ name, origin: "agent", outcome: "unknown" });
+                    // Count the drift on the metric plane (no duration — the invented call never ran).
+                    recordToolCall({ toolName: name, origin: "agent", outcome: "unknown" });
+                }
+            }
+            void (async () => {
+                try {
+                    if (held.pending)
+                        await held.pending;
+                    await setGenAiContentAttribute(span, GEN_AI.OUTPUT_MESSAGES, outputMessages);
+                }
+                finally {
+                    try {
+                        span.end();
+                    }
+                    catch {
+                        /* best effort */
+                    }
+                }
+            })();
+        }
+        catch (err) {
+            logger.warn("llm_output handler failed", { err: errMsg(err) });
+        }
+    });
+    // ─── Model-call outcome (enriches the held `chat` span) ────────────────────
+    // The provider-call result: `outcome`, `failureKind`, timing. This is the ONLY
+    // signal for a failed/aborted turn — `llm_output` never fires when the call
+    // errors, so without this a wedged turn is indistinguishable from a healthy one.
+    api.on("model_call_ended", (event, agentCtx) => {
+        try {
+            const sessionKey = readSessionKey(agentCtx, event);
+            const runId = readRunId(agentCtx, event);
+            if (!sessionKey || !runId)
+                return;
+            const held = modelSpans.get(sessionLlmKey(sessionKey, runId));
+            if (!held)
+                return; // span already finalized, or not one we opened — best effort.
+            const ev = asRecord(event) ?? {};
+            const outcome = nonEmptyString(ev.outcome);
+            const failureKind = nonEmptyString(ev.failureKind);
+            const errorCategory = nonEmptyString(ev.errorCategory);
+            setDefinedAttributes(held.span, {
+                "senpi.gen_ai.outcome": outcome,
+                "senpi.gen_ai.failure_kind": failureKind,
+                "senpi.gen_ai.error_category": errorCategory,
+                "senpi.gen_ai.model_call_duration_ms": firstNumber(ev.durationMs),
+            });
+            if (outcome === "error") {
+                // Mark errored so the agent_end sweep doesn't re-record the same failure.
+                held.errored = true;
+                setDefinedAttributes(held.span, {
+                    [GEN_AI.RESPONSE_FINISH_REASONS]: toFinishReasons(failureKind ?? "error"),
+                });
+                recordSpanError(held.span, failureKind ?? errorCategory ?? "model_call_error");
+            }
+        }
+        catch (err) {
+            logger.warn("model_call_ended handler failed", { err: errMsg(err) });
+        }
+    });
+    // ─── Tool (`execute_tool` span) ────────────────────────────────────────────
+    api.on("before_tool_call", (event, toolCtx) => {
+        try {
+            const identity = readIdentity(toolCtx);
+            const resolved = resolveToolSession(toolCtx);
+            if (!resolved) {
+                logger.debug("before_tool_call session unresolved (no llm_input seen yet); skipping");
+                return;
+            }
+            const ev = asRecord(event);
+            const toolName = nonEmptyString(ev?.toolName) ?? nonEmptyString(ev?.name) ?? "unknown";
+            const toolCallId = nonEmptyString(ev?.toolCallId) ??
+                nonEmptyString(asRecord(toolCtx)?.toolCallId) ??
+                null;
+            const key = toolKey(resolved.sessionKey, resolved.runId, toolCallId, toolName, ++toolSeq);
+            const span = tracer.startSpan(`execute_tool ${toolName}`, { kind: SpanKind.INTERNAL }, parentContextForModel(resolved.sessionKey, resolved.runId));
+            const held = {
+                span,
+                sessionKey: resolved.sessionKey,
+                runId: resolved.runId,
+                startMs: Date.now(),
+                toolName,
+            };
+            toolSpans.set(key, held);
+            setDefinedAttributes(span, {
+                [GEN_AI.OPERATION_NAME]: "execute_tool",
+                [GEN_AI.CONVERSATION_ID]: resolved.sessionKey,
+                [GEN_AI.TOOL_NAME]: toolName,
+                [GEN_AI.TOOL_CALL_ID]: toolCallId ?? undefined,
+                // Best-effort skill attribution from a skill-file read (no first-class skill hook exists).
+                "senpi.skill.name": skillNameFromToolArgs(ev?.params ?? ev?.input),
+                ...identityAttributes(identity),
+            });
+            // Args stamping is async; the finalize (after_tool_call) awaits this so a fast
+            // tool doesn't end the span before the args land.
+            held.pending = setGenAiContentAttribute(span, GEN_AI.TOOL_CALL_ARGUMENTS, ev?.params ?? ev?.input);
+        }
+        catch (err) {
+            logger.warn("before_tool_call handler failed", { err: errMsg(err) });
+        }
+    });
+    api.on("after_tool_call", (event, toolCtx) => {
+        try {
+            const resolved = resolveToolSession(toolCtx);
+            if (!resolved) {
+                logger.debug("after_tool_call session unresolved; skipping");
+                return;
+            }
+            const ev = asRecord(event);
+            const toolName = nonEmptyString(ev?.toolName) ?? nonEmptyString(ev?.name) ?? "unknown";
+            const error = ev?.error;
+            const errored = error !== undefined && error !== null;
+            // Tool-plane event (narrative): a failed tool call — the agent-blind hole, greppable instead
+            // of span-only. A successful call already rides its span + (state-changing) its domain event,
+            // so it is not re-evented here. Emitted before the span lookup so a failure is never lost when
+            // the matching before_tool_call span wasn't registered.
+            if (errored) {
+                emitToolCalled({
+                    name: toolName,
+                    origin: "agent",
+                    outcome: "error",
+                    errorType: errorTypeOf(error),
+                    errorMessage: error instanceof Error ? error.message : String(error),
+                });
+            }
+            const toolCallId = nonEmptyString(ev?.toolCallId) ??
+                nonEmptyString(asRecord(toolCtx)?.toolCallId) ??
+                null;
+            const foundKey = findToolKey(toolSpans, resolved.sessionKey, resolved.runId, toolCallId, toolName);
+            if (!foundKey) {
+                logger.debug("after_tool_call: no open tool span", { toolName, toolCallId });
+                return;
+            }
+            const held = toolSpans.get(foundKey);
+            const span = held.span;
+            toolSpans.delete(foundKey);
+            const result = ev?.result;
+            recordToolCall({
+                toolName: held.toolName,
+                origin: "agent",
+                outcome: errored ? "error" : "ok",
+                errorType: errored ? errorTypeOf(error) : undefined,
+                durationSec: elapsedSeconds(held.startMs),
+            });
+            void (async () => {
+                try {
+                    if (held.pending)
+                        await held.pending;
+                    if (errored) {
+                        recordSpanError(span, error);
+                    }
+                    else {
+                        await setGenAiContentAttribute(span, GEN_AI.TOOL_CALL_RESULT, result);
+                    }
+                }
+                finally {
+                    try {
+                        span.end();
+                    }
+                    catch {
+                        /* best effort */
+                    }
+                }
+            })();
+        }
+        catch (err) {
+            logger.warn("after_tool_call handler failed", { err: errMsg(err) });
+        }
+    });
+    // ─── Subagent (`invoke_agent` span, four-hook lifecycle) ───────────────────
+    api.on("subagent_spawning", (event, subagentCtx) => {
+        try {
+            const identity = readIdentity(subagentCtx);
+            const ev = asRecord(event);
+            const ctxObj = asRecord(subagentCtx);
+            const hostSessionKey = nonEmptyString(ctxObj?.requesterSessionKey) ??
+                nonEmptyString(ctxObj?.sessionKey) ??
+                "";
+            const hostRunId = nonEmptyString(ctxObj?.runId) ?? nonEmptyString(ctxObj?.requesterRunId) ?? "";
+            const childSessionKey = nonEmptyString(ev?.childSessionKey) ?? nonEmptyString(ctxObj?.childSessionKey);
+            if (!childSessionKey) {
+                logger.warn("subagent_spawning missing childSessionKey; skipping span");
+                return;
+            }
+            const prior = subagentSpans.get(childSessionKey);
+            if (prior)
+                endSpan(prior.span);
+            const agentName = nonEmptyString(ev?.label) ?? nonEmptyString(ev?.agentId) ?? "unknown";
+            const span = tracer.startSpan(`invoke_agent ${agentName}`, { kind: SpanKind.INTERNAL }, hostSessionKey && hostRunId
+                ? parentContextForModel(hostSessionKey, hostRunId)
+                : otelContext.active());
+            // sessionKey/runId carry the HOST run so the agent_end sweep can reach this span.
+            subagentSpans.set(childSessionKey, {
+                span,
+                sessionKey: hostSessionKey,
+                runId: hostRunId,
+                startMs: Date.now(),
+                agentName,
+            });
+            setDefinedAttributes(span, {
+                [GEN_AI.OPERATION_NAME]: "invoke_agent",
+                [GEN_AI.CONVERSATION_ID]: nonEmptyString(hostSessionKey),
+                [GEN_AI.AGENT_NAME]: agentName,
+                "senpi.subagent.mode": nonEmptyString(ev?.mode),
+                "senpi.subagent.requester": nonEmptyString(ev?.requester),
+                "senpi.subagent.thread_requested": typeof ev?.threadRequested === "boolean" ? ev.threadRequested : undefined,
+                ...identityAttributes(identity),
+            });
+        }
+        catch (err) {
+            logger.warn("subagent_spawning handler failed", { err: errMsg(err) });
+        }
+    });
+    api.on("subagent_spawned", (event, subagentCtx) => {
+        try {
+            const ev = asRecord(event);
+            const ctxObj = asRecord(subagentCtx);
+            const childSessionKey = nonEmptyString(ev?.childSessionKey) ?? nonEmptyString(ctxObj?.childSessionKey);
+            if (!childSessionKey)
+                return;
+            const held = subagentSpans.get(childSessionKey);
+            if (!held)
+                return;
+            const span = held.span;
+            setDefinedAttributes(span, {
+                "senpi.subagent.run_id": nonEmptyString(ev?.runId) ?? nonEmptyString(ctxObj?.runId),
+                "senpi.subagent.mode": nonEmptyString(ev?.mode),
+                "senpi.subagent.thread_requested": typeof ev?.threadRequested === "boolean" ? ev.threadRequested : undefined,
+            });
+        }
+        catch (err) {
+            logger.warn("subagent_spawned handler failed", { err: errMsg(err) });
+        }
+    });
+    api.on("subagent_delivery_target", (event, subagentCtx) => {
+        try {
+            const ev = asRecord(event);
+            const ctxObj = asRecord(subagentCtx);
+            const childSessionKey = nonEmptyString(ev?.childSessionKey) ?? nonEmptyString(ctxObj?.childSessionKey);
+            if (!childSessionKey)
+                return;
+            const held = subagentSpans.get(childSessionKey);
+            if (!held)
+                return;
+            const span = held.span;
+            const origin = asRecord(ev?.requesterOrigin) ?? {};
+            setDefinedAttributes(span, {
+                "senpi.subagent.delivery.channel": nonEmptyString(origin.channel),
+                "senpi.subagent.delivery.account_id": nonEmptyString(origin.accountId),
+                "senpi.subagent.delivery.to": nonEmptyString(origin.to),
+                "senpi.subagent.delivery.child_run_id": nonEmptyString(ev?.childRunId),
+                "senpi.subagent.delivery.spawn_mode": nonEmptyString(ev?.spawnMode),
+            });
+        }
+        catch (err) {
+            logger.warn("subagent_delivery_target handler failed", { err: errMsg(err) });
+        }
+    });
+    api.on("subagent_ended", (event, subagentCtx) => {
+        try {
+            const ev = asRecord(event);
+            const ctxObj = asRecord(subagentCtx);
+            const childSessionKey = nonEmptyString(ev?.targetSessionKey) ??
+                nonEmptyString(ctxObj?.childSessionKey) ??
+                nonEmptyString(ev?.childSessionKey);
+            if (!childSessionKey)
+                return;
+            const held = subagentSpans.get(childSessionKey);
+            if (!held)
+                return;
+            const span = held.span;
+            subagentSpans.delete(childSessionKey);
+            const error = ev?.error;
+            const errored = error !== undefined && error !== null;
+            recordInvokeAgentDuration(elapsedSeconds(held.startMs), {
+                agentName: held.agentName,
+                errorType: errored ? errorTypeOf(error) : undefined,
+            });
+            if (errored) {
+                recordSpanError(span, error);
+            }
+            else {
+                setDefinedAttributes(span, {
+                    "senpi.subagent.outcome": nonEmptyString(ev?.outcome),
+                    "senpi.subagent.reason": nonEmptyString(ev?.reason),
+                });
+            }
+            try {
+                span.end();
+            }
+            catch {
+                /* best effort */
+            }
+        }
+        catch (err) {
+            logger.warn("subagent_ended handler failed", { err: errMsg(err) });
+        }
+    });
+    // ─── Agent end (orphan sweep) ──────────────────────────────────────────────
+    api.on("agent_end", (event, agentCtx) => {
+        try {
+            const sessionKey = readSessionKey(agentCtx, event);
+            const runId = readRunId(agentCtx, event);
+            if (!sessionKey || !runId) {
+                logger.warn("agent_end missing sessionKey/runId; skipping sweep");
+                return;
+            }
+            const ev = asRecord(event);
+            const outcomeError = ev?.error;
+            // End any span still open under this (session, run): the model span, its tool
+            // children, and any subagents it hosted. Matching is by the stored host
+            // session/run, so nothing leaks past the run's end (incl. subagents, which have
+            // no other guaranteed terminator).
+            const llmKey = sessionLlmKey(sessionKey, runId);
+            const model = modelSpans.get(llmKey);
+            if (model) {
+                const errored = outcomeError !== undefined && outcomeError !== null;
+                recordOperationDuration(elapsedSeconds(model.startMs), {
+                    provider: model.provider,
+                    model: model.model,
+                    errorType: errored ? errorTypeOf(outcomeError) : undefined,
+                });
+                // Skip if model_call_ended already recorded the failure on this span.
+                if (errored && !model.errored)
+                    recordSpanError(model.span, outcomeError);
+                // Defer the span end: agent_end fires ~1 ms before the trailing llm_output that
+                // carries token usage + output messages. Hold the span (leave it in modelSpans) a
+                // short grace so that llm_output can finalize it; if none arrives, the timer ends it.
+                if (!model.endTimer) {
+                    model.endTimer = setTimeout(() => {
+                        if (modelSpans.get(llmKey) === model)
+                            modelSpans.delete(llmKey);
+                        endSpan(model.span);
+                    }, MODEL_SPAN_END_GRACE_MS);
+                    model.endTimer.unref?.();
+                }
+            }
+            for (const [k, held] of toolSpans) {
+                if (held.sessionKey === sessionKey && held.runId === runId) {
+                    toolSpans.delete(k);
+                    endSpan(held.span);
+                }
+            }
+            for (const [k, held] of subagentSpans) {
+                if (held.sessionKey === sessionKey && held.runId === runId) {
+                    subagentSpans.delete(k);
+                    endSpan(held.span);
+                }
+            }
+            // Evict the session-resolution memos for this run so they can't grow unbounded
+            // over the lifetime of the (long-lived) host process.
+            for (const [id, sess] of sessionByAgentId) {
+                if (sess.sessionKey === sessionKey && sess.runId === runId)
+                    sessionByAgentId.delete(id);
+            }
+            if (lastActive && lastActive.sessionKey === sessionKey && lastActive.runId === runId) {
+                lastActive = null;
+            }
+        }
+        catch (err) {
+            logger.warn("agent_end handler failed", { err: errMsg(err) });
+        }
+    });
+}
+// ─── Span key helpers ───────────────────────────────────────────────────────
+const SEP = " ";
+function sessionLlmKey(sessionKey, runId) {
+    return `${sessionKey}${SEP}${runId}${SEP}llm`;
+}
+function toolKey(sessionKey, runId, toolCallId, toolName, seq) {
+    return toolCallId
+        ? `${sessionKey}${SEP}${runId}${SEP}toolcall${SEP}${toolCallId}`
+        : `${sessionKey}${SEP}${runId}${SEP}tool${SEP}${toolName}${SEP}${seq}`;
+}
+/** Find the open tool span key within one (session, run): by toolCallId first, else by tool name. */
+function findToolKey(toolSpans, sessionKey, runId, toolCallId, toolName) {
+    const prefix = `${sessionKey}${SEP}${runId}${SEP}`;
+    if (toolCallId) {
+        const suffix = `${SEP}toolcall${SEP}${toolCallId}`;
+        for (const key of toolSpans.keys()) {
+            if (key.startsWith(prefix) && key.endsWith(suffix))
+                return key;
+        }
+    }
+    const namePart = `${SEP}tool${SEP}${toolName}${SEP}`;
+    for (const key of toolSpans.keys()) {
+        if (key.startsWith(prefix) && key.includes(namePart))
+            return key;
+    }
+    return undefined;
+}
+function endSpan(span) {
+    try {
+        span.end();
+    }
+    catch {
+        /* best effort */
+    }
+}
+// ─── Identity + trace correlation ─────────────────────────────────────────────
+const ADDRESS_RE = /^0x[a-f0-9]{40}$/i;
+const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+/** Cap on a free-form header value stamped as an attribute (defence vs oversized header). */
+const MAX_HEADER_ATTR_CHARS = 256;
+function clampHeader(v) {
+    return v.length > MAX_HEADER_ATTR_CHARS ? v.slice(0, MAX_HEADER_ATTR_CHARS) : v;
+}
+/**
+ * Map the senpi identity headers to `senpi.*` span attributes (keys match the on-site
+ * identity bag; the headers carry the runtime id as `senpi.runtime.id`, vs on-site
+ * `senpi.runtime`). Headers are caller-influenceable, so join keys are validated — a
+ * malformed address / correlation id is dropped, not stamped as garbage.
+ */
+function identityAttributes(identity) {
+    const attrs = {};
+    if (identity.address) {
+        const lowered = identity.address.toLowerCase();
+        if (ADDRESS_RE.test(lowered))
+            attrs["senpi.strategy.address"] = lowered;
+    }
+    if (identity.recipeName)
+        attrs["senpi.recipe"] = clampHeader(identity.recipeName);
+    if (identity.runtimeVersion)
+        attrs["senpi.recipe.version"] = clampHeader(identity.runtimeVersion);
+    if (identity.runtimeId)
+        attrs["senpi.runtime.id"] = clampHeader(identity.runtimeId);
+    // Fallback join key to the trading trace when traceparent didn't thread through.
+    if (identity.runId && UUID_RE.test(identity.runId)) {
+        attrs["senpi.run.id"] = identity.runId;
+    }
+    return attrs;
+}
+function readIdentity(agentCtx) {
+    const headers = readHeaders(agentCtx);
+    return {
+        runId: headers[HEADER_CORRELATION_ID],
+        address: headers[HEADER_ADDRESS],
+        recipeName: headers[HEADER_RECIPE_NAME],
+        runtimeId: headers[HEADER_RUNTIME_ID],
+        runtimeVersion: headers[HEADER_RUNTIME_VERSION],
+    };
+}
+/** Extract a W3C traceparent from the request headers → a parent Context (else active). */
+function parentContextFromHeaders(agentCtx) {
+    try {
+        const headers = readHeaders(agentCtx);
+        if (headers.traceparent === undefined)
+            return otelContext.active();
+        return propagation.extract(otelContext.active(), headers);
+    }
+    catch {
+        return otelContext.active();
+    }
+}
+// ─── Field readers ────────────────────────────────────────────────────────────
+function asRecord(v) {
+    if (v && typeof v === "object" && !Array.isArray(v)) {
+        return v;
+    }
+    return undefined;
+}
+function nonEmptyString(v) {
+    return typeof v === "string" && v.length > 0 ? v : undefined;
+}
+/** A skill loaded by the agent reading a file under `…/skills/<name>/…` (e.g. its SKILL.md). */
+const SKILL_PATH_RE = /(?:^|\/)skills\/([^/]+)\//;
+/**
+ * Best-effort skill attribution: there is no first-class skill-invocation hook, so a skill is
+ * detectable only by the agent reading its file. Returns the skill name when a tool-call arg path
+ * points under a `skills/<name>/` dir, else undefined. Heuristic — a real gateway skill hook,
+ * once it exists, would supersede this.
+ */
+function skillNameFromToolArgs(args) {
+    const rec = asRecord(args);
+    if (!rec)
+        return undefined;
+    const path = nonEmptyString(rec.path) ?? nonEmptyString(rec.file_path) ?? nonEmptyString(rec.filePath);
+    if (!path)
+        return undefined;
+    return SKILL_PATH_RE.exec(path)?.[1];
+}
+/** Pull request headers out of agentCtx; several common locations are checked. */
+function readHeaders(agentCtx) {
+    const ctxObj = asRecord(agentCtx);
+    if (!ctxObj)
+        return {};
+    const candidates = [
+        asRecord(ctxObj.headers),
+        asRecord(asRecord(ctxObj.request)?.headers),
+        asRecord(ctxObj.requestHeaders),
+        asRecord(asRecord(ctxObj.meta)?.headers),
+    ];
+    const out = {};
+    for (const candidate of candidates) {
+        if (!candidate)
+            continue;
+        for (const [k, v] of Object.entries(candidate)) {
+            const lk = k.toLowerCase();
+            if (typeof v === "string" && v.length > 0 && out[lk] === undefined) {
+                out[lk] = v;
+            }
+            else if (Array.isArray(v) && v.length > 0 && typeof v[0] === "string") {
+                out[lk] = v[0];
+            }
+        }
+    }
+    return out;
+}
+function readSessionKey(ctx, event) {
+    return (nonEmptyString(asRecord(ctx)?.sessionKey) ??
+        nonEmptyString(asRecord(ctx)?.sessionId) ??
+        nonEmptyString(asRecord(event)?.sessionKey) ??
+        nonEmptyString(asRecord(event)?.sessionId));
+}
+function readRunId(ctx, event) {
+    return nonEmptyString(asRecord(ctx)?.runId) ?? nonEmptyString(asRecord(event)?.runId);
+}
+// ─── gen_ai content builders ──────────────────────────────────────────────────
+/** System instructions as `[{type:"text", content}]` per the gen_ai shape. */
+function buildSystemInstructions(ev) {
+    const system = ev.systemPrompt ?? ev.system;
+    if (typeof system !== "string" || system.length === 0)
+        return undefined;
+    return [{ type: "text", content: system }];
+}
+/** Input messages: the history (or messages), with any standalone `prompt` as a user turn. */
+function buildInputMessages(ev) {
+    const history = ev.historyMessages ?? ev.messages;
+    const messages = Array.isArray(history) ? [...history] : history ? [history] : [];
+    if (typeof ev.prompt === "string" && ev.prompt.length > 0) {
+        messages.push({ role: "user", content: ev.prompt });
+    }
+    return normalizeGenAiMessages(messages, "user");
+}
+/**
+ * Output messages. Prefer the structured `lastAssistant` — it carries tool calls, which the
+ * text-only `assistantTexts` does not. The catch the old `assistantTexts ?? lastAssistant`
+ * missed: `assistantTexts` is `[]` (not nullish) on a tool-call-only turn, so `??` never fell
+ * through and the turn's output was dropped. Fall back to the text chunks, then any raw content.
+ */
+function buildOutputMessages(ev) {
+    const last = ev.lastAssistant;
+    if (last !== undefined && last !== null) {
+        // A full message object (has a role) passes through; a content array/string/object
+        // becomes the body of one assistant message.
+        const rec = asRecord(last);
+        const message = rec && typeof rec.role === "string" ? last : { role: "assistant", content: last };
+        return normalizeGenAiMessages([message], "assistant");
+    }
+    const texts = Array.isArray(ev.assistantTexts)
+        ? ev.assistantTexts.filter((t) => typeof t === "string" && t.length > 0)
+        : [];
+    const content = texts.length > 0 ? texts : ev.content;
+    if (content === undefined || content === null)
+        return undefined;
+    return normalizeGenAiMessages([{ role: "assistant", content }], "assistant");
+}
+function errMsg(err) {
+    return err instanceof Error ? err.message : String(err);
+}
+//# sourceMappingURL=span-hooks.js.map