@desplega.ai/agent-swarm 1.79.4 → 1.80.1

package/src/commands/runner.ts

@@ -2,6 +2,16 @@ import { existsSync, statSync } from "node:fs";
 import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
 import { ensure, initialize } from "@desplega.ai/business-use";
 import type { TemplateResponse } from "../../templates/schema.ts";
+import {
+  type Attributes,
+  initOtel,
+  injectTraceContext,
+  isPollTracingEnabled,
+  type SwarmSpan,
+  startSpan,
+  withSpan,
+  withSpanContext,
+} from "../otel.ts";
 import { type BasePromptArgs, getBasePrompt } from "../prompts/base-prompt.ts";
 import {
   generateDefaultClaudeMd,
@@ -15,12 +25,14 @@ import { authJsonToCredentialSelection } from "../providers/codex-oauth/auth-jso
 import {
   type CostData,
   createProviderAdapter,
+  type ProviderEvent,
   type ProviderResult,
   type ProviderSession,
   type ProviderSessionConfig,
 } from "../providers/index.ts";
 import { initTelemetry, telemetry } from "../telemetry.ts";
 import type { ProviderName, RepoGuidelines } from "../types.ts";
+import { getApiKey } from "../utils/api-key.ts";
 import { computeBudgetBackoffMs } from "../utils/budget-backoff.ts";
 import { getContextWindowSize } from "../utils/context-window.ts";
 import { type CredentialSelection, resolveCredentialPools } from "../utils/credentials.ts";
@@ -402,6 +414,12 @@ const SWARM_TOOL_LABELS: Record<string, string | null> = {
   "list-services": "🔌 Listing services",
   "unregister-service": "🔌 Unregistering service",
   "update-service-status": "🔌 Updating service status",
+  // Reusable scripts
+  "script-search": "📜 Searching scripts",
+  "script-run": "📜 Running script",
+  "script-upsert": "📜 Saving script",
+  "script-delete": "📜 Deleting script",
+  "script-query-types": "📜 Reading script types",
 };
 
 /** Convert kebab-case to sentence case: "get-task-details" → "Get task details" */
@@ -477,13 +495,11 @@ export function toolCallToProgress(toolName: string, args: unknown): string | nu
       }
 
       // Pi-mono exposes tools from the built-in swarm MCP endpoint as bare
-      // names ("store-progress", "send-task", ...), not as mcp__ names.
+      // names ("store-progress", "send-task", "script-run", ...), not as mcp__ names.
       // Treat those names as agent-swarm tools so activity stays readable.
-      if (toolName.includes("-")) {
-        const label = SWARM_TOOL_LABELS[toolName];
-        if (label === null) return null;
-        if (label) return label;
-      }
+      const label = SWARM_TOOL_LABELS[toolName];
+      if (label === null) return null;
+      if (label) return label;
 
       return `🔧 ${toolName}`;
     }
@@ -1503,6 +1519,25 @@ async function registerAgent(opts: {
 
 /** Poll for triggers via HTTP API */
 async function pollForTrigger(opts: PollOptions): Promise<Trigger | null> {
+  if (!isPollTracingEnabled()) {
+    return pollForTriggerOnce(opts);
+  }
+  return withSpan(
+    "worker.poll",
+    async (span) => {
+      const trigger = await pollForTriggerOnce(opts);
+      span.setAttribute("agentswarm.poll.result", trigger ? trigger.type : "empty");
+      return trigger;
+    },
+    {
+      "agent.id": opts.agentId,
+      "agentswarm.worker.poll_timeout_ms": opts.pollTimeout,
+      "agentswarm.worker.poll_interval_ms": opts.pollInterval,
+    },
+  );
+}
+
+async function pollForTriggerOnce(opts: PollOptions): Promise<Trigger | null> {
   const startTime = Date.now();
   const headers: Record<string, string> = {
     "X-Agent-ID": opts.agentId,
@@ -1510,6 +1545,7 @@ async function pollForTrigger(opts: PollOptions): Promise<Trigger | null> {
   if (opts.apiKey) {
     headers.Authorization = `Bearer ${opts.apiKey}`;
   }
+  injectTraceContext(headers);
 
   while (Date.now() - startTime < opts.pollTimeout) {
     try {
@@ -1739,6 +1775,198 @@ function extractToolKey(toolName: string, args: unknown): Record<string, string
   }
 }
 
+const OTEL_PREVIEW_LIMIT = 500;
+
+function telemetryPreview(value: unknown): string | undefined {
+  if (value === undefined) return undefined;
+  try {
+    const serialized = typeof value === "string" ? value : JSON.stringify(value);
+    if (!serialized) return undefined;
+    const scrubbed = scrubSecrets(serialized);
+    return scrubbed.length > OTEL_PREVIEW_LIMIT
+      ? `${scrubbed.slice(0, OTEL_PREVIEW_LIMIT)}...`
+      : scrubbed;
+  } catch {
+    return "[unserializable]";
+  }
+}
+
+type ToolTelemetry = {
+  kind: "mcp" | "harness" | "skill" | "agent" | "shell" | "file" | "unknown";
+  name: string;
+  normalizedName: string;
+  mcpServer?: string;
+  mcpTool?: string;
+};
+
+function classifyTool(toolName: string, args: unknown): ToolTelemetry {
+  const argRecord = args && typeof args === "object" ? (args as Record<string, unknown>) : {};
+
+  if (toolName.startsWith("mcp__")) {
+    const [, server, ...toolParts] = toolName.split("__");
+    const mcpTool = toolParts.join("__") || undefined;
+    return {
+      kind: "mcp",
+      name: toolName,
+      normalizedName: server && mcpTool ? `${server}.${mcpTool}` : toolName,
+      mcpServer: server,
+      mcpTool,
+    };
+  }
+
+  if (typeof argRecord.server === "string" && typeof argRecord.tool === "string") {
+    return {
+      kind: "mcp",
+      name: toolName,
+      normalizedName: `${argRecord.server}.${argRecord.tool}`,
+      mcpServer: argRecord.server,
+      mcpTool: argRecord.tool,
+    };
+  }
+
+  if (toolName.includes(":")) {
+    const [server, ...toolParts] = toolName.split(":");
+    const mcpTool = toolParts.join(":") || undefined;
+    return {
+      kind: "mcp",
+      name: toolName,
+      normalizedName: server && mcpTool ? `${server}.${mcpTool}` : toolName,
+      mcpServer: server,
+      mcpTool,
+    };
+  }
+
+  switch (toolName) {
+    case "Bash":
+    case "bash":
+    case "command_execution":
+      return { kind: "shell", name: toolName, normalizedName: toolName };
+    case "Read":
+    case "Edit":
+    case "Write":
+    case "Delete":
+    case "Grep":
+    case "Glob":
+    case "file_change":
+      return { kind: "file", name: toolName, normalizedName: toolName };
+    case "Skill":
+      return { kind: "skill", name: toolName, normalizedName: toolName };
+    case "Agent":
+      return { kind: "agent", name: toolName, normalizedName: toolName };
+    default:
+      return { kind: "harness", name: toolName, normalizedName: toolName };
+  }
+}
+
+function providerEventAttributes(event: ProviderEvent): Attributes {
+  switch (event.type) {
+    case "session_init":
+      return {
+        "agentswarm.provider.session_id": event.sessionId,
+        "agentswarm.provider.name": event.provider,
+        "agentswarm.provider.meta_preview": telemetryPreview(event.providerMeta),
+      };
+    case "message":
+      return {
+        "gen_ai.message.role": event.role,
+        "gen_ai.message.content_preview": telemetryPreview(event.content),
+      };
+    case "tool_start":
+    case "tool_end": {
+      const tool = classifyTool(
+        event.toolName,
+        event.type === "tool_start" ? event.args : undefined,
+      );
+      return {
+        "agentswarm.tool.name": tool.name,
+        "agentswarm.tool.normalized_name": tool.normalizedName,
+        "agentswarm.tool.kind": tool.kind,
+        "agentswarm.tool.call_id": event.toolCallId,
+        "mcp.server.name": tool.mcpServer,
+        "mcp.tool.name": tool.mcpTool,
+      };
+    }
+    case "result":
+      return {
+        "agentswarm.session.outcome": event.isError ? "error" : "ok",
+        "agentswarm.error.category": event.errorCategory,
+        "gen_ai.response.model": event.cost.model,
+        "gen_ai.usage.input_tokens": event.cost.inputTokens ?? 0,
+        "gen_ai.usage.output_tokens": event.cost.outputTokens ?? 0,
+        "agentswarm.cost.total_usd": event.cost.totalCostUsd ?? 0,
+      };
+    case "error":
+      return {
+        "agentswarm.error.category": event.category,
+        "exception.message": telemetryPreview(event.message),
+      };
+    case "progress":
+      return { "agentswarm.progress.message": telemetryPreview(event.message) };
+    case "context_usage":
+      return {
+        "agentswarm.context.used_tokens": event.contextUsedTokens ?? undefined,
+        "agentswarm.context.total_tokens": event.contextTotalTokens ?? undefined,
+        "agentswarm.context.percent": event.contextPercent ?? undefined,
+        "gen_ai.usage.output_tokens": event.outputTokens ?? undefined,
+      };
+    case "compaction":
+      return {
+        "agentswarm.compaction.trigger": event.compactTrigger,
+        "agentswarm.compaction.pre_tokens": event.preCompactTokens,
+        "agentswarm.context.total_tokens": event.contextTotalTokens,
+      };
+    case "custom":
+      return {
+        "agentswarm.provider.event_name": event.name,
+        "agentswarm.provider.event_data_preview": telemetryPreview(event.data),
+      };
+    case "raw_log":
+    case "raw_stderr":
+      return {};
+  }
+}
+
+/**
+ * Entry shape for the `activeToolSpans` map maintained by `runWithSession`.
+ * Exported for unit tests that exercise `implicitCloseActiveToolSpans`.
+ */
+export type ActiveToolSpanEntry = {
+  span: SwarmSpan;
+  startedAt: number;
+};
+
+/**
+ * Closes any still-open tool spans (both `worker.tool` and `worker.mcp.tool`)
+ * in `activeToolSpans` at the assistant-message boundary, tagging them with
+ * `agentswarm.tool.implicit_close=true` and removing them from the map.
+ *
+ * The Claude SDK adapter doesn't emit per-tool completion events for any
+ * tool kind, MCP or harness-side — so the assistant-message boundary serves
+ * as an implicit `tool_end` for everything. Spans closed here are still
+ * successful (code 1, OK); the boundary is just a signal that the prior
+ * tool turn is done.
+ *
+ * Returns the number of spans closed (handy for tests / metrics).
+ */
+export function implicitCloseActiveToolSpans(
+  activeToolSpans: Map<string, ActiveToolSpanEntry>,
+  now: number = Date.now(),
+): number {
+  let closed = 0;
+  for (const [toolCallId, active] of activeToolSpans) {
+    active.span.setAttributes({
+      "agentswarm.tool.duration_ms": now - active.startedAt,
+      "agentswarm.tool.implicit_close": true,
+      "agentswarm.tool.call_id": toolCallId,
+    });
+    active.span.setStatus({ code: 1 });
+    active.span.end();
+    activeToolSpans.delete(toolCallId);
+    closed++;
+  }
+  return closed;
+}
+
 async function spawnProviderProcess(
   adapter: ReturnType<typeof createProviderAdapter>,
   opts: {
@@ -1806,7 +2034,25 @@ async function spawnProviderProcess(
     env: freshEnv as Record<string, string>,
   };
 
-  const session = await adapter.createSession(config);
+  const session = await withSpan(
+    "worker.session.create",
+    async (span) => {
+      const createdSession = await adapter.createSession(config);
+      span.setAttribute("agentswarm.provider.session_id", createdSession.sessionId || "pending");
+      return createdSession;
+    },
+    {
+      "agent.id": opts.agentId,
+      "agentswarm.task.id": effectiveTaskId,
+      "agentswarm.task.real_id": realTaskId,
+      "agentswarm.agent.role": opts.role,
+      "agentswarm.harness_provider": opts.harnessProvider,
+      "gen_ai.request.model": model || undefined,
+      "agentswarm.session.cwd": config.cwd,
+      "agentswarm.session.vcs_repo": opts.vcsRepo,
+      "agentswarm.session.additional_args_count": opts.additionalArgs?.length ?? 0,
+    },
+  );
   const initialModelReport = buildLatestModelReport({
     model,
     taskModel: opts.model,
@@ -1884,6 +2130,25 @@ async function spawnProviderProcess(
 
   const eventFlushTimer = setInterval(flushEvents, EVENT_FLUSH_INTERVAL_MS);
   const sessionStartTime = Date.now();
+  let providerSessionId = session.sessionId;
+  const activeToolSpans = new Map<
+    string,
+    {
+      span: SwarmSpan;
+      startedAt: number;
+    }
+  >();
+  const sessionSpan = startSpan("worker.session", {
+    "agent.id": opts.agentId,
+    "agentswarm.task.id": effectiveTaskId,
+    "agentswarm.task.real_id": realTaskId,
+    "agentswarm.agent.role": opts.role,
+    "agentswarm.harness_provider": opts.harnessProvider,
+    "agentswarm.provider.session_id": providerSessionId,
+    "agentswarm.session.cwd": config.cwd,
+    "agentswarm.session.vcs_repo": opts.vcsRepo,
+    "gen_ai.request.model": model || undefined,
+  });
 
   // Auto-progress throttle: don't update more than once per 3 seconds
   let lastProgressTime = 0;
@@ -1892,130 +2157,248 @@ async function spawnProviderProcess(
   let lastContextPostTime = 0;
   const CONTEXT_THROTTLE_MS = 30_000;
 
-  session.onEvent((event) => {
-    switch (event.type) {
-      case "session_init":
-        if (realTaskId) {
-          saveProviderSessionId(
-            opts.apiUrl,
-            opts.apiKey,
-            realTaskId,
-            event.sessionId,
-            event.provider,
-            event.providerMeta,
-            model,
-          ).catch((err) => console.warn(`[runner] Failed to save session ID: ${err}`));
-        } else {
-          // Pool task: save provider session ID on active session so it can be
-          // propagated to the real task when the agent claims one
-          saveProviderSessionIdOnActiveSession(
-            opts.apiUrl,
-            opts.apiKey,
-            effectiveTaskId,
-            event.sessionId,
-          ).catch((err) =>
-            console.warn(`[runner] Failed to save provider session on active session: ${err}`),
-          );
-        }
-
-        // Buffer session start event
-        bufferEvent({
-          category: "session",
-          event: "session.start",
-          source: "worker",
-          agentId: opts.agentId,
-          taskId: effectiveTaskId,
-          sessionId: event.sessionId,
-        });
-        break;
-      case "tool_start": {
-        // Auto-progress: report tool activity as task progress (throttled)
-        const now = Date.now();
-        if (effectiveTaskId && opts.apiUrl && now - lastProgressTime >= PROGRESS_THROTTLE_MS) {
-          const progress = toolCallToProgress(event.toolName, event.args);
-          if (progress) {
-            lastProgressTime = now;
-            updateProgressViaAPI(opts.apiUrl, opts.apiKey, effectiveTaskId, progress).catch(
-              () => {},
+  // Phase 10: accumulate per-turn output tokens across the session so progress
+  // snapshots carry a real `cumulativeOutputTokens` (was 0 until completion).
+  let cumulativeProgressOutputTokens = 0;
+
+  session.onEvent((event) =>
+    withSpanContext(sessionSpan, () => {
+      sessionSpan.addEvent(`provider.${event.type}`, providerEventAttributes(event));
+      switch (event.type) {
+        case "session_init":
+          providerSessionId = event.sessionId;
+          sessionSpan.setAttributes({
+            "agentswarm.provider.session_id": providerSessionId,
+            "agentswarm.provider.name": event.provider,
+            "agentswarm.provider.meta_preview": telemetryPreview(event.providerMeta),
+          });
+          if (realTaskId) {
+            saveProviderSessionId(
+              opts.apiUrl,
+              opts.apiKey,
+              realTaskId,
+              event.sessionId,
+              event.provider,
+              event.providerMeta,
+              model,
+            ).catch((err) => console.warn(`[runner] Failed to save session ID: ${err}`));
+          } else {
+            // Pool task: save provider session ID on active session so it can be
+            // propagated to the real task when the agent claims one
+            saveProviderSessionIdOnActiveSession(
+              opts.apiUrl,
+              opts.apiKey,
+              effectiveTaskId,
+              event.sessionId,
+            ).catch((err) =>
+              console.warn(`[runner] Failed to save provider session on active session: ${err}`),
             );
           }
+
+          // Buffer session start event
+          bufferEvent({
+            category: "session",
+            event: "session.start",
+            source: "worker",
+            agentId: opts.agentId,
+            taskId: effectiveTaskId,
+            sessionId: event.sessionId,
+          });
+          break;
+        case "message": {
+          // Assistant-message boundary acts as an implicit tool_end for ALL
+          // still-open tool spans (both `worker.tool` and `worker.mcp.tool`).
+          // The Claude SDK adapter doesn't emit per-tool completion events for
+          // any tool kind, so without this their spans would only close at
+          // session shutdown via `closeActiveToolSpans` and report wall-clock
+          // duration from tool_start to session end.
+          if (event.role === "assistant") {
+            implicitCloseActiveToolSpans(activeToolSpans);
+          }
+          break;
         }
+        case "tool_start": {
+          const tool = classifyTool(event.toolName, event.args);
+          const toolSpan = startSpan(tool.kind === "mcp" ? "worker.mcp.tool" : "worker.tool", {
+            "agent.id": opts.agentId,
+            "agentswarm.task.id": effectiveTaskId,
+            "agentswarm.task.real_id": realTaskId,
+            "agentswarm.agent.role": opts.role,
+            "agentswarm.harness_provider": opts.harnessProvider,
+            "agentswarm.provider.session_id": providerSessionId,
+            "agentswarm.tool.name": tool.name,
+            "agentswarm.tool.normalized_name": tool.normalizedName,
+            "agentswarm.tool.kind": tool.kind,
+            "agentswarm.tool.call_id": event.toolCallId,
+            "mcp.server.name": tool.mcpServer,
+            "mcp.tool.name": tool.mcpTool,
+            "agentswarm.tool.args_preview": telemetryPreview(event.args),
+          });
+          activeToolSpans.set(event.toolCallId, {
+            span: toolSpan,
+            startedAt: Date.now(),
+          });
 
-        // Buffer tool event
-        bufferEvent({
-          category: "tool",
-          event: "tool.start",
-          source: "worker",
-          agentId: opts.agentId,
-          taskId: effectiveTaskId,
-          sessionId: opts.runnerSessionId,
-          data: {
-            toolName: event.toolName,
-            toolCallId: event.toolCallId,
-            ...extractToolKey(event.toolName, event.args),
-            clientTimestamp: new Date().toISOString(),
-          },
-        });
+          // Auto-progress: report tool activity as task progress (throttled)
+          const now = Date.now();
+          if (effectiveTaskId && opts.apiUrl && now - lastProgressTime >= PROGRESS_THROTTLE_MS) {
+            const progress = toolCallToProgress(event.toolName, event.args);
+            if (progress) {
+              lastProgressTime = now;
+              updateProgressViaAPI(opts.apiUrl, opts.apiKey, effectiveTaskId, progress).catch(
+                () => {},
+              );
+            }
+          }
 
-        // Also emit skill event when tool is Skill
-        if (event.toolName === "Skill") {
-          const args = event.args as Record<string, unknown>;
+          // Buffer tool event
           bufferEvent({
-            category: "skill",
-            event: "skill.invoke",
+            category: "tool",
+            event: "tool.start",
             source: "worker",
             agentId: opts.agentId,
             taskId: effectiveTaskId,
             sessionId: opts.runnerSessionId,
             data: {
-              skillName: args.skill as string,
+              toolName: event.toolName,
+              toolCallId: event.toolCallId,
+              ...extractToolKey(event.toolName, event.args),
               clientTimestamp: new Date().toISOString(),
             },
           });
+
+          // Also emit skill event when tool is Skill
+          if (event.toolName === "Skill") {
+            const args = event.args as Record<string, unknown>;
+            bufferEvent({
+              category: "skill",
+              event: "skill.invoke",
+              source: "worker",
+              agentId: opts.agentId,
+              taskId: effectiveTaskId,
+              sessionId: opts.runnerSessionId,
+              data: {
+                skillName: args.skill as string,
+                clientTimestamp: new Date().toISOString(),
+              },
+            });
+          }
+          break;
         }
-        break;
-      }
-      case "result":
-        {
-          const latestModel = buildLatestModelReport({
-            model: event.cost.model,
-            taskModel: opts.model,
-            configModel,
-            taskId: realTaskId,
-            harnessProvider: opts.harnessProvider,
+        case "tool_end": {
+          const active = activeToolSpans.get(event.toolCallId);
+          const now = Date.now();
+          if (active) {
+            active.span.setAttributes({
+              "agentswarm.tool.duration_ms": now - active.startedAt,
+              "agentswarm.tool.result_preview": telemetryPreview(event.result),
+            });
+            active.span.setStatus({ code: 1 });
+            active.span.end();
+            activeToolSpans.delete(event.toolCallId);
+          } else {
+            const tool = classifyTool(event.toolName, undefined);
+            const span = startSpan(tool.kind === "mcp" ? "worker.mcp.tool" : "worker.tool", {
+              "agent.id": opts.agentId,
+              "agentswarm.task.id": effectiveTaskId,
+              "agentswarm.task.real_id": realTaskId,
+              "agentswarm.agent.role": opts.role,
+              "agentswarm.harness_provider": opts.harnessProvider,
+              "agentswarm.provider.session_id": providerSessionId,
+              "agentswarm.tool.name": tool.name,
+              "agentswarm.tool.normalized_name": tool.normalizedName,
+              "agentswarm.tool.kind": tool.kind,
+              "agentswarm.tool.call_id": event.toolCallId,
+              "mcp.server.name": tool.mcpServer,
+              "mcp.tool.name": tool.mcpTool,
+              "agentswarm.tool.result_preview": telemetryPreview(event.result),
+              "agentswarm.tool.missing_start": true,
+            });
+            span.end();
+          }
+          break;
+        }
+        case "result":
+          {
+            const latestModel = buildLatestModelReport({
+              model: event.cost.model,
+              taskModel: opts.model,
+              configModel,
+              taskId: realTaskId,
+              harnessProvider: opts.harnessProvider,
+            });
+            if (latestModel) {
+              reportLatestModel(opts.apiUrl, opts.apiKey, opts.agentId, latestModel).catch((err) =>
+                console.warn(`[runner] Failed to report latest model: ${err}`),
+              );
+            }
+          }
+          // Cost save is handled in waitForCompletion().then() to ensure
+          // it completes before the process exits (fire-and-forget here
+          // races with container shutdown).
+
+          // Buffer session end event
+          bufferEvent({
+            category: "session",
+            event: "session.end",
+            source: "worker",
+            agentId: opts.agentId,
+            taskId: effectiveTaskId,
+            sessionId: opts.runnerSessionId,
+            status: event.isError ? "error" : "ok",
+            durationMs: Date.now() - sessionStartTime,
+            data: {
+              model: event.cost.model,
+              totalCostUsd: event.cost.totalCostUsd,
+              inputTokens: event.cost.inputTokens,
+              outputTokens: event.cost.outputTokens,
+            },
           });
-          if (latestModel) {
-            reportLatestModel(opts.apiUrl, opts.apiKey, opts.agentId, latestModel).catch((err) =>
-              console.warn(`[runner] Failed to report latest model: ${err}`),
-            );
+          break;
+        case "error":
+          sessionSpan.setStatus({
+            code: 2,
+            message: event.message,
+          });
+          break;
+        case "context_usage": {
+          const now2 = Date.now();
+          if (now2 - lastContextPostTime >= CONTEXT_THROTTLE_MS) {
+            lastContextPostTime = now2;
+            // Phase 10: track cumulative output tokens on the worker side and
+            // forward them on every progress snapshot. Previously these were
+            // 0 until the `completion` snapshot at session end, so the
+            // dashboard's "tokens consumed" line was a flat zero throughout.
+            cumulativeProgressOutputTokens += event.outputTokens ?? 0;
+            // For inputs we don't get a per-turn delta on the `context_usage`
+            // event (the unified formula bakes it into contextUsedTokens), so
+            // we report the latest contextUsedTokens as the running input proxy.
+            // The DB column is `cumulativeInputTokens` but the semantic on
+            // progress rows is "running used-tokens" — both inputs and outputs
+            // contribute, exact decomposition lives on the cost row.
+            fetch(`${opts.apiUrl}/api/tasks/${realTaskId}/context`, {
+              method: "POST",
+              headers: {
+                "Content-Type": "application/json",
+                "X-Agent-ID": opts.agentId,
+                Authorization: `Bearer ${opts.apiKey}`,
+              },
+              body: JSON.stringify({
+                eventType: "progress",
+                sessionId: opts.runnerSessionId,
+                contextUsedTokens: event.contextUsedTokens,
+                contextTotalTokens: event.contextTotalTokens,
+                contextPercent: event.contextPercent,
+                cumulativeInputTokens: event.contextUsedTokens,
+                cumulativeOutputTokens: cumulativeProgressOutputTokens,
+                contextFormula: event.contextFormula,
+              }),
+            }).catch(() => {});
           }
+          break;
         }
-        // Cost save is handled in waitForCompletion().then() to ensure
-        // it completes before the process exits (fire-and-forget here
-        // races with container shutdown).
-
-        // Buffer session end event
-        bufferEvent({
-          category: "session",
-          event: "session.end",
-          source: "worker",
-          agentId: opts.agentId,
-          taskId: effectiveTaskId,
-          sessionId: opts.runnerSessionId,
-          status: event.isError ? "error" : "ok",
-          durationMs: Date.now() - sessionStartTime,
-          data: {
-            model: event.cost.model,
-            totalCostUsd: event.cost.totalCostUsd,
-            inputTokens: event.cost.inputTokens,
-            outputTokens: event.cost.outputTokens,
-          },
-        });
-        break;
-      case "context_usage": {
-        const now2 = Date.now();
-        if (now2 - lastContextPostTime >= CONTEXT_THROTTLE_MS) {
-          lastContextPostTime = now2;
+        case "compaction": {
+          // Always record compaction events (no throttle)
           fetch(`${opts.apiUrl}/api/tasks/${realTaskId}/context`, {
             method: "POST",
             headers: {
@@ -2024,153 +2407,186 @@ async function spawnProviderProcess(
               Authorization: `Bearer ${opts.apiKey}`,
             },
             body: JSON.stringify({
-              eventType: "progress",
+              eventType: "compaction",
               sessionId: opts.runnerSessionId,
-              contextUsedTokens: event.contextUsedTokens,
+              preCompactTokens: event.preCompactTokens,
+              compactTrigger: event.compactTrigger,
               contextTotalTokens: event.contextTotalTokens,
-              contextPercent: event.contextPercent,
             }),
           }).catch(() => {});
+          break;
         }
-        break;
-      }
-      case "compaction": {
-        // Always record compaction events (no throttle)
-        fetch(`${opts.apiUrl}/api/tasks/${realTaskId}/context`, {
-          method: "POST",
-          headers: {
-            "Content-Type": "application/json",
-            "X-Agent-ID": opts.agentId,
-            Authorization: `Bearer ${opts.apiKey}`,
-          },
-          body: JSON.stringify({
-            eventType: "compaction",
-            sessionId: opts.runnerSessionId,
-            preCompactTokens: event.preCompactTokens,
-            compactTrigger: event.compactTrigger,
-            contextTotalTokens: event.contextTotalTokens,
-          }),
-        }).catch(() => {});
-        break;
-      }
-      case "raw_log":
-        prettyPrintLine(event.content, opts.role);
-        if (shouldStream) {
-          logBuffer.lines.push(event.content);
-          const shouldFlush =
-            logBuffer.lines.length >= LOG_BUFFER_SIZE ||
-            Date.now() - logBuffer.lastFlush >= LOG_FLUSH_INTERVAL_MS;
-          if (shouldFlush) {
-            flushLogBuffer(logBuffer, {
-              apiUrl: opts.apiUrl,
-              apiKey: opts.apiKey,
-              agentId: opts.agentId,
-              sessionId: opts.runnerSessionId,
-              iteration: opts.iteration,
-              taskId: effectiveTaskId,
-              cli: adapter.name,
-            }).catch(() => {});
+        case "raw_log":
+          prettyPrintLine(event.content, opts.role);
+          if (shouldStream) {
+            logBuffer.lines.push(event.content);
+            const shouldFlush =
+              logBuffer.lines.length >= LOG_BUFFER_SIZE ||
+              Date.now() - logBuffer.lastFlush >= LOG_FLUSH_INTERVAL_MS;
+            if (shouldFlush) {
+              flushLogBuffer(logBuffer, {
+                apiUrl: opts.apiUrl,
+                apiKey: opts.apiKey,
+                agentId: opts.agentId,
+                sessionId: opts.runnerSessionId,
+                iteration: opts.iteration,
+                taskId: effectiveTaskId,
+                cli: adapter.name,
+              }).catch(() => {});
+            }
           }
-        }
-        break;
-      case "raw_stderr":
-        prettyPrintStderr(event.content, opts.role);
-        break;
-
-      case "progress": {
-        if (effectiveTaskId && opts.apiUrl) {
-          const now = Date.now();
-          if (now - lastProgressTime >= PROGRESS_THROTTLE_MS) {
-            lastProgressTime = now;
-            updateProgressViaAPI(opts.apiUrl, opts.apiKey, effectiveTaskId, event.message).catch(
-              () => {},
-            );
+          break;
+        case "raw_stderr":
+          prettyPrintStderr(event.content, opts.role);
+          break;
+
+        case "progress": {
+          if (effectiveTaskId && opts.apiUrl) {
+            const now = Date.now();
+            if (now - lastProgressTime >= PROGRESS_THROTTLE_MS) {
+              lastProgressTime = now;
+              updateProgressViaAPI(opts.apiUrl, opts.apiKey, effectiveTaskId, event.message).catch(
+                () => {},
+              );
+            }
           }
+          break;
         }
-        break;
       }
-    }
-  });
+    }),
+  );
 
-  // Create promise that handles completion
-  const promise: Promise<ProviderResult> = session.waitForCompletion().then(async (result) => {
-    // Stop event flush timer and do a final flush
-    clearInterval(eventFlushTimer);
-    await flushEvents();
-
-    // Final log flush
-    if (shouldStream && logBuffer.lines.length > 0) {
-      await flushLogBuffer(logBuffer, {
-        apiUrl: opts.apiUrl,
-        apiKey: opts.apiKey,
-        agentId: opts.agentId,
-        sessionId: opts.runnerSessionId,
-        iteration: opts.iteration,
-        taskId: effectiveTaskId,
-        cli: adapter.name,
+  function closeActiveToolSpans(status: "ok" | "error", message?: string) {
+    for (const [toolCallId, active] of activeToolSpans) {
+      active.span.setAttributes({
+        "agentswarm.tool.duration_ms": Date.now() - active.startedAt,
+        "agentswarm.tool.unclosed": true,
+        "agentswarm.tool.call_id": toolCallId,
       });
+      if (status === "error") {
+        active.span.setStatus({ code: 2, message: message || "session ended before tool_end" });
+      }
+      active.span.end();
+      activeToolSpans.delete(toolCallId);
     }
+  }
 
-    // Error logging for non-zero exit
-    if (result.exitCode !== 0) {
-      const errorLog = {
-        timestamp: new Date().toISOString(),
-        iteration: opts.iteration,
-        exitCode: result.exitCode,
-        taskId: effectiveTaskId,
-        error: true,
-      };
+  // Create promise that handles completion
+  const promise: Promise<ProviderResult> = session
+    .waitForCompletion()
+    .then((result) =>
+      withSpanContext(sessionSpan, async () => {
+        // Stop event flush timer and do a final flush
+        clearInterval(eventFlushTimer);
+        await flushEvents();
+
+        // Final log flush
+        if (shouldStream && logBuffer.lines.length > 0) {
+          await flushLogBuffer(logBuffer, {
+            apiUrl: opts.apiUrl,
+            apiKey: opts.apiKey,
+            agentId: opts.agentId,
+            sessionId: opts.runnerSessionId,
+            iteration: opts.iteration,
+            taskId: effectiveTaskId,
+            cli: adapter.name,
+          });
+        }
 
-      const errorsFile = `${logDir}/errors.jsonl`;
-      const errorsFileRef = Bun.file(errorsFile);
-      const existingErrors = (await errorsFileRef.exists()) ? await errorsFileRef.text() : "";
-      await Bun.write(errorsFile, `${existingErrors}${JSON.stringify(errorLog)}\n`);
+        // Error logging for non-zero exit
+        if (result.exitCode !== 0) {
+          const errorLog = {
+            timestamp: new Date().toISOString(),
+            iteration: opts.iteration,
+            exitCode: result.exitCode,
+            taskId: effectiveTaskId,
+            error: true,
+          };
 
-      if (!isYolo) {
-        console.error(
-          `[${opts.role}] Task ${effectiveTaskId.slice(0, 8)} exited with code ${result.exitCode}.`,
-        );
-      } else {
-        console.warn(
-          `[${opts.role}] Task ${effectiveTaskId.slice(0, 8)} exited with code ${result.exitCode}. YOLO mode - continuing...`,
-        );
-      }
-    }
+          const errorsFile = `${logDir}/errors.jsonl`;
+          const errorsFileRef = Bun.file(errorsFile);
+          const existingErrors = (await errorsFileRef.exists()) ? await errorsFileRef.text() : "";
+          await Bun.write(errorsFile, `${existingErrors}${JSON.stringify(errorLog)}\n`);
 
-    // Save cost data (awaited to ensure it completes before container exits)
-    if (result.cost) {
-      try {
-        await saveCostData(
-          { ...result.cost, taskId: realTaskId, sessionId: opts.runnerSessionId },
-          opts.apiUrl,
-          opts.apiKey,
-        );
-      } catch (err) {
-        console.warn(`[runner] Failed to save cost: ${err}`);
-      }
-    }
+          if (!isYolo) {
+            console.error(
+              `[${opts.role}] Task ${effectiveTaskId.slice(0, 8)} exited with code ${result.exitCode}.`,
+            );
+          } else {
+            console.warn(
+              `[${opts.role}] Task ${effectiveTaskId.slice(0, 8)} exited with code ${result.exitCode}. YOLO mode - continuing...`,
+            );
+          }
+        }
 
-    // Post completion context usage snapshot
-    if (result.cost && realTaskId) {
-      fetch(`${opts.apiUrl}/api/tasks/${realTaskId}/context`, {
-        method: "POST",
-        headers: {
-          "Content-Type": "application/json",
-          "X-Agent-ID": opts.agentId,
-          Authorization: `Bearer ${opts.apiKey}`,
-        },
-        body: JSON.stringify({
-          eventType: "completion",
-          sessionId: opts.runnerSessionId,
-          cumulativeInputTokens: result.cost.inputTokens ?? 0,
-          cumulativeOutputTokens: result.cost.outputTokens ?? 0,
-          contextTotalTokens: getContextWindowSize(result.cost.model || "default"),
-        }),
-      }).catch(() => {});
-    }
+        // Save cost data (awaited to ensure it completes before container exits)
+        if (result.cost) {
+          sessionSpan.setAttributes({
+            "gen_ai.response.model": result.cost.model,
+            "gen_ai.usage.input_tokens": result.cost.inputTokens ?? 0,
+            "gen_ai.usage.output_tokens": result.cost.outputTokens ?? 0,
+            "agentswarm.cost.total_usd": result.cost.totalCostUsd ?? 0,
+          });
+          try {
+            await saveCostData(
+              { ...result.cost, taskId: realTaskId, sessionId: opts.runnerSessionId },
+              opts.apiUrl,
+              opts.apiKey,
+            );
+          } catch (err) {
+            console.warn(`[runner] Failed to save cost: ${err}`);
+          }
+        }
 
-    return result;
-  });
+        // Post completion context usage snapshot
+        if (result.cost && realTaskId) {
+          fetch(`${opts.apiUrl}/api/tasks/${realTaskId}/context`, {
+            method: "POST",
+            headers: {
+              "Content-Type": "application/json",
+              "X-Agent-ID": opts.agentId,
+              Authorization: `Bearer ${opts.apiKey}`,
+            },
+            body: JSON.stringify({
+              eventType: "completion",
+              sessionId: opts.runnerSessionId,
+              cumulativeInputTokens: result.cost.inputTokens ?? 0,
+              cumulativeOutputTokens: result.cost.outputTokens ?? 0,
+              contextTotalTokens: getContextWindowSize(result.cost.model || "default"),
+            }),
+          }).catch(() => {});
+        }
+
+        sessionSpan.setAttributes({
+          "agentswarm.session.duration_ms": Date.now() - sessionStartTime,
+          "agentswarm.session.exit_code": result.exitCode,
+          "agentswarm.session.outcome": result.exitCode === 0 ? "ok" : "error",
+        });
+        if (result.exitCode !== 0) {
+          sessionSpan.setStatus({
+            code: 2,
+            message: result.failureReason || `Provider exited with ${result.exitCode}`,
+          });
+        }
+        closeActiveToolSpans(result.exitCode === 0 ? "ok" : "error", result.failureReason);
+        sessionSpan.end();
+
+        return result;
+      }),
+    )
+    .catch((error) =>
+      withSpanContext(sessionSpan, () => {
+        clearInterval(eventFlushTimer);
+        sessionSpan.recordException(error);
+        sessionSpan.setStatus({
+          code: 2,
+          message: error instanceof Error ? error.message : String(error),
+        });
+        closeActiveToolSpans("error", error instanceof Error ? error.message : String(error));
+        sessionSpan.end();
+        throw error;
+      }),
+    );
 
   // Build credential info for rate limit tracking
   const primarySelection = credentialSelections[0] ?? oauthSelection;
@@ -2402,6 +2818,7 @@ export async function runAgent(config: RunnerConfig, opts: RunnerOptions) {
 
   // Initialize Business-Use SDK for worker-side instrumentation
   initialize();
+  await initOtel(role);
 
   const sessionId = process.env.SESSION_ID || crypto.randomUUID().slice(0, 8);
   const baseLogDir = opts.logsDir || process.env.LOG_DIR || "/logs";
@@ -2417,7 +2834,7 @@ export async function runAgent(config: RunnerConfig, opts: RunnerOptions) {
 
   const apiUrl = process.env.MCP_BASE_URL || `http://localhost:${process.env.PORT || "3013"}`;
   const swarmUrl = process.env.SWARM_URL || "localhost";
-  const apiKey = process.env.API_KEY || "";
+  const apiKey = getApiKey();
 
   // Resolve the boot harness provider from swarm_config (repo > agent > global,
   // overlaid on top of `process.env`). This is what selects the adapter for
@@ -2443,8 +2860,8 @@ export async function runAgent(config: RunnerConfig, opts: RunnerOptions) {
   let adapter = createProviderAdapter(bootProvider);
 
   // Configure HTTP-based template resolution (workers resolve via API, not local DB)
-  if (process.env.API_KEY) {
-    configureHttpResolver(apiUrl, process.env.API_KEY);
+  if (apiKey) {
+    configureHttpResolver(apiUrl, apiKey);
   }
 
   // Initialize anonymized telemetry (opt-out via ANONYMIZED_TELEMETRY=false).
@@ -2455,7 +2872,7 @@ export async function runAgent(config: RunnerConfig, opts: RunnerOptions) {
   // skips telemetry instead of minting a fresh `install_<hex>` ID per
   // restart, which floods prod metrics with phantom installs.
   {
-    const telemetryApiKey = process.env.API_KEY;
+    const telemetryApiKey = apiKey || undefined;
     await initTelemetry(
       "worker",
       async (key) => {