@dogpile/sdk 0.4.0 → 0.6.0

package/src/runtime/engine.ts

@@ -1,7 +1,9 @@
 import { DogpileError } from "../types.js";
 import type {
   AbortedEvent,
+  BudgetStopEvent,
   BudgetTier,
+  CostSummary,
   DogpileErrorCode,
   DogpileOptions,
   Engine,
@@ -9,11 +11,13 @@ import type {
   FinalEvent,
   JsonObject,
   JsonValue,
+  ModelRequestEvent,
   ProtocolSelection,
   RunCallOptions,
   RunEvaluation,
   RunEvent,
   RunResult,
+  ReplayTraceProviderCall,
   SubRunFailedEvent,
   StreamErrorEvent,
   StreamEvent,
@@ -25,6 +29,7 @@ import type {
 import { runBroadcast } from "./broadcast.js";
 import { runCoordinator, type AbortDrainFn } from "./coordinator.js";
 import {
+  addCost,
   createReplayTraceFinalOutput,
   createReplayTraceBudgetStateChanges,
   canonicalizeRunResult,
@@ -34,12 +39,14 @@ import {
   createRunMetadata,
   createRunUsage,
   defaultAgents,
+  emptyCost,
   normalizeProtocol,
   orderAgentsForTemperature,
   recomputeAccountingFromTrace,
   resolveOnChildFailure,
   tierTemperature
 } from "./defaults.js";
+import { computeHealth, DEFAULT_HEALTH_THRESHOLDS } from "./health.js";
 import { runSequential } from "./sequential.js";
 import { runShared } from "./shared.js";
 import {
@@ -56,9 +63,13 @@ import {
   validateProviderLocality,
   validateRunCallOptions
 } from "./validation.js";
+import { DOGPILE_SPAN_NAMES, type DogpileSpan, type DogpileTracer } from "./tracing.js";
+import type { Logger } from "./logger.js";
+import type { MetricsHook, RunMetricsSnapshot } from "./metrics.js";
 
 const DEFAULT_MAX_DEPTH = 4;
 const DEFAULT_MAX_CONCURRENT_CHILDREN = 4;
+const DEFAULT_MAX_CONCURRENT_AGENT_TURNS = 4;
 
 const defaultHighLevelProtocol = "sequential";
 const defaultHighLevelTier = "balanced";
@@ -89,6 +100,7 @@ export function createEngine(options: EngineOptions): Engine {
   const terminate = options.terminate ?? (options.budget ? conditionFromBudget(options.budget) : undefined);
   const engineMaxDepth = options.maxDepth ?? DEFAULT_MAX_DEPTH;
   const engineMaxConcurrentChildren = options.maxConcurrentChildren ?? DEFAULT_MAX_CONCURRENT_CHILDREN;
+  const engineMaxConcurrentAgentTurns = options.maxConcurrentAgentTurns ?? DEFAULT_MAX_CONCURRENT_AGENT_TURNS;
   const engineOnChildFailure = options.onChildFailure;
 
   return {
@@ -110,6 +122,15 @@ export function createEngine(options: EngineOptions): Engine {
         engineMaxConcurrentChildren,
         runOptions?.maxConcurrentChildren ?? Number.POSITIVE_INFINITY
       );
+      assertRunDoesNotRaiseEngineMax(
+        "maxConcurrentAgentTurns",
+        runOptions?.maxConcurrentAgentTurns,
+        engineMaxConcurrentAgentTurns
+      );
+      const effectiveMaxConcurrentAgentTurns = Math.min(
+        engineMaxConcurrentAgentTurns,
+        runOptions?.maxConcurrentAgentTurns ?? Number.POSITIVE_INFINITY
+      );
       const onChildFailure = resolveOnChildFailure(runOptions?.onChildFailure, engineOnChildFailure);
 
       const startedAtMs = Date.now();
@@ -130,9 +151,13 @@ export function createEngine(options: EngineOptions): Engine {
         ...(terminate ? { terminate } : {}),
         ...(options.wrapUpHint ? { wrapUpHint: options.wrapUpHint } : {}),
         ...(options.evaluate ? { evaluate: options.evaluate } : {}),
+        ...(options.tracer ? { tracer: options.tracer } : {}),
+        ...(options.metricsHook ? { metricsHook: options.metricsHook } : {}),
+        ...(options.logger ? { logger: options.logger } : {}),
         currentDepth: 0,
         effectiveMaxDepth,
         effectiveMaxConcurrentChildren,
+        effectiveMaxConcurrentAgentTurns,
         onChildFailure,
         ...(parentDeadlineMs !== undefined ? { parentDeadlineMs } : {}),
         ...(options.defaultSubRunTimeoutMs !== undefined
@@ -159,10 +184,18 @@ export function createEngine(options: EngineOptions): Engine {
         engineMaxConcurrentChildren,
         runOptions?.maxConcurrentChildren ?? Number.POSITIVE_INFINITY
       );
+      assertRunDoesNotRaiseEngineMax(
+        "maxConcurrentAgentTurns",
+        runOptions?.maxConcurrentAgentTurns,
+        engineMaxConcurrentAgentTurns
+      );
+      const effectiveMaxConcurrentAgentTurns = Math.min(
+        engineMaxConcurrentAgentTurns,
+        runOptions?.maxConcurrentAgentTurns ?? Number.POSITIVE_INFINITY
+      );
       const onChildFailure = resolveOnChildFailure(runOptions?.onChildFailure, engineOnChildFailure);
 
-      const pendingEvents: StreamEvent[] = [];
-      const pendingResolvers: Array<(value: IteratorResult<StreamEvent>) => void> = [];
+      const pendingIteratorResolvers: Array<() => void> = [];
       const emittedEvents: StreamEvent[] = [];
       const subscribers = new Set<StreamEventSubscriber>();
       const abortController = new AbortController();
@@ -199,12 +232,20 @@ export function createEngine(options: EngineOptions): Engine {
           cancelRun();
         },
         subscribe(subscriber: StreamEventSubscriber) {
-          subscribers.add(subscriber);
-
           for (const event of emittedEvents) {
             subscriber(event);
           }
 
+          if (complete) {
+            return {
+              unsubscribe(): void {
+                // Completed streams replay synchronously and have no live source.
+              }
+            };
+          }
+
+          subscribers.add(subscriber);
+
           return {
             unsubscribe(): void {
               subscribers.delete(subscriber);
@@ -212,20 +253,29 @@ export function createEngine(options: EngineOptions): Engine {
           };
         },
         [Symbol.asyncIterator](): AsyncIterator<StreamEvent> {
+          let cursor = 0;
+
           return {
             next(): Promise<IteratorResult<StreamEvent>> {
-              const event = pendingEvents.shift();
-              if (event) {
-                return Promise.resolve({ done: false, value: event });
-              }
-              if (complete) {
-                return Promise.resolve({ done: true, value: undefined });
-              }
-              return new Promise<IteratorResult<StreamEvent>>((resolve) => {
-                pendingResolvers.push(resolve);
-              });
+              return readNext();
             }
           };
+
+          function readNext(): Promise<IteratorResult<StreamEvent>> {
+            const event = emittedEvents[cursor];
+            if (event !== undefined) {
+              cursor += 1;
+              return Promise.resolve({ done: false, value: event });
+            }
+            if (complete) {
+              return Promise.resolve({ done: true, value: undefined });
+            }
+            return new Promise<IteratorResult<StreamEvent>>((resolve) => {
+              pendingIteratorResolvers.push(() => {
+                void readNext().then(resolve);
+              });
+            });
+          }
         }
       };
 
@@ -253,11 +303,15 @@ export function createEngine(options: EngineOptions): Engine {
             currentDepth: 0,
             effectiveMaxDepth,
             effectiveMaxConcurrentChildren,
+            effectiveMaxConcurrentAgentTurns,
             onChildFailure,
             ...(streamParentDeadlineMs !== undefined ? { parentDeadlineMs: streamParentDeadlineMs } : {}),
             ...(options.defaultSubRunTimeoutMs !== undefined
               ? { defaultSubRunTimeoutMs: options.defaultSubRunTimeoutMs }
               : {}),
+            ...(options.tracer ? { tracer: options.tracer } : {}),
+            ...(options.metricsHook ? { metricsHook: options.metricsHook } : {}),
+            ...(options.logger ? { logger: options.logger } : {}),
             streamEvents: true,
             emit(event: RunEvent): void {
               if (status !== "running") {
@@ -346,8 +400,8 @@ export function createEngine(options: EngineOptions): Engine {
         timeoutLifecycle.cleanup();
         abortRace.cleanup();
         subscribers.clear();
-        for (const resolver of pendingResolvers.splice(0)) {
-          resolver({ done: true, value: undefined });
+        for (const resolvePending of pendingIteratorResolvers.splice(0)) {
+          resolvePending();
         }
       }
 
@@ -367,12 +421,9 @@ export function createEngine(options: EngineOptions): Engine {
           }
         }
 
-        const resolver = pendingResolvers.shift();
-        if (resolver) {
-          resolver({ done: false, value: canonicalEvent });
-          return;
+        for (const resolvePending of pendingIteratorResolvers.splice(0)) {
+          resolvePending();
         }
-        pendingEvents.push(canonicalEvent);
       }
     }
   };
@@ -659,15 +710,17 @@ interface RunProtocolOptions {
   /**
    * Current recursion depth. Top-level runs use 0; the coordinator dispatch
    * loop increments before invoking {@link runProtocol} for a child run.
-   * Plan 04 will wire `effectiveMaxDepth` validation around this value.
+   * Depth validation is enforced before delegate dispatch.
    */
   readonly currentDepth?: number;
   /**
-   * Effective max recursion depth. Plan 04 enforces; Plan 03 plumbs the param.
+   * Effective max recursion depth after engine and per-run ceilings are merged.
    */
   readonly effectiveMaxDepth?: number;
   /** Effective max delegated child concurrency resolved at run start. */
   readonly effectiveMaxConcurrentChildren?: number;
+  /** Effective max agent-turn fan-out resolved at run start. */
+  readonly effectiveMaxConcurrentAgentTurns?: number;
   readonly onChildFailure?: EngineOptions["onChildFailure"];
   /**
    * Root-run deadline (epoch ms) threaded through every recursive coordinator
@@ -682,10 +735,489 @@ interface RunProtocolOptions {
   readonly defaultSubRunTimeoutMs?: number;
   readonly registerAbortDrain?: (drain: AbortDrainFn) => void;
   readonly failureInstancesByChildRunId?: Map<string, DogpileError>;
+  readonly tracer?: EngineOptions["tracer"];
+  readonly metricsHook?: EngineOptions["metricsHook"];
+  readonly logger?: EngineOptions["logger"];
+  /**
+   * Optional parent span for the next runProtocol invocation. Threaded by the
+   * coordinator when dispatching child runs so that the child's `dogpile.run`
+   * span is correctly nested under its parent's `dogpile.sub-run` span.
+   * Internal-only; not part of the public surface.
+   */
+  readonly parentSpan?: DogpileSpan;
+  /**
+   * Per-child sub-run span lookup, keyed by childRunId. Populated by the
+   * parent's emit closure on `sub-run-started`. The coordinator dispatcher
+   * reads this to thread the correct per-child span as parent for the
+   * recursive runProtocol call. Internal-only.
+   */
+  readonly subRunSpansByChildId?: ReadonlyMap<string, DogpileSpan>;
 }
 
 type NonStreamingProtocolOptions = Omit<RunProtocolOptions, "emit"> & Pick<EngineOptions, "evaluate">;
 
+interface TracingState {
+  readonly tracer: DogpileTracer;
+  readonly runSpan: DogpileSpan;
+  readonly subRunSpans: Map<string, DogpileSpan>;
+  readonly agentTurnSpans: Map<string, DogpileSpan>;
+  readonly modelCallSpans: Map<string, DogpileSpan>;
+  readonly pendingModelRequests: Map<string, ModelRequestEvent>;
+  readonly agentTurnCounters: Map<string, number>;
+  readonly turnAccumByAgent: Map<string, TurnAccum>;
+  readonly agentIds: Set<string>;
+  runId?: string;
+  turnCount: number;
+  lastCost: CostSummary;
+}
+
+interface TurnAccum {
+  inputTokens: number;
+  outputTokens: number;
+  costUsd: number;
+}
+
+function openRunTracing(options: {
+  readonly tracer?: DogpileTracer;
+  readonly parentSpan?: DogpileSpan;
+  readonly intent: string;
+  readonly protocolKind: string;
+  readonly tier: unknown;
+}): TracingState | undefined {
+  if (!options.tracer) {
+    return undefined;
+  }
+
+  const runSpan = options.tracer.startSpan(DOGPILE_SPAN_NAMES.RUN, {
+    ...(options.parentSpan ? { parent: options.parentSpan } : {}),
+    attributes: {
+      "dogpile.run.protocol": options.protocolKind,
+      "dogpile.run.tier": String(options.tier),
+      "dogpile.run.intent": options.intent.slice(0, 200)
+    }
+  });
+
+  return {
+    tracer: options.tracer,
+    runSpan,
+    subRunSpans: new Map(),
+    agentTurnSpans: new Map(),
+    modelCallSpans: new Map(),
+    pendingModelRequests: new Map(),
+    agentTurnCounters: new Map(),
+    turnAccumByAgent: new Map(),
+    agentIds: new Set(),
+    turnCount: 0,
+    lastCost: emptyCost()
+  };
+}
+
+interface MetricsState {
+  readonly metricsHook: MetricsHook;
+  readonly logger: Logger | undefined;
+  readonly startedAtMs: number;
+  readonly subRunStartTimes: Map<string, number>;
+  totalCost: CostSummary;
+  nestedCost: CostSummary;
+  turns: number;
+}
+
+function openRunMetrics(options: {
+  readonly metricsHook?: MetricsHook;
+  readonly logger?: Logger;
+}): MetricsState | undefined {
+  if (!options.metricsHook) {
+    return undefined;
+  }
+
+  return {
+    metricsHook: options.metricsHook,
+    logger: options.logger,
+    startedAtMs: Date.now(),
+    subRunStartTimes: new Map(),
+    totalCost: emptyCost(),
+    nestedCost: emptyCost(),
+    turns: 0
+  };
+}
+
+function routeMetricsError(err: unknown, logger: Logger | undefined): void {
+  const msg = err instanceof Error ? err.message : String(err);
+  try {
+    if (logger !== undefined) {
+      logger.error("dogpile:metricsHook threw", { error: msg });
+    } else {
+      console.error("dogpile:metricsHook threw", { error: msg });
+    }
+  } catch {
+    // A logger that throws from error() cannot be helped.
+  }
+}
+
+function fireHook(
+  callback: ((snapshot: RunMetricsSnapshot) => void | Promise<void>) | undefined,
+  snapshot: RunMetricsSnapshot,
+  logger: Logger | undefined
+): void {
+  if (!callback) {
+    return;
+  }
+
+  try {
+    const result = callback(snapshot);
+    if (result && typeof (result as Promise<void>).catch === "function") {
+      (result as Promise<void>).catch((err: unknown) => {
+        routeMetricsError(err, logger);
+      });
+    }
+  } catch (err: unknown) {
+    routeMetricsError(err, logger);
+  }
+}
+
+function buildRunSnapshot(
+  result: RunResult,
+  startedAtMs: number
+): RunMetricsSnapshot {
+  const nestedCosts = nestedSubRunCosts(result);
+  const budgetStopEvent = result.trace.events.find((event): event is BudgetStopEvent => event.type === "budget-stop");
+  const outcome: RunMetricsSnapshot["outcome"] = budgetStopEvent !== undefined ? "budget-stopped" : "completed";
+  const totalInputTokens = result.cost.inputTokens;
+  const totalOutputTokens = result.cost.outputTokens;
+  const totalCostUsd = result.cost.usd;
+  const ownInputTokens =
+    totalInputTokens - nestedCosts.reduce((sum, cost) => sum + cost.inputTokens, 0);
+  const ownOutputTokens =
+    totalOutputTokens - nestedCosts.reduce((sum, cost) => sum + cost.outputTokens, 0);
+  const ownCostUsd =
+    totalCostUsd - nestedCosts.reduce((sum, cost) => sum + cost.usd, 0);
+  const turns = result.trace.events.filter((event) => event.type === "agent-turn").length;
+
+  return {
+    outcome,
+    inputTokens: ownInputTokens,
+    outputTokens: ownOutputTokens,
+    costUsd: ownCostUsd,
+    totalInputTokens,
+    totalOutputTokens,
+    totalCostUsd,
+    turns,
+    durationMs: Date.now() - startedAtMs
+  };
+}
+
+function buildSubRunSnapshot(
+  subResult: RunResult,
+  durationMs: number
+): RunMetricsSnapshot {
+  const nestedCosts = nestedSubRunCosts(subResult);
+  const budgetStopEvent = subResult.trace.events.find((event): event is BudgetStopEvent => event.type === "budget-stop");
+  const outcome: RunMetricsSnapshot["outcome"] = budgetStopEvent !== undefined ? "budget-stopped" : "completed";
+  const totalInputTokens = subResult.cost.inputTokens;
+  const totalOutputTokens = subResult.cost.outputTokens;
+  const totalCostUsd = subResult.cost.usd;
+  const ownInputTokens =
+    totalInputTokens - nestedCosts.reduce((sum, cost) => sum + cost.inputTokens, 0);
+  const ownOutputTokens =
+    totalOutputTokens - nestedCosts.reduce((sum, cost) => sum + cost.outputTokens, 0);
+  const ownCostUsd =
+    totalCostUsd - nestedCosts.reduce((sum, cost) => sum + cost.usd, 0);
+  const turns = subResult.trace.events.filter((event) => event.type === "agent-turn").length;
+
+  return {
+    outcome,
+    inputTokens: ownInputTokens,
+    outputTokens: ownOutputTokens,
+    costUsd: ownCostUsd,
+    totalInputTokens,
+    totalOutputTokens,
+    totalCostUsd,
+    turns,
+    durationMs
+  };
+}
+
+function nestedSubRunCosts(result: RunResult): CostSummary[] {
+  return result.trace.events.flatMap((event) => {
+    if (event.type === "sub-run-completed") {
+      return [event.subResult.cost];
+    }
+    if (event.type === "sub-run-failed") {
+      return [event.partialCost];
+    }
+    return [];
+  });
+}
+
+function subtractCost(total: CostSummary, nested: CostSummary): CostSummary {
+  return {
+    usd: total.usd - nested.usd,
+    inputTokens: total.inputTokens - nested.inputTokens,
+    outputTokens: total.outputTokens - nested.outputTokens,
+    totalTokens: total.totalTokens - nested.totalTokens
+  };
+}
+
+function handleMetricsEvent(state: MetricsState, event: RunEvent): void {
+  const parentRunIds = (event as { readonly parentRunIds?: readonly string[] }).parentRunIds;
+  if (parentRunIds !== undefined) {
+    return;
+  }
+
+  switch (event.type) {
+    case "agent-turn": {
+      state.totalCost = event.cost;
+      state.turns += 1;
+      break;
+    }
+    case "broadcast":
+    case "budget-stop":
+    case "final": {
+      state.totalCost = event.cost;
+      break;
+    }
+    case "sub-run-started": {
+      state.subRunStartTimes.set(event.childRunId, Date.now());
+      break;
+    }
+    case "sub-run-completed": {
+      state.totalCost = addCost(state.totalCost, event.subResult.cost);
+      state.nestedCost = addCost(state.nestedCost, event.subResult.cost);
+      const startMs = state.subRunStartTimes.get(event.childRunId);
+      const durationMs = startMs !== undefined ? Date.now() - startMs : 0;
+      state.subRunStartTimes.delete(event.childRunId);
+      const snapshot = buildSubRunSnapshot(event.subResult, durationMs);
+      fireHook(state.metricsHook.onSubRunComplete, snapshot, state.logger);
+      break;
+    }
+    case "sub-run-failed": {
+      state.totalCost = addCost(state.totalCost, event.partialCost);
+      state.nestedCost = addCost(state.nestedCost, event.partialCost);
+      state.subRunStartTimes.delete(event.childRunId);
+      break;
+    }
+    default:
+      break;
+  }
+}
+
+function closeRunMetrics(state: MetricsState, result: RunResult | undefined): void {
+  if (result !== undefined) {
+    const snapshot = buildRunSnapshot(result, state.startedAtMs);
+    fireHook(state.metricsHook.onRunComplete, snapshot, state.logger);
+    return;
+  }
+
+  const ownCost = subtractCost(state.totalCost, state.nestedCost);
+  const snapshot: RunMetricsSnapshot = {
+    outcome: "aborted",
+    inputTokens: ownCost.inputTokens,
+    outputTokens: ownCost.outputTokens,
+    costUsd: ownCost.usd,
+    totalInputTokens: state.totalCost.inputTokens,
+    totalOutputTokens: state.totalCost.outputTokens,
+    totalCostUsd: state.totalCost.usd,
+    turns: state.turns,
+    durationMs: Date.now() - state.startedAtMs
+  };
+  fireHook(state.metricsHook.onRunComplete, snapshot, state.logger);
+}
+
+function handleTracingEvent(state: TracingState, event: RunEvent): void {
+  const parentRunIds = (event as { readonly parentRunIds?: readonly string[] }).parentRunIds;
+  if (parentRunIds !== undefined) {
+    return;
+  }
+
+  if (state.runId === undefined) {
+    state.runId = event.runId;
+    state.runSpan.setAttribute("dogpile.run.id", event.runId);
+  }
+
+  switch (event.type) {
+    case "model-request": {
+      state.pendingModelRequests.set(event.callId, event);
+      state.agentIds.add(event.agentId);
+
+      if (!state.agentTurnSpans.has(event.agentId)) {
+        const turnNumber = (state.agentTurnCounters.get(event.agentId) ?? 0) + 1;
+        state.agentTurnCounters.set(event.agentId, turnNumber);
+        const turnParent = state.subRunSpans.get(event.runId) ?? state.runSpan;
+        const turnSpan = state.tracer.startSpan(DOGPILE_SPAN_NAMES.AGENT_TURN, {
+          parent: turnParent,
+          attributes: {
+            "dogpile.agent.id": event.agentId,
+            "dogpile.agent.role": event.role,
+            "dogpile.turn.number": turnNumber,
+            "dogpile.model.id": event.modelId
+          }
+        });
+        state.agentTurnSpans.set(event.agentId, turnSpan);
+      }
+
+      const callParent =
+        state.agentTurnSpans.get(event.agentId) ??
+        state.subRunSpans.get(event.runId) ??
+        state.runSpan;
+      const callSpan = state.tracer.startSpan(DOGPILE_SPAN_NAMES.MODEL_CALL, {
+        parent: callParent,
+        attributes: {
+          "dogpile.model.id": event.modelId,
+          "dogpile.call.id": event.callId,
+          "dogpile.provider.id": event.providerId
+        }
+      });
+      state.modelCallSpans.set(event.callId, callSpan);
+      break;
+    }
+    case "model-response": {
+      const span = state.modelCallSpans.get(event.callId);
+      if (span) {
+        const inputTokens = event.response.usage?.inputTokens ?? 0;
+        const outputTokens = event.response.usage?.outputTokens ?? 0;
+        const responseCost: CostSummary = {
+          usd: event.response.costUsd ?? 0,
+          inputTokens,
+          outputTokens,
+          totalTokens: event.response.usage?.totalTokens ?? inputTokens + outputTokens
+        };
+        span.setAttribute("dogpile.model.input_tokens", inputTokens);
+        span.setAttribute("dogpile.model.output_tokens", outputTokens);
+        if (event.response.costUsd !== undefined) {
+          span.setAttribute("dogpile.model.cost_usd", event.response.costUsd);
+        }
+        span.setStatus("ok");
+        span.end();
+        state.modelCallSpans.delete(event.callId);
+        const accum = state.turnAccumByAgent.get(event.agentId) ?? {
+          inputTokens: 0,
+          outputTokens: 0,
+          costUsd: 0
+        };
+        accum.inputTokens += inputTokens;
+        accum.outputTokens += outputTokens;
+        accum.costUsd += responseCost.usd;
+        state.turnAccumByAgent.set(event.agentId, accum);
+        state.lastCost = addCost(state.lastCost, responseCost);
+      }
+      state.pendingModelRequests.delete(event.callId);
+      break;
+    }
+    case "agent-turn": {
+      state.agentIds.add(event.agentId);
+      state.turnCount += 1;
+      state.lastCost = event.cost;
+      const turnSpan = state.agentTurnSpans.get(event.agentId);
+      if (turnSpan) {
+        turnSpan.setAttribute("dogpile.agent.role", event.role);
+        const accum = state.turnAccumByAgent.get(event.agentId);
+        turnSpan.setAttribute("dogpile.turn.cost_usd", accum?.costUsd ?? 0);
+        turnSpan.setAttribute("dogpile.turn.input_tokens", accum?.inputTokens ?? 0);
+        turnSpan.setAttribute("dogpile.turn.output_tokens", accum?.outputTokens ?? 0);
+        turnSpan.setStatus("ok");
+        turnSpan.end();
+        state.agentTurnSpans.delete(event.agentId);
+      }
+      state.turnAccumByAgent.delete(event.agentId);
+      break;
+    }
+    case "broadcast":
+    case "budget-stop":
+    case "final": {
+      state.lastCost = event.cost;
+      break;
+    }
+    case "sub-run-started": {
+      const span = state.tracer.startSpan(DOGPILE_SPAN_NAMES.SUB_RUN, {
+        parent: state.runSpan,
+        attributes: {
+          "dogpile.sub_run.child_run_id": event.childRunId,
+          "dogpile.sub_run.parent_run_id": event.parentRunId,
+          "dogpile.sub_run.depth": event.depth
+        }
+      });
+      state.subRunSpans.set(event.childRunId, span);
+      break;
+    }
+    case "sub-run-completed": {
+      const span = state.subRunSpans.get(event.childRunId);
+      if (span) {
+        span.setStatus("ok");
+        span.end();
+        state.subRunSpans.delete(event.childRunId);
+      }
+      break;
+    }
+    case "sub-run-failed": {
+      const span = state.subRunSpans.get(event.childRunId);
+      if (span) {
+        span.setStatus("error", event.error.message);
+        span.end();
+        state.subRunSpans.delete(event.childRunId);
+      }
+      break;
+    }
+    default:
+      break;
+  }
+}
+
+function closeRunTracing(state: TracingState, result: RunResult | undefined, error?: unknown): void {
+  if (error !== undefined) {
+    if (state.runId !== undefined) {
+      state.runSpan.setAttribute("dogpile.run.id", state.runId);
+    }
+    state.runSpan.setAttribute("dogpile.run.agent_count", state.agentIds.size);
+    state.runSpan.setAttribute("dogpile.run.turn_count", state.turnCount);
+    state.runSpan.setAttribute("dogpile.run.cost_usd", state.lastCost.usd);
+    state.runSpan.setAttribute("dogpile.run.input_tokens", state.lastCost.inputTokens);
+    state.runSpan.setAttribute("dogpile.run.output_tokens", state.lastCost.outputTokens);
+    state.runSpan.setAttribute("dogpile.run.outcome", "aborted");
+    state.runSpan.setStatus("error", error instanceof Error ? error.message : String(error));
+    closeOpenTracingSpans(state);
+    state.runSpan.end();
+    return;
+  }
+
+  if (result === undefined) {
+    closeOpenTracingSpans(state);
+    state.runSpan.end();
+    return;
+  }
+
+  const budgetStopEvent = result.trace.events.find((event): event is BudgetStopEvent => event.type === "budget-stop");
+  const terminationReason = budgetStopEvent?.reason;
+  const outcome = terminationReason !== undefined ? "budget-stopped" : "completed";
+  state.runSpan.setAttribute("dogpile.run.id", result.trace.runId);
+  state.runSpan.setAttribute("dogpile.run.agent_count", result.trace.agentsUsed.length);
+  state.runSpan.setAttribute("dogpile.run.turn_count", result.trace.events.filter((event) => event.type === "agent-turn").length);
+  state.runSpan.setAttribute("dogpile.run.cost_usd", result.cost.usd);
+  state.runSpan.setAttribute("dogpile.run.input_tokens", result.cost.inputTokens);
+  state.runSpan.setAttribute("dogpile.run.output_tokens", result.cost.outputTokens);
+  state.runSpan.setAttribute("dogpile.run.outcome", outcome);
+  if (terminationReason !== undefined) {
+    state.runSpan.setAttribute("dogpile.run.termination_reason", terminationReason);
+  }
+  state.runSpan.setStatus("ok");
+  closeOpenTracingSpans(state);
+  state.runSpan.end();
+}
+
+function closeOpenTracingSpans(state: TracingState): void {
+  for (const span of state.modelCallSpans.values()) {
+    span.end();
+  }
+  state.modelCallSpans.clear();
+  for (const span of state.agentTurnSpans.values()) {
+    span.end();
+  }
+  state.agentTurnSpans.clear();
+  for (const span of state.subRunSpans.values()) {
+    span.end();
+  }
+  state.subRunSpans.clear();
+}
+
 async function runNonStreamingProtocol(options: NonStreamingProtocolOptions): Promise<RunResult> {
   const failureInstancesByChildRunId = new Map<string, DogpileError>();
   const abortLifecycle = createNonStreamingAbortLifecycle({
@@ -728,7 +1260,8 @@ async function runNonStreamingProtocol(options: NonStreamingProtocolOptions): Pr
         events
       }),
       eventLog: createRunEventLog(trace.runId, trace.protocol, events),
-      trace
+      trace,
+      health: computeHealth(trace, DEFAULT_HEALTH_THRESHOLDS)
     };
     const terminalThrow = resolveRuntimeTerminalThrow(runResult.trace, failureInstancesByChildRunId);
     if (terminalThrow) {
@@ -781,7 +1314,61 @@ function finalEventWithEvaluation(event: FinalEvent, evaluation: RunEvaluation):
   };
 }
 
-function runProtocol(options: RunProtocolOptions): Promise<RunResult> {
+async function runProtocol(options: RunProtocolOptions): Promise<RunResult> {
+  const tracing = openRunTracing({
+    ...(options.tracer ? { tracer: options.tracer } : {}),
+    ...(options.parentSpan ? { parentSpan: options.parentSpan } : {}),
+    intent: options.intent,
+    protocolKind: options.protocol.kind,
+    tier: options.tier
+  });
+  const metrics = openRunMetrics({
+    ...(options.metricsHook ? { metricsHook: options.metricsHook } : {}),
+    ...(options.logger ? { logger: options.logger } : {})
+  });
+  const emitForProtocol =
+    tracing || metrics || options.emit
+      ? (event: RunEvent): void => {
+        if (tracing) {
+          handleTracingEvent(tracing, event);
+        }
+        if (metrics) {
+          handleMetricsEvent(metrics, event);
+        }
+        options.emit?.(event);
+      }
+      : undefined;
+  const protocolOptions = tracing
+    ? {
+      ...options,
+      subRunSpansByChildId: tracing.subRunSpans
+    }
+    : options;
+
+  try {
+    const result = await runProtocolInner(protocolOptions, emitForProtocol);
+    if (tracing) {
+      closeRunTracing(tracing, result);
+    }
+    if (metrics && (options.currentDepth === 0 || options.currentDepth === undefined)) {
+      closeRunMetrics(metrics, result);
+    }
+    return result;
+  } catch (error) {
+    if (tracing) {
+      closeRunTracing(tracing, undefined, error);
+    }
+    if (metrics && (options.currentDepth === 0 || options.currentDepth === undefined)) {
+      closeRunMetrics(metrics, undefined);
+    }
+    throw error;
+  }
+}
+
+function runProtocolInner(
+  options: RunProtocolOptions,
+  emitForProtocol?: (event: RunEvent) => void
+): Promise<RunResult> {
   switch (options.protocol.kind) {
     case "sequential":
       return runSequential({
@@ -797,7 +1384,7 @@ function runProtocol(options: RunProtocolOptions): Promise<RunResult> {
         ...(options.signal !== undefined ? { signal: options.signal } : {}),
         ...(options.terminate ? { terminate: options.terminate } : {}),
         ...(options.wrapUpHint ? { wrapUpHint: options.wrapUpHint } : {}),
-        ...(options.emit ? { emit: options.emit } : {})
+        ...(emitForProtocol ? { emit: emitForProtocol } : {})
       });
     case "broadcast":
       return runBroadcast({
@@ -813,7 +1400,10 @@ function runProtocol(options: RunProtocolOptions): Promise<RunResult> {
         ...(options.signal !== undefined ? { signal: options.signal } : {}),
         ...(options.terminate ? { terminate: options.terminate } : {}),
         ...(options.wrapUpHint ? { wrapUpHint: options.wrapUpHint } : {}),
-        ...(options.emit ? { emit: options.emit } : {})
+        ...(options.effectiveMaxConcurrentAgentTurns !== undefined
+          ? { maxConcurrentAgentTurns: options.effectiveMaxConcurrentAgentTurns }
+          : {}),
+        ...(emitForProtocol ? { emit: emitForProtocol } : {})
       });
     case "coordinator":
       return runCoordinator({
@@ -829,11 +1419,16 @@ function runProtocol(options: RunProtocolOptions): Promise<RunResult> {
         ...(options.signal !== undefined ? { signal: options.signal } : {}),
         ...(options.terminate ? { terminate: options.terminate } : {}),
         ...(options.wrapUpHint ? { wrapUpHint: options.wrapUpHint } : {}),
-        ...(options.emit ? { emit: options.emit } : {}),
+        ...(options.effectiveMaxConcurrentAgentTurns !== undefined
+          ? { maxConcurrentAgentTurns: options.effectiveMaxConcurrentAgentTurns }
+          : {}),
+        ...(emitForProtocol ? { emit: emitForProtocol } : {}),
         ...(options.streamEvents !== undefined ? { streamEvents: options.streamEvents } : {}),
         currentDepth: options.currentDepth ?? 0,
         effectiveMaxDepth: options.effectiveMaxDepth ?? Infinity,
         effectiveMaxConcurrentChildren: options.effectiveMaxConcurrentChildren ?? DEFAULT_MAX_CONCURRENT_CHILDREN,
+        effectiveMaxConcurrentAgentTurns:
+          options.effectiveMaxConcurrentAgentTurns ?? DEFAULT_MAX_CONCURRENT_AGENT_TURNS,
         onChildFailure: options.onChildFailure ?? "continue",
         ...(options.parentDeadlineMs !== undefined ? { parentDeadlineMs: options.parentDeadlineMs } : {}),
         ...(options.defaultSubRunTimeoutMs !== undefined
@@ -843,11 +1438,17 @@ function runProtocol(options: RunProtocolOptions): Promise<RunResult> {
         ...(options.failureInstancesByChildRunId !== undefined
           ? { failureInstancesByChildRunId: options.failureInstancesByChildRunId }
           : {}),
-        runProtocol: (childInput) =>
-          runProtocol({
-            ...childInput,
-            protocol: normalizeProtocol(childInput.protocol)
-          })
+        runProtocol: (childInput) => {
+          const { runId: childRunId, ...childProtocolInput } = childInput;
+          const childParent = options.subRunSpansByChildId?.get(childRunId) ?? options.parentSpan;
+          return runProtocol({
+            ...childProtocolInput,
+            protocol: normalizeProtocol(childProtocolInput.protocol),
+            ...(options.tracer ? { tracer: options.tracer } : {}),
+            ...(childParent ? { parentSpan: childParent } : {}),
+            ...(options.logger ? { logger: options.logger } : {})
+          });
+        }
       });
     case "shared":
       return runShared({
@@ -863,7 +1464,10 @@ function runProtocol(options: RunProtocolOptions): Promise<RunResult> {
         ...(options.signal !== undefined ? { signal: options.signal } : {}),
         ...(options.terminate ? { terminate: options.terminate } : {}),
         ...(options.wrapUpHint ? { wrapUpHint: options.wrapUpHint } : {}),
-        ...(options.emit ? { emit: options.emit } : {})
+        ...(options.effectiveMaxConcurrentAgentTurns !== undefined
+          ? { maxConcurrentAgentTurns: options.effectiveMaxConcurrentAgentTurns }
+          : {}),
+        ...(emitForProtocol ? { emit: emitForProtocol } : {})
       });
   }
 }
@@ -916,7 +1520,14 @@ export function stream(options: DogpileOptions): StreamHandle {
  * the ergonomic {@link RunResult} wrapper from the JSON-serializable
  * {@link Trace} returned by a previous `run()`, `stream()`, or
  * `Dogpile.pile()` call.
+ *
+ * Tracing and metrics: replay is intentionally tracing-free and metrics-free.
+ * Even when an engine instance has been configured with a `tracer` or
+ * `metricsHook` on its `EngineOptions`, calling this function emits no spans
+ * or callbacks — replaying historical events with current timestamps would
+ * confuse observability backends. See `docs/developer-usage.md`.
  */
+// Tracing/metrics-free: replay never uses EngineOptions tracer or metricsHook.
 export function replay(trace: Trace): RunResult {
   const cost = trace.finalOutput.cost;
   const lastEvent = trace.events.at(-1);
@@ -931,7 +1542,11 @@ export function replay(trace: Trace): RunResult {
   }
   const baseResult = {
     output: trace.finalOutput.output,
-    eventLog: createRunEventLog(trace.runId, trace.protocol, trace.events),
+    eventLog: createRunEventLog(
+      trace.runId,
+      trace.protocol,
+      synthesizeProviderEvents(trace, trace.providerCalls)
+    ),
     trace,
     transcript: trace.transcript,
     usage: createRunUsage(cost),
@@ -944,7 +1559,8 @@ export function replay(trace: Trace): RunResult {
       events: trace.events
     }),
     accounting,
-    cost
+    cost,
+    health: computeHealth(trace, DEFAULT_HEALTH_THRESHOLDS)
   };
 
   if (lastEvent?.type !== "final") {
@@ -958,6 +1574,60 @@ export function replay(trace: Trace): RunResult {
   };
 }
 
+function synthesizeProviderEvents(
+  trace: Trace,
+  providerCalls: readonly ReplayTraceProviderCall[]
+): readonly RunEvent[] {
+  const hasLiveProvenance = trace.events.some(
+    (event) => event.type === "model-request" || event.type === "model-response"
+  );
+  if (hasLiveProvenance) {
+    return trace.events;
+  }
+
+  const baseEvents = trace.events.filter(
+    (event) => event.type !== "model-request" && event.type !== "model-response"
+  );
+  const result: RunEvent[] = [];
+  let turnCount = 0;
+
+  for (const event of baseEvents) {
+    if (event.type === "agent-turn") {
+      const call = providerCalls[turnCount];
+      if (call !== undefined) {
+        const modelId = typeof call.modelId === "string" && call.modelId.length > 0 ? call.modelId : call.providerId;
+        result.push({
+          type: "model-request",
+          runId: trace.runId,
+          callId: call.callId,
+          providerId: call.providerId,
+          modelId,
+          startedAt: call.startedAt,
+          agentId: call.agentId,
+          role: call.role,
+          request: call.request
+        });
+        result.push({
+          type: "model-response",
+          runId: trace.runId,
+          callId: call.callId,
+          providerId: call.providerId,
+          modelId,
+          startedAt: call.startedAt,
+          completedAt: call.completedAt,
+          agentId: call.agentId,
+          role: call.role,
+          response: call.response
+        });
+      }
+      turnCount += 1;
+    }
+    result.push(event);
+  }
+
+  return result;
+}
+
 function resolveRuntimeTerminalThrow(
   trace: Trace,
   failureInstancesByChildRunId: ReadonlyMap<string, DogpileError>
@@ -1060,14 +1730,21 @@ function dogpileErrorFromSerializedPayload(input: {
  * Replay a saved completed trace as a stream without invoking a model provider.
  *
  * @remarks
- * This is the streaming counterpart to {@link replay}. It yields the exact
- * saved {@link Trace.events} in order and resolves {@link StreamHandle.result}
- * to the rehydrated {@link RunResult}. Since all data comes from the trace,
- * replay remains storage-free and provider-free.
+ * This is the streaming counterpart to {@link replay}. It yields the same
+ * event sequence exposed by the replayed result event log, including legacy
+ * provenance synthesis when a saved trace predates model request/response
+ * events. Since all data comes from the trace, replay remains storage-free and
+ * provider-free.
+ *
+ * Tracing and metrics: replayStream is intentionally tracing-free and
+ * metrics-free. Even when an engine instance has been configured with a
+ * `tracer` or `metricsHook` on its `EngineOptions`, calling this function
+ * emits no spans or callbacks — replaying historical events with current
+ * timestamps would confuse observability backends. See `docs/developer-usage.md`.
  */
+// Tracing/metrics-free: replayStream never uses EngineOptions tracer or metricsHook.
 export function replayStream(trace: Trace): StreamHandle {
   const result = Promise.resolve(replay(trace));
-  const replayEvents = replayStreamEvents(trace);
 
   return {
     get status(): StreamHandleStatus {
@@ -1078,7 +1755,7 @@ export function replayStream(trace: Trace): StreamHandle {
       // Replay streams are already completed snapshots, so cancellation is a no-op.
     },
     subscribe(subscriber: StreamEventSubscriber) {
-      for (const event of replayEvents) {
+      for (const event of replayStreamEvents(trace)) {
         subscriber(event);
       }
 
@@ -1089,34 +1766,24 @@ export function replayStream(trace: Trace): StreamHandle {
       };
     },
     [Symbol.asyncIterator](): AsyncIterator<StreamEvent> {
-      let index = 0;
+      const iterator = replayStreamEvents(trace)[Symbol.iterator]();
 
       return {
         next(): Promise<IteratorResult<StreamEvent>> {
-          const event = replayEvents[index];
-          if (event) {
-            index += 1;
-            return Promise.resolve({ done: false, value: event });
-          }
-
-          return Promise.resolve({ done: true, value: undefined });
+          return Promise.resolve(iterator.next());
         }
       };
     }
   };
 }
 
-function replayStreamEvents(trace: Trace, parentRunIds: readonly string[] = []): StreamEvent[] {
-  const events: StreamEvent[] = [];
-
-  for (const event of trace.events) {
+function* replayStreamEvents(trace: Trace, parentRunIds: readonly string[] = []): IterableIterator<StreamEvent> {
+  for (const event of synthesizeProviderEvents(trace, trace.providerCalls)) {
     if (event.type === "sub-run-completed") {
-      events.push(...replayStreamEvents(event.subResult.trace, [...parentRunIds, trace.runId]));
+      yield* replayStreamEvents(event.subResult.trace, [...parentRunIds, trace.runId]);
     }
-    events.push(wrapReplayStreamEvent(event, parentRunIds));
+    yield wrapReplayStreamEvent(event, parentRunIds);
   }
-
-  return events;
 }
 
 function wrapReplayStreamEvent(event: RunEvent, parentRunIds: readonly string[]): StreamEvent {