@dogpile/sdk 0.4.0 → 0.6.0

package/dist/runtime/engine.js

@@ -1,14 +1,17 @@
 import { DogpileError } from "../types.js";
 import { runBroadcast } from "./broadcast.js";
 import { runCoordinator } from "./coordinator.js";
-import { createReplayTraceFinalOutput, createReplayTraceBudgetStateChanges, canonicalizeRunResult, canonicalizeSerializable, createRunAccounting, createRunEventLog, createRunMetadata, createRunUsage, defaultAgents, normalizeProtocol, orderAgentsForTemperature, recomputeAccountingFromTrace, resolveOnChildFailure, tierTemperature } from "./defaults.js";
+import { addCost, createReplayTraceFinalOutput, createReplayTraceBudgetStateChanges, canonicalizeRunResult, canonicalizeSerializable, createRunAccounting, createRunEventLog, createRunMetadata, createRunUsage, defaultAgents, emptyCost, normalizeProtocol, orderAgentsForTemperature, recomputeAccountingFromTrace, resolveOnChildFailure, tierTemperature } from "./defaults.js";
+import { computeHealth, DEFAULT_HEALTH_THRESHOLDS } from "./health.js";
 import { runSequential } from "./sequential.js";
 import { runShared } from "./shared.js";
 import { classifyChildTimeoutSource, createAbortErrorFromSignal, createEngineDeadlineTimeoutError, createTimeoutError } from "./cancellation.js";
 import { budget as budgetCondition } from "./termination.js";
 import { validateDogpileOptions, validateEngineOptions, validateMissionIntent, validateProviderLocality, validateRunCallOptions } from "./validation.js";
+import { DOGPILE_SPAN_NAMES } from "./tracing.js";
 const DEFAULT_MAX_DEPTH = 4;
 const DEFAULT_MAX_CONCURRENT_CHILDREN = 4;
+const DEFAULT_MAX_CONCURRENT_AGENT_TURNS = 4;
 const defaultHighLevelProtocol = "sequential";
 const defaultHighLevelTier = "balanced";
 /**
@@ -31,6 +34,7 @@ export function createEngine(options) {
     const terminate = options.terminate ?? (options.budget ? conditionFromBudget(options.budget) : undefined);
     const engineMaxDepth = options.maxDepth ?? DEFAULT_MAX_DEPTH;
     const engineMaxConcurrentChildren = options.maxConcurrentChildren ?? DEFAULT_MAX_CONCURRENT_CHILDREN;
+    const engineMaxConcurrentAgentTurns = options.maxConcurrentAgentTurns ?? DEFAULT_MAX_CONCURRENT_AGENT_TURNS;
     const engineOnChildFailure = options.onChildFailure;
     return {
         run(intent, runOptions) {
@@ -40,6 +44,8 @@ export function createEngine(options) {
             const effectiveMaxDepth = Math.min(engineMaxDepth, runOptions?.maxDepth ?? Number.POSITIVE_INFINITY);
             assertRunDoesNotRaiseEngineMax("maxConcurrentChildren", runOptions?.maxConcurrentChildren, engineMaxConcurrentChildren);
             const effectiveMaxConcurrentChildren = Math.min(engineMaxConcurrentChildren, runOptions?.maxConcurrentChildren ?? Number.POSITIVE_INFINITY);
+            assertRunDoesNotRaiseEngineMax("maxConcurrentAgentTurns", runOptions?.maxConcurrentAgentTurns, engineMaxConcurrentAgentTurns);
+            const effectiveMaxConcurrentAgentTurns = Math.min(engineMaxConcurrentAgentTurns, runOptions?.maxConcurrentAgentTurns ?? Number.POSITIVE_INFINITY);
             const onChildFailure = resolveOnChildFailure(runOptions?.onChildFailure, engineOnChildFailure);
             const startedAtMs = Date.now();
             const parentDeadlineMs = options.budget?.timeoutMs !== undefined ? startedAtMs + options.budget.timeoutMs : undefined;
@@ -57,9 +63,13 @@ export function createEngine(options) {
                 ...(terminate ? { terminate } : {}),
                 ...(options.wrapUpHint ? { wrapUpHint: options.wrapUpHint } : {}),
                 ...(options.evaluate ? { evaluate: options.evaluate } : {}),
+                ...(options.tracer ? { tracer: options.tracer } : {}),
+                ...(options.metricsHook ? { metricsHook: options.metricsHook } : {}),
+                ...(options.logger ? { logger: options.logger } : {}),
                 currentDepth: 0,
                 effectiveMaxDepth,
                 effectiveMaxConcurrentChildren,
+                effectiveMaxConcurrentAgentTurns,
                 onChildFailure,
                 ...(parentDeadlineMs !== undefined ? { parentDeadlineMs } : {}),
                 ...(options.defaultSubRunTimeoutMs !== undefined
@@ -74,9 +84,10 @@ export function createEngine(options) {
             const effectiveMaxDepth = Math.min(engineMaxDepth, runOptions?.maxDepth ?? Number.POSITIVE_INFINITY);
             assertRunDoesNotRaiseEngineMax("maxConcurrentChildren", runOptions?.maxConcurrentChildren, engineMaxConcurrentChildren);
             const effectiveMaxConcurrentChildren = Math.min(engineMaxConcurrentChildren, runOptions?.maxConcurrentChildren ?? Number.POSITIVE_INFINITY);
+            assertRunDoesNotRaiseEngineMax("maxConcurrentAgentTurns", runOptions?.maxConcurrentAgentTurns, engineMaxConcurrentAgentTurns);
+            const effectiveMaxConcurrentAgentTurns = Math.min(engineMaxConcurrentAgentTurns, runOptions?.maxConcurrentAgentTurns ?? Number.POSITIVE_INFINITY);
             const onChildFailure = resolveOnChildFailure(runOptions?.onChildFailure, engineOnChildFailure);
-            const pendingEvents = [];
-            const pendingResolvers = [];
+            const pendingIteratorResolvers = [];
             const emittedEvents = [];
             const subscribers = new Set();
             const abortController = new AbortController();
@@ -111,10 +122,17 @@ export function createEngine(options) {
                     cancelRun();
                 },
                 subscribe(subscriber) {
-                    subscribers.add(subscriber);
                     for (const event of emittedEvents) {
                         subscriber(event);
                     }
+                    if (complete) {
+                        return {
+                            unsubscribe() {
+                                // Completed streams replay synchronously and have no live source.
+                            }
+                        };
+                    }
+                    subscribers.add(subscriber);
                     return {
                         unsubscribe() {
                             subscribers.delete(subscriber);
@@ -122,20 +140,27 @@ export function createEngine(options) {
                     };
                 },
                 [Symbol.asyncIterator]() {
+                    let cursor = 0;
                     return {
                         next() {
-                            const event = pendingEvents.shift();
-                            if (event) {
-                                return Promise.resolve({ done: false, value: event });
-                            }
-                            if (complete) {
-                                return Promise.resolve({ done: true, value: undefined });
-                            }
-                            return new Promise((resolve) => {
-                                pendingResolvers.push(resolve);
-                            });
+                            return readNext();
                         }
                     };
+                    function readNext() {
+                        const event = emittedEvents[cursor];
+                        if (event !== undefined) {
+                            cursor += 1;
+                            return Promise.resolve({ done: false, value: event });
+                        }
+                        if (complete) {
+                            return Promise.resolve({ done: true, value: undefined });
+                        }
+                        return new Promise((resolve) => {
+                            pendingIteratorResolvers.push(() => {
+                                void readNext().then(resolve);
+                            });
+                        });
+                    }
                 }
             };
             async function execute() {
@@ -160,11 +185,15 @@ export function createEngine(options) {
                         currentDepth: 0,
                         effectiveMaxDepth,
                         effectiveMaxConcurrentChildren,
+                        effectiveMaxConcurrentAgentTurns,
                         onChildFailure,
                         ...(streamParentDeadlineMs !== undefined ? { parentDeadlineMs: streamParentDeadlineMs } : {}),
                         ...(options.defaultSubRunTimeoutMs !== undefined
                             ? { defaultSubRunTimeoutMs: options.defaultSubRunTimeoutMs }
                             : {}),
+                        ...(options.tracer ? { tracer: options.tracer } : {}),
+                        ...(options.metricsHook ? { metricsHook: options.metricsHook } : {}),
+                        ...(options.logger ? { logger: options.logger } : {}),
                         streamEvents: true,
                         emit(event) {
                             if (status !== "running") {
@@ -246,8 +275,8 @@ export function createEngine(options) {
                 timeoutLifecycle.cleanup();
                 abortRace.cleanup();
                 subscribers.clear();
-                for (const resolver of pendingResolvers.splice(0)) {
-                    resolver({ done: true, value: undefined });
+                for (const resolvePending of pendingIteratorResolvers.splice(0)) {
+                    resolvePending();
                 }
             }
             function publish(event) {
@@ -264,12 +293,9 @@ export function createEngine(options) {
                         // Subscriber failures should not cancel the underlying SDK run.
                     }
                 }
-                const resolver = pendingResolvers.shift();
-                if (resolver) {
-                    resolver({ done: false, value: canonicalEvent });
-                    return;
+                for (const resolvePending of pendingIteratorResolvers.splice(0)) {
+                    resolvePending();
                 }
-                pendingEvents.push(canonicalEvent);
             }
         }
     };
@@ -485,6 +511,387 @@ function dogpileErrorStreamDetail(error) {
     }
     return detail;
 }
+function openRunTracing(options) {
+    if (!options.tracer) {
+        return undefined;
+    }
+    const runSpan = options.tracer.startSpan(DOGPILE_SPAN_NAMES.RUN, {
+        ...(options.parentSpan ? { parent: options.parentSpan } : {}),
+        attributes: {
+            "dogpile.run.protocol": options.protocolKind,
+            "dogpile.run.tier": String(options.tier),
+            "dogpile.run.intent": options.intent.slice(0, 200)
+        }
+    });
+    return {
+        tracer: options.tracer,
+        runSpan,
+        subRunSpans: new Map(),
+        agentTurnSpans: new Map(),
+        modelCallSpans: new Map(),
+        pendingModelRequests: new Map(),
+        agentTurnCounters: new Map(),
+        turnAccumByAgent: new Map(),
+        agentIds: new Set(),
+        turnCount: 0,
+        lastCost: emptyCost()
+    };
+}
+function openRunMetrics(options) {
+    if (!options.metricsHook) {
+        return undefined;
+    }
+    return {
+        metricsHook: options.metricsHook,
+        logger: options.logger,
+        startedAtMs: Date.now(),
+        subRunStartTimes: new Map(),
+        totalCost: emptyCost(),
+        nestedCost: emptyCost(),
+        turns: 0
+    };
+}
+function routeMetricsError(err, logger) {
+    const msg = err instanceof Error ? err.message : String(err);
+    try {
+        if (logger !== undefined) {
+            logger.error("dogpile:metricsHook threw", { error: msg });
+        }
+        else {
+            console.error("dogpile:metricsHook threw", { error: msg });
+        }
+    }
+    catch {
+        // A logger that throws from error() cannot be helped.
+    }
+}
+function fireHook(callback, snapshot, logger) {
+    if (!callback) {
+        return;
+    }
+    try {
+        const result = callback(snapshot);
+        if (result && typeof result.catch === "function") {
+            result.catch((err) => {
+                routeMetricsError(err, logger);
+            });
+        }
+    }
+    catch (err) {
+        routeMetricsError(err, logger);
+    }
+}
+function buildRunSnapshot(result, startedAtMs) {
+    const nestedCosts = nestedSubRunCosts(result);
+    const budgetStopEvent = result.trace.events.find((event) => event.type === "budget-stop");
+    const outcome = budgetStopEvent !== undefined ? "budget-stopped" : "completed";
+    const totalInputTokens = result.cost.inputTokens;
+    const totalOutputTokens = result.cost.outputTokens;
+    const totalCostUsd = result.cost.usd;
+    const ownInputTokens = totalInputTokens - nestedCosts.reduce((sum, cost) => sum + cost.inputTokens, 0);
+    const ownOutputTokens = totalOutputTokens - nestedCosts.reduce((sum, cost) => sum + cost.outputTokens, 0);
+    const ownCostUsd = totalCostUsd - nestedCosts.reduce((sum, cost) => sum + cost.usd, 0);
+    const turns = result.trace.events.filter((event) => event.type === "agent-turn").length;
+    return {
+        outcome,
+        inputTokens: ownInputTokens,
+        outputTokens: ownOutputTokens,
+        costUsd: ownCostUsd,
+        totalInputTokens,
+        totalOutputTokens,
+        totalCostUsd,
+        turns,
+        durationMs: Date.now() - startedAtMs
+    };
+}
+function buildSubRunSnapshot(subResult, durationMs) {
+    const nestedCosts = nestedSubRunCosts(subResult);
+    const budgetStopEvent = subResult.trace.events.find((event) => event.type === "budget-stop");
+    const outcome = budgetStopEvent !== undefined ? "budget-stopped" : "completed";
+    const totalInputTokens = subResult.cost.inputTokens;
+    const totalOutputTokens = subResult.cost.outputTokens;
+    const totalCostUsd = subResult.cost.usd;
+    const ownInputTokens = totalInputTokens - nestedCosts.reduce((sum, cost) => sum + cost.inputTokens, 0);
+    const ownOutputTokens = totalOutputTokens - nestedCosts.reduce((sum, cost) => sum + cost.outputTokens, 0);
+    const ownCostUsd = totalCostUsd - nestedCosts.reduce((sum, cost) => sum + cost.usd, 0);
+    const turns = subResult.trace.events.filter((event) => event.type === "agent-turn").length;
+    return {
+        outcome,
+        inputTokens: ownInputTokens,
+        outputTokens: ownOutputTokens,
+        costUsd: ownCostUsd,
+        totalInputTokens,
+        totalOutputTokens,
+        totalCostUsd,
+        turns,
+        durationMs
+    };
+}
+function nestedSubRunCosts(result) {
+    return result.trace.events.flatMap((event) => {
+        if (event.type === "sub-run-completed") {
+            return [event.subResult.cost];
+        }
+        if (event.type === "sub-run-failed") {
+            return [event.partialCost];
+        }
+        return [];
+    });
+}
+function subtractCost(total, nested) {
+    return {
+        usd: total.usd - nested.usd,
+        inputTokens: total.inputTokens - nested.inputTokens,
+        outputTokens: total.outputTokens - nested.outputTokens,
+        totalTokens: total.totalTokens - nested.totalTokens
+    };
+}
+function handleMetricsEvent(state, event) {
+    const parentRunIds = event.parentRunIds;
+    if (parentRunIds !== undefined) {
+        return;
+    }
+    switch (event.type) {
+        case "agent-turn": {
+            state.totalCost = event.cost;
+            state.turns += 1;
+            break;
+        }
+        case "broadcast":
+        case "budget-stop":
+        case "final": {
+            state.totalCost = event.cost;
+            break;
+        }
+        case "sub-run-started": {
+            state.subRunStartTimes.set(event.childRunId, Date.now());
+            break;
+        }
+        case "sub-run-completed": {
+            state.totalCost = addCost(state.totalCost, event.subResult.cost);
+            state.nestedCost = addCost(state.nestedCost, event.subResult.cost);
+            const startMs = state.subRunStartTimes.get(event.childRunId);
+            const durationMs = startMs !== undefined ? Date.now() - startMs : 0;
+            state.subRunStartTimes.delete(event.childRunId);
+            const snapshot = buildSubRunSnapshot(event.subResult, durationMs);
+            fireHook(state.metricsHook.onSubRunComplete, snapshot, state.logger);
+            break;
+        }
+        case "sub-run-failed": {
+            state.totalCost = addCost(state.totalCost, event.partialCost);
+            state.nestedCost = addCost(state.nestedCost, event.partialCost);
+            state.subRunStartTimes.delete(event.childRunId);
+            break;
+        }
+        default:
+            break;
+    }
+}
+function closeRunMetrics(state, result) {
+    if (result !== undefined) {
+        const snapshot = buildRunSnapshot(result, state.startedAtMs);
+        fireHook(state.metricsHook.onRunComplete, snapshot, state.logger);
+        return;
+    }
+    const ownCost = subtractCost(state.totalCost, state.nestedCost);
+    const snapshot = {
+        outcome: "aborted",
+        inputTokens: ownCost.inputTokens,
+        outputTokens: ownCost.outputTokens,
+        costUsd: ownCost.usd,
+        totalInputTokens: state.totalCost.inputTokens,
+        totalOutputTokens: state.totalCost.outputTokens,
+        totalCostUsd: state.totalCost.usd,
+        turns: state.turns,
+        durationMs: Date.now() - state.startedAtMs
+    };
+    fireHook(state.metricsHook.onRunComplete, snapshot, state.logger);
+}
+function handleTracingEvent(state, event) {
+    const parentRunIds = event.parentRunIds;
+    if (parentRunIds !== undefined) {
+        return;
+    }
+    if (state.runId === undefined) {
+        state.runId = event.runId;
+        state.runSpan.setAttribute("dogpile.run.id", event.runId);
+    }
+    switch (event.type) {
+        case "model-request": {
+            state.pendingModelRequests.set(event.callId, event);
+            state.agentIds.add(event.agentId);
+            if (!state.agentTurnSpans.has(event.agentId)) {
+                const turnNumber = (state.agentTurnCounters.get(event.agentId) ?? 0) + 1;
+                state.agentTurnCounters.set(event.agentId, turnNumber);
+                const turnParent = state.subRunSpans.get(event.runId) ?? state.runSpan;
+                const turnSpan = state.tracer.startSpan(DOGPILE_SPAN_NAMES.AGENT_TURN, {
+                    parent: turnParent,
+                    attributes: {
+                        "dogpile.agent.id": event.agentId,
+                        "dogpile.agent.role": event.role,
+                        "dogpile.turn.number": turnNumber,
+                        "dogpile.model.id": event.modelId
+                    }
+                });
+                state.agentTurnSpans.set(event.agentId, turnSpan);
+            }
+            const callParent = state.agentTurnSpans.get(event.agentId) ??
+                state.subRunSpans.get(event.runId) ??
+                state.runSpan;
+            const callSpan = state.tracer.startSpan(DOGPILE_SPAN_NAMES.MODEL_CALL, {
+                parent: callParent,
+                attributes: {
+                    "dogpile.model.id": event.modelId,
+                    "dogpile.call.id": event.callId,
+                    "dogpile.provider.id": event.providerId
+                }
+            });
+            state.modelCallSpans.set(event.callId, callSpan);
+            break;
+        }
+        case "model-response": {
+            const span = state.modelCallSpans.get(event.callId);
+            if (span) {
+                const inputTokens = event.response.usage?.inputTokens ?? 0;
+                const outputTokens = event.response.usage?.outputTokens ?? 0;
+                const responseCost = {
+                    usd: event.response.costUsd ?? 0,
+                    inputTokens,
+                    outputTokens,
+                    totalTokens: event.response.usage?.totalTokens ?? inputTokens + outputTokens
+                };
+                span.setAttribute("dogpile.model.input_tokens", inputTokens);
+                span.setAttribute("dogpile.model.output_tokens", outputTokens);
+                if (event.response.costUsd !== undefined) {
+                    span.setAttribute("dogpile.model.cost_usd", event.response.costUsd);
+                }
+                span.setStatus("ok");
+                span.end();
+                state.modelCallSpans.delete(event.callId);
+                const accum = state.turnAccumByAgent.get(event.agentId) ?? {
+                    inputTokens: 0,
+                    outputTokens: 0,
+                    costUsd: 0
+                };
+                accum.inputTokens += inputTokens;
+                accum.outputTokens += outputTokens;
+                accum.costUsd += responseCost.usd;
+                state.turnAccumByAgent.set(event.agentId, accum);
+                state.lastCost = addCost(state.lastCost, responseCost);
+            }
+            state.pendingModelRequests.delete(event.callId);
+            break;
+        }
+        case "agent-turn": {
+            state.agentIds.add(event.agentId);
+            state.turnCount += 1;
+            state.lastCost = event.cost;
+            const turnSpan = state.agentTurnSpans.get(event.agentId);
+            if (turnSpan) {
+                turnSpan.setAttribute("dogpile.agent.role", event.role);
+                const accum = state.turnAccumByAgent.get(event.agentId);
+                turnSpan.setAttribute("dogpile.turn.cost_usd", accum?.costUsd ?? 0);
+                turnSpan.setAttribute("dogpile.turn.input_tokens", accum?.inputTokens ?? 0);
+                turnSpan.setAttribute("dogpile.turn.output_tokens", accum?.outputTokens ?? 0);
+                turnSpan.setStatus("ok");
+                turnSpan.end();
+                state.agentTurnSpans.delete(event.agentId);
+            }
+            state.turnAccumByAgent.delete(event.agentId);
+            break;
+        }
+        case "broadcast":
+        case "budget-stop":
+        case "final": {
+            state.lastCost = event.cost;
+            break;
+        }
+        case "sub-run-started": {
+            const span = state.tracer.startSpan(DOGPILE_SPAN_NAMES.SUB_RUN, {
+                parent: state.runSpan,
+                attributes: {
+                    "dogpile.sub_run.child_run_id": event.childRunId,
+                    "dogpile.sub_run.parent_run_id": event.parentRunId,
+                    "dogpile.sub_run.depth": event.depth
+                }
+            });
+            state.subRunSpans.set(event.childRunId, span);
+            break;
+        }
+        case "sub-run-completed": {
+            const span = state.subRunSpans.get(event.childRunId);
+            if (span) {
+                span.setStatus("ok");
+                span.end();
+                state.subRunSpans.delete(event.childRunId);
+            }
+            break;
+        }
+        case "sub-run-failed": {
+            const span = state.subRunSpans.get(event.childRunId);
+            if (span) {
+                span.setStatus("error", event.error.message);
+                span.end();
+                state.subRunSpans.delete(event.childRunId);
+            }
+            break;
+        }
+        default:
+            break;
+    }
+}
+function closeRunTracing(state, result, error) {
+    if (error !== undefined) {
+        if (state.runId !== undefined) {
+            state.runSpan.setAttribute("dogpile.run.id", state.runId);
+        }
+        state.runSpan.setAttribute("dogpile.run.agent_count", state.agentIds.size);
+        state.runSpan.setAttribute("dogpile.run.turn_count", state.turnCount);
+        state.runSpan.setAttribute("dogpile.run.cost_usd", state.lastCost.usd);
+        state.runSpan.setAttribute("dogpile.run.input_tokens", state.lastCost.inputTokens);
+        state.runSpan.setAttribute("dogpile.run.output_tokens", state.lastCost.outputTokens);
+        state.runSpan.setAttribute("dogpile.run.outcome", "aborted");
+        state.runSpan.setStatus("error", error instanceof Error ? error.message : String(error));
+        closeOpenTracingSpans(state);
+        state.runSpan.end();
+        return;
+    }
+    if (result === undefined) {
+        closeOpenTracingSpans(state);
+        state.runSpan.end();
+        return;
+    }
+    const budgetStopEvent = result.trace.events.find((event) => event.type === "budget-stop");
+    const terminationReason = budgetStopEvent?.reason;
+    const outcome = terminationReason !== undefined ? "budget-stopped" : "completed";
+    state.runSpan.setAttribute("dogpile.run.id", result.trace.runId);
+    state.runSpan.setAttribute("dogpile.run.agent_count", result.trace.agentsUsed.length);
+    state.runSpan.setAttribute("dogpile.run.turn_count", result.trace.events.filter((event) => event.type === "agent-turn").length);
+    state.runSpan.setAttribute("dogpile.run.cost_usd", result.cost.usd);
+    state.runSpan.setAttribute("dogpile.run.input_tokens", result.cost.inputTokens);
+    state.runSpan.setAttribute("dogpile.run.output_tokens", result.cost.outputTokens);
+    state.runSpan.setAttribute("dogpile.run.outcome", outcome);
+    if (terminationReason !== undefined) {
+        state.runSpan.setAttribute("dogpile.run.termination_reason", terminationReason);
+    }
+    state.runSpan.setStatus("ok");
+    closeOpenTracingSpans(state);
+    state.runSpan.end();
+}
+function closeOpenTracingSpans(state) {
+    for (const span of state.modelCallSpans.values()) {
+        span.end();
+    }
+    state.modelCallSpans.clear();
+    for (const span of state.agentTurnSpans.values()) {
+        span.end();
+    }
+    state.agentTurnSpans.clear();
+    for (const span of state.subRunSpans.values()) {
+        span.end();
+    }
+    state.subRunSpans.clear();
+}
 async function runNonStreamingProtocol(options) {
     const failureInstancesByChildRunId = new Map();
     const abortLifecycle = createNonStreamingAbortLifecycle({
@@ -524,7 +931,8 @@ async function runNonStreamingProtocol(options) {
                 events
             }),
             eventLog: createRunEventLog(trace.runId, trace.protocol, events),
-            trace
+            trace,
+            health: computeHealth(trace, DEFAULT_HEALTH_THRESHOLDS)
         };
         const terminalThrow = resolveRuntimeTerminalThrow(runResult.trace, failureInstancesByChildRunId);
         if (terminalThrow) {
@@ -570,7 +978,56 @@ function finalEventWithEvaluation(event, evaluation) {
         evaluation
     };
 }
-function runProtocol(options) {
+async function runProtocol(options) {
+    const tracing = openRunTracing({
+        ...(options.tracer ? { tracer: options.tracer } : {}),
+        ...(options.parentSpan ? { parentSpan: options.parentSpan } : {}),
+        intent: options.intent,
+        protocolKind: options.protocol.kind,
+        tier: options.tier
+    });
+    const metrics = openRunMetrics({
+        ...(options.metricsHook ? { metricsHook: options.metricsHook } : {}),
+        ...(options.logger ? { logger: options.logger } : {})
+    });
+    const emitForProtocol = tracing || metrics || options.emit
+        ? (event) => {
+            if (tracing) {
+                handleTracingEvent(tracing, event);
+            }
+            if (metrics) {
+                handleMetricsEvent(metrics, event);
+            }
+            options.emit?.(event);
+        }
+        : undefined;
+    const protocolOptions = tracing
+        ? {
+            ...options,
+            subRunSpansByChildId: tracing.subRunSpans
+        }
+        : options;
+    try {
+        const result = await runProtocolInner(protocolOptions, emitForProtocol);
+        if (tracing) {
+            closeRunTracing(tracing, result);
+        }
+        if (metrics && (options.currentDepth === 0 || options.currentDepth === undefined)) {
+            closeRunMetrics(metrics, result);
+        }
+        return result;
+    }
+    catch (error) {
+        if (tracing) {
+            closeRunTracing(tracing, undefined, error);
+        }
+        if (metrics && (options.currentDepth === 0 || options.currentDepth === undefined)) {
+            closeRunMetrics(metrics, undefined);
+        }
+        throw error;
+    }
+}
+function runProtocolInner(options, emitForProtocol) {
     switch (options.protocol.kind) {
         case "sequential":
             return runSequential({
@@ -586,7 +1043,7 @@ function runProtocol(options) {
                 ...(options.signal !== undefined ? { signal: options.signal } : {}),
                 ...(options.terminate ? { terminate: options.terminate } : {}),
                 ...(options.wrapUpHint ? { wrapUpHint: options.wrapUpHint } : {}),
-                ...(options.emit ? { emit: options.emit } : {})
+                ...(emitForProtocol ? { emit: emitForProtocol } : {})
             });
         case "broadcast":
             return runBroadcast({
@@ -602,7 +1059,10 @@ function runProtocol(options) {
                 ...(options.signal !== undefined ? { signal: options.signal } : {}),
                 ...(options.terminate ? { terminate: options.terminate } : {}),
                 ...(options.wrapUpHint ? { wrapUpHint: options.wrapUpHint } : {}),
-                ...(options.emit ? { emit: options.emit } : {})
+                ...(options.effectiveMaxConcurrentAgentTurns !== undefined
+                    ? { maxConcurrentAgentTurns: options.effectiveMaxConcurrentAgentTurns }
+                    : {}),
+                ...(emitForProtocol ? { emit: emitForProtocol } : {})
             });
         case "coordinator":
             return runCoordinator({
@@ -618,11 +1078,15 @@ function runProtocol(options) {
                 ...(options.signal !== undefined ? { signal: options.signal } : {}),
                 ...(options.terminate ? { terminate: options.terminate } : {}),
                 ...(options.wrapUpHint ? { wrapUpHint: options.wrapUpHint } : {}),
-                ...(options.emit ? { emit: options.emit } : {}),
+                ...(options.effectiveMaxConcurrentAgentTurns !== undefined
+                    ? { maxConcurrentAgentTurns: options.effectiveMaxConcurrentAgentTurns }
+                    : {}),
+                ...(emitForProtocol ? { emit: emitForProtocol } : {}),
                 ...(options.streamEvents !== undefined ? { streamEvents: options.streamEvents } : {}),
                 currentDepth: options.currentDepth ?? 0,
                 effectiveMaxDepth: options.effectiveMaxDepth ?? Infinity,
                 effectiveMaxConcurrentChildren: options.effectiveMaxConcurrentChildren ?? DEFAULT_MAX_CONCURRENT_CHILDREN,
+                effectiveMaxConcurrentAgentTurns: options.effectiveMaxConcurrentAgentTurns ?? DEFAULT_MAX_CONCURRENT_AGENT_TURNS,
                 onChildFailure: options.onChildFailure ?? "continue",
                 ...(options.parentDeadlineMs !== undefined ? { parentDeadlineMs: options.parentDeadlineMs } : {}),
                 ...(options.defaultSubRunTimeoutMs !== undefined
@@ -632,10 +1096,17 @@ function runProtocol(options) {
                 ...(options.failureInstancesByChildRunId !== undefined
                     ? { failureInstancesByChildRunId: options.failureInstancesByChildRunId }
                     : {}),
-                runProtocol: (childInput) => runProtocol({
-                    ...childInput,
-                    protocol: normalizeProtocol(childInput.protocol)
-                })
+                runProtocol: (childInput) => {
+                    const { runId: childRunId, ...childProtocolInput } = childInput;
+                    const childParent = options.subRunSpansByChildId?.get(childRunId) ?? options.parentSpan;
+                    return runProtocol({
+                        ...childProtocolInput,
+                        protocol: normalizeProtocol(childProtocolInput.protocol),
+                        ...(options.tracer ? { tracer: options.tracer } : {}),
+                        ...(childParent ? { parentSpan: childParent } : {}),
+                        ...(options.logger ? { logger: options.logger } : {})
+                    });
+                }
             });
         case "shared":
             return runShared({
@@ -651,7 +1122,10 @@ function runProtocol(options) {
                 ...(options.signal !== undefined ? { signal: options.signal } : {}),
                 ...(options.terminate ? { terminate: options.terminate } : {}),
                 ...(options.wrapUpHint ? { wrapUpHint: options.wrapUpHint } : {}),
-                ...(options.emit ? { emit: options.emit } : {})
+                ...(options.effectiveMaxConcurrentAgentTurns !== undefined
+                    ? { maxConcurrentAgentTurns: options.effectiveMaxConcurrentAgentTurns }
+                    : {}),
+                ...(emitForProtocol ? { emit: emitForProtocol } : {})
             });
     }
 }
@@ -699,7 +1173,14 @@ export function stream(options) {
  * the ergonomic {@link RunResult} wrapper from the JSON-serializable
  * {@link Trace} returned by a previous `run()`, `stream()`, or
  * `Dogpile.pile()` call.
+ *
+ * Tracing and metrics: replay is intentionally tracing-free and metrics-free.
+ * Even when an engine instance has been configured with a `tracer` or
+ * `metricsHook` on its `EngineOptions`, calling this function emits no spans
+ * or callbacks — replaying historical events with current timestamps would
+ * confuse observability backends. See `docs/developer-usage.md`.
  */
+// Tracing/metrics-free: replay never uses EngineOptions tracer or metricsHook.
 export function replay(trace) {
     const cost = trace.finalOutput.cost;
     const lastEvent = trace.events.at(-1);
@@ -714,7 +1195,7 @@ export function replay(trace) {
     }
     const baseResult = {
         output: trace.finalOutput.output,
-        eventLog: createRunEventLog(trace.runId, trace.protocol, trace.events),
+        eventLog: createRunEventLog(trace.runId, trace.protocol, synthesizeProviderEvents(trace, trace.providerCalls)),
         trace,
         transcript: trace.transcript,
         usage: createRunUsage(cost),
@@ -727,7 +1208,8 @@ export function replay(trace) {
             events: trace.events
         }),
         accounting,
-        cost
+        cost,
+        health: computeHealth(trace, DEFAULT_HEALTH_THRESHOLDS)
     };
     if (lastEvent?.type !== "final") {
         return baseResult;
@@ -738,6 +1220,49 @@ export function replay(trace) {
         ...(lastEvent.evaluation !== undefined ? { evaluation: lastEvent.evaluation } : {})
     };
 }
+function synthesizeProviderEvents(trace, providerCalls) {
+    const hasLiveProvenance = trace.events.some((event) => event.type === "model-request" || event.type === "model-response");
+    if (hasLiveProvenance) {
+        return trace.events;
+    }
+    const baseEvents = trace.events.filter((event) => event.type !== "model-request" && event.type !== "model-response");
+    const result = [];
+    let turnCount = 0;
+    for (const event of baseEvents) {
+        if (event.type === "agent-turn") {
+            const call = providerCalls[turnCount];
+            if (call !== undefined) {
+                const modelId = typeof call.modelId === "string" && call.modelId.length > 0 ? call.modelId : call.providerId;
+                result.push({
+                    type: "model-request",
+                    runId: trace.runId,
+                    callId: call.callId,
+                    providerId: call.providerId,
+                    modelId,
+                    startedAt: call.startedAt,
+                    agentId: call.agentId,
+                    role: call.role,
+                    request: call.request
+                });
+                result.push({
+                    type: "model-response",
+                    runId: trace.runId,
+                    callId: call.callId,
+                    providerId: call.providerId,
+                    modelId,
+                    startedAt: call.startedAt,
+                    completedAt: call.completedAt,
+                    agentId: call.agentId,
+                    role: call.role,
+                    response: call.response
+                });
+            }
+            turnCount += 1;
+        }
+        result.push(event);
+    }
+    return result;
+}
 function resolveRuntimeTerminalThrow(trace, failureInstancesByChildRunId) {
     if (trace.triggeringFailureForAbortMode !== undefined) {
         return failureInstancesByChildRunId.get(trace.triggeringFailureForAbortMode.childRunId) ?? null;
@@ -816,14 +1341,21 @@ function dogpileErrorFromSerializedPayload(input) {
  * Replay a saved completed trace as a stream without invoking a model provider.
  *
  * @remarks
- * This is the streaming counterpart to {@link replay}. It yields the exact
- * saved {@link Trace.events} in order and resolves {@link StreamHandle.result}
- * to the rehydrated {@link RunResult}. Since all data comes from the trace,
- * replay remains storage-free and provider-free.
+ * This is the streaming counterpart to {@link replay}. It yields the same
+ * event sequence exposed by the replayed result event log, including legacy
+ * provenance synthesis when a saved trace predates model request/response
+ * events. Since all data comes from the trace, replay remains storage-free and
+ * provider-free.
+ *
+ * Tracing and metrics: replayStream is intentionally tracing-free and
+ * metrics-free. Even when an engine instance has been configured with a
+ * `tracer` or `metricsHook` on its `EngineOptions`, calling this function
+ * emits no spans or callbacks — replaying historical events with current
+ * timestamps would confuse observability backends. See `docs/developer-usage.md`.
  */
+// Tracing/metrics-free: replayStream never uses EngineOptions tracer or metricsHook.
 export function replayStream(trace) {
     const result = Promise.resolve(replay(trace));
-    const replayEvents = replayStreamEvents(trace);
     return {
         get status() {
             return "completed";
@@ -833,7 +1365,7 @@ export function replayStream(trace) {
             // Replay streams are already completed snapshots, so cancellation is a no-op.
         },
         subscribe(subscriber) {
-            for (const event of replayEvents) {
+            for (const event of replayStreamEvents(trace)) {
                 subscriber(event);
             }
             return {
@@ -843,29 +1375,22 @@ export function replayStream(trace) {
             };
         },
         [Symbol.asyncIterator]() {
-            let index = 0;
+            const iterator = replayStreamEvents(trace)[Symbol.iterator]();
             return {
                 next() {
-                    const event = replayEvents[index];
-                    if (event) {
-                        index += 1;
-                        return Promise.resolve({ done: false, value: event });
-                    }
-                    return Promise.resolve({ done: true, value: undefined });
+                    return Promise.resolve(iterator.next());
                 }
             };
         }
     };
 }
-function replayStreamEvents(trace, parentRunIds = []) {
-    const events = [];
-    for (const event of trace.events) {
+function* replayStreamEvents(trace, parentRunIds = []) {
+    for (const event of synthesizeProviderEvents(trace, trace.providerCalls)) {
         if (event.type === "sub-run-completed") {
-            events.push(...replayStreamEvents(event.subResult.trace, [...parentRunIds, trace.runId]));
+            yield* replayStreamEvents(event.subResult.trace, [...parentRunIds, trace.runId]);
         }
-        events.push(wrapReplayStreamEvent(event, parentRunIds));
+        yield wrapReplayStreamEvent(event, parentRunIds);
     }
-    return events;
 }
 function wrapReplayStreamEvent(event, parentRunIds) {
     if (parentRunIds.length === 0) {