@dogpile/sdk 0.3.1 → 0.5.0

package/dist/runtime/engine.js

@@ -1,12 +1,16 @@
 import { DogpileError } from "../types.js";
 import { runBroadcast } from "./broadcast.js";
 import { runCoordinator } from "./coordinator.js";
-import { createReplayTraceFinalOutput, createReplayTraceBudgetStateChanges, canonicalizeRunResult, canonicalizeSerializable, createRunAccounting, createRunEventLog, createRunMetadata, createRunUsage, defaultAgents, normalizeProtocol, orderAgentsForTemperature, tierTemperature } from "./defaults.js";
+import { addCost, createReplayTraceFinalOutput, createReplayTraceBudgetStateChanges, canonicalizeRunResult, canonicalizeSerializable, createRunAccounting, createRunEventLog, createRunMetadata, createRunUsage, defaultAgents, emptyCost, normalizeProtocol, orderAgentsForTemperature, recomputeAccountingFromTrace, resolveOnChildFailure, tierTemperature } from "./defaults.js";
+import { computeHealth, DEFAULT_HEALTH_THRESHOLDS } from "./health.js";
 import { runSequential } from "./sequential.js";
 import { runShared } from "./shared.js";
-import { createAbortErrorFromSignal, createTimeoutError } from "./cancellation.js";
+import { classifyChildTimeoutSource, createAbortErrorFromSignal, createEngineDeadlineTimeoutError, createTimeoutError } from "./cancellation.js";
 import { budget as budgetCondition } from "./termination.js";
-import { validateDogpileOptions, validateEngineOptions, validateMissionIntent } from "./validation.js";
+import { validateDogpileOptions, validateEngineOptions, validateMissionIntent, validateProviderLocality, validateRunCallOptions } from "./validation.js";
+import { DOGPILE_SPAN_NAMES } from "./tracing.js";
+const DEFAULT_MAX_DEPTH = 4;
+const DEFAULT_MAX_CONCURRENT_CHILDREN = 4;
 const defaultHighLevelProtocol = "sequential";
 const defaultHighLevelTier = "balanced";
 /**
@@ -27,9 +31,20 @@ export function createEngine(options) {
     const temperature = options.temperature ?? tierTemperature(options.tier);
     const agents = orderAgentsForTemperature(options.agents ?? defaultAgents(), temperature, options.seed);
     const terminate = options.terminate ?? (options.budget ? conditionFromBudget(options.budget) : undefined);
+    const engineMaxDepth = options.maxDepth ?? DEFAULT_MAX_DEPTH;
+    const engineMaxConcurrentChildren = options.maxConcurrentChildren ?? DEFAULT_MAX_CONCURRENT_CHILDREN;
+    const engineOnChildFailure = options.onChildFailure;
     return {
-        run(intent) {
+        run(intent, runOptions) {
             validateMissionIntent(intent);
+            validateRunCallOptions(runOptions);
+            validateProviderLocality(options.model, "model");
+            const effectiveMaxDepth = Math.min(engineMaxDepth, runOptions?.maxDepth ?? Number.POSITIVE_INFINITY);
+            assertRunDoesNotRaiseEngineMax("maxConcurrentChildren", runOptions?.maxConcurrentChildren, engineMaxConcurrentChildren);
+            const effectiveMaxConcurrentChildren = Math.min(engineMaxConcurrentChildren, runOptions?.maxConcurrentChildren ?? Number.POSITIVE_INFINITY);
+            const onChildFailure = resolveOnChildFailure(runOptions?.onChildFailure, engineOnChildFailure);
+            const startedAtMs = Date.now();
+            const parentDeadlineMs = options.budget?.timeoutMs !== undefined ? startedAtMs + options.budget.timeoutMs : undefined;
             return runNonStreamingProtocol({
                 intent,
                 protocol,
@@ -43,11 +58,28 @@ export function createEngine(options) {
                 ...(options.signal !== undefined ? { signal: options.signal } : {}),
                 ...(terminate ? { terminate } : {}),
                 ...(options.wrapUpHint ? { wrapUpHint: options.wrapUpHint } : {}),
-                ...(options.evaluate ? { evaluate: options.evaluate } : {})
+                ...(options.evaluate ? { evaluate: options.evaluate } : {}),
+                ...(options.tracer ? { tracer: options.tracer } : {}),
+                ...(options.metricsHook ? { metricsHook: options.metricsHook } : {}),
+                ...(options.logger ? { logger: options.logger } : {}),
+                currentDepth: 0,
+                effectiveMaxDepth,
+                effectiveMaxConcurrentChildren,
+                onChildFailure,
+                ...(parentDeadlineMs !== undefined ? { parentDeadlineMs } : {}),
+                ...(options.defaultSubRunTimeoutMs !== undefined
+                    ? { defaultSubRunTimeoutMs: options.defaultSubRunTimeoutMs }
+                    : {})
             });
         },
-        stream(intent) {
+        stream(intent, runOptions) {
             validateMissionIntent(intent);
+            validateRunCallOptions(runOptions);
+            validateProviderLocality(options.model, "model");
+            const effectiveMaxDepth = Math.min(engineMaxDepth, runOptions?.maxDepth ?? Number.POSITIVE_INFINITY);
+            assertRunDoesNotRaiseEngineMax("maxConcurrentChildren", runOptions?.maxConcurrentChildren, engineMaxConcurrentChildren);
+            const effectiveMaxConcurrentChildren = Math.min(engineMaxConcurrentChildren, runOptions?.maxConcurrentChildren ?? Number.POSITIVE_INFINITY);
+            const onChildFailure = resolveOnChildFailure(runOptions?.onChildFailure, engineOnChildFailure);
             const pendingEvents = [];
             const pendingResolvers = [];
             const emittedEvents = [];
@@ -61,7 +93,10 @@ export function createEngine(options) {
             const abortRace = createAbortRace(abortController.signal, options.model.id);
             let complete = false;
             let lastRunId = "";
+            let rootRunId;
             let pendingFinalEvent;
+            let activeAbortDrain;
+            const failureInstancesByChildRunId = new Map();
             let status = "running";
             let resolveResult;
             let rejectResult;
@@ -113,6 +148,8 @@ export function createEngine(options) {
                     return;
                 }
                 try {
+                    const streamStartedAtMs = Date.now();
+                    const streamParentDeadlineMs = options.budget?.timeoutMs !== undefined ? streamStartedAtMs + options.budget.timeoutMs : undefined;
                     const baseResult = await abortRace.run(runProtocol({
                         intent,
                         protocol,
@@ -125,21 +162,45 @@ export function createEngine(options) {
                         ...(options.seed !== undefined ? { seed: options.seed } : {}),
                         signal: abortController.signal,
                         ...(terminate ? { terminate } : {}),
+                        currentDepth: 0,
+                        effectiveMaxDepth,
+                        effectiveMaxConcurrentChildren,
+                        onChildFailure,
+                        ...(streamParentDeadlineMs !== undefined ? { parentDeadlineMs: streamParentDeadlineMs } : {}),
+                        ...(options.defaultSubRunTimeoutMs !== undefined
+                            ? { defaultSubRunTimeoutMs: options.defaultSubRunTimeoutMs }
+                            : {}),
+                        ...(options.tracer ? { tracer: options.tracer } : {}),
+                        ...(options.metricsHook ? { metricsHook: options.metricsHook } : {}),
+                        ...(options.logger ? { logger: options.logger } : {}),
+                        streamEvents: true,
                         emit(event) {
                             if (status !== "running") {
                                 return;
                             }
+                            const parentRunIds = event.parentRunIds;
+                            if (rootRunId === undefined && parentRunIds === undefined) {
+                                rootRunId = event.runId;
+                            }
                             lastRunId = event.runId;
-                            if (event.type === "final") {
+                            if (event.type === "final" && event.runId === rootRunId) {
                                 pendingFinalEvent = event;
                                 return;
                             }
                             publish(event);
-                        }
+                        },
+                        registerAbortDrain(drain) {
+                            activeAbortDrain = drain;
+                        },
+                        failureInstancesByChildRunId
                     }));
                     if (status !== "running") {
                         return;
                     }
+                    const terminalThrow = resolveRuntimeTerminalThrow(baseResult.trace, failureInstancesByChildRunId);
+                    if (terminalThrow) {
+                        throw terminalThrow;
+                    }
                     const finalizedResult = await abortRace.run(applyRunEvaluation(baseResult, options.evaluate));
                     if (status !== "running") {
                         return;
@@ -161,6 +222,10 @@ export function createEngine(options) {
                     }
                     const runtimeError = timeoutLifecycle.translateError(error);
                     status = isCancellationError(runtimeError) ? "cancelled" : "failed";
+                    if (shouldPublishAborted(runtimeError)) {
+                        activeAbortDrain?.(runtimeError);
+                        publish(createStreamAbortedEvent(runtimeError, lastRunId));
+                    }
                     publish(createStreamErrorEvent(runtimeError, lastRunId));
                     closeStream();
                     rejectResult(runtimeError);
@@ -171,9 +236,11 @@ export function createEngine(options) {
                     return;
                 }
                 const error = createStreamCancellationError(options.model.id, cause);
-                status = "cancelled";
                 abortController.abort(error);
+                activeAbortDrain?.(error);
+                publish(createStreamAbortedEvent(error, lastRunId));
                 publish(createStreamErrorEvent(error, lastRunId));
+                status = "cancelled";
                 closeStream();
                 rejectResult(error);
             }
@@ -182,6 +249,7 @@ export function createEngine(options) {
                     return;
                 }
                 complete = true;
+                failureInstancesByChildRunId.clear();
                 removeCallerAbortListener();
                 timeoutLifecycle.cleanup();
                 abortRace.cleanup();
@@ -242,7 +310,8 @@ function createNonStreamingAbortLifecycle(options) {
     const timeoutLifecycle = createTimeoutAbortLifecycle({
         abortController,
         timeoutMs: options.timeoutMs,
-        providerId: options.providerId
+        providerId: options.providerId,
+        timeoutErrorSource: options.timeoutErrorSource ?? "runtime"
     });
     const abortRace = createAbortRace(abortController.signal, options.providerId);
     const removeCallerAbortListener = wireCallerAbortSignal(options.callerSignal, abortController, () => {
@@ -272,7 +341,13 @@ function createTimeoutAbortLifecycle(options) {
             cleanup() { }
         };
     }
-    const timeoutError = createTimeoutError(options.providerId, options.timeoutMs);
+    const timeoutSource = classifyChildTimeoutSource(undefined, {
+        ...(options.timeoutErrorSource === "engine" ? { engineDefaultTimeoutMs: options.timeoutMs } : {}),
+        isProviderError: false
+    });
+    const timeoutError = options.timeoutErrorSource === "engine" && timeoutSource === "engine"
+        ? createEngineDeadlineTimeoutError(options.providerId, options.timeoutMs)
+        : createTimeoutError(options.providerId, options.timeoutMs);
     const timeoutId = setTimeout(() => {
         options.abortController.abort(timeoutError);
     }, options.timeoutMs);
@@ -354,6 +429,25 @@ function timeoutMsFromTermination(condition) {
 function readAbortSignalReason(signal) {
     return signal?.aborted ? signal.reason : undefined;
 }
+function createStreamAbortedEvent(error, runId) {
+    return {
+        type: "aborted",
+        runId,
+        at: new Date().toISOString(),
+        reason: streamAbortedReason(error)
+    };
+}
+function shouldPublishAborted(error) {
+    return DogpileError.isInstance(error) && (error.code === "aborted" || error.code === "timeout");
+}
+function streamAbortedReason(error) {
+    if (DogpileError.isInstance(error)) {
+        if (error.code === "timeout" || error.detail?.["reason"] === "timeout") {
+            return "timeout";
+        }
+    }
+    return "parent-aborted";
+}
 function createStreamErrorEvent(error, runId) {
     if (DogpileError.isInstance(error)) {
         return {
@@ -399,11 +493,398 @@ function dogpileErrorStreamDetail(error) {
     }
     return detail;
 }
+function openRunTracing(options) {
+    if (!options.tracer) {
+        return undefined;
+    }
+    const runSpan = options.tracer.startSpan(DOGPILE_SPAN_NAMES.RUN, {
+        ...(options.parentSpan ? { parent: options.parentSpan } : {}),
+        attributes: {
+            "dogpile.run.protocol": options.protocolKind,
+            "dogpile.run.tier": String(options.tier),
+            "dogpile.run.intent": options.intent.slice(0, 200)
+        }
+    });
+    return {
+        tracer: options.tracer,
+        runSpan,
+        subRunSpans: new Map(),
+        agentTurnSpans: new Map(),
+        modelCallSpans: new Map(),
+        pendingModelRequests: new Map(),
+        agentTurnCounters: new Map(),
+        turnAccumByAgent: new Map(),
+        agentIds: new Set(),
+        turnCount: 0,
+        lastCost: emptyCost()
+    };
+}
+function openRunMetrics(options) {
+    if (!options.metricsHook) {
+        return undefined;
+    }
+    return {
+        metricsHook: options.metricsHook,
+        logger: options.logger,
+        startedAtMs: Date.now(),
+        subRunStartTimes: new Map(),
+        totalCost: emptyCost(),
+        nestedCost: emptyCost(),
+        turns: 0
+    };
+}
+function routeMetricsError(err, logger) {
+    const msg = err instanceof Error ? err.message : String(err);
+    try {
+        if (logger !== undefined) {
+            logger.error("dogpile:metricsHook threw", { error: msg });
+        }
+        else {
+            console.error("dogpile:metricsHook threw", { error: msg });
+        }
+    }
+    catch {
+        // A logger that throws from error() cannot be helped.
+    }
+}
+function fireHook(callback, snapshot, logger) {
+    if (!callback) {
+        return;
+    }
+    try {
+        const result = callback(snapshot);
+        if (result && typeof result.catch === "function") {
+            result.catch((err) => {
+                routeMetricsError(err, logger);
+            });
+        }
+    }
+    catch (err) {
+        routeMetricsError(err, logger);
+    }
+}
+function buildRunSnapshot(result, startedAtMs) {
+    const nestedCosts = nestedSubRunCosts(result);
+    const budgetStopEvent = result.trace.events.find((event) => event.type === "budget-stop");
+    const outcome = budgetStopEvent !== undefined ? "budget-stopped" : "completed";
+    const totalInputTokens = result.cost.inputTokens;
+    const totalOutputTokens = result.cost.outputTokens;
+    const totalCostUsd = result.cost.usd;
+    const ownInputTokens = totalInputTokens - nestedCosts.reduce((sum, cost) => sum + cost.inputTokens, 0);
+    const ownOutputTokens = totalOutputTokens - nestedCosts.reduce((sum, cost) => sum + cost.outputTokens, 0);
+    const ownCostUsd = totalCostUsd - nestedCosts.reduce((sum, cost) => sum + cost.usd, 0);
+    const turns = result.trace.events.filter((event) => event.type === "agent-turn").length;
+    return {
+        outcome,
+        inputTokens: ownInputTokens,
+        outputTokens: ownOutputTokens,
+        costUsd: ownCostUsd,
+        totalInputTokens,
+        totalOutputTokens,
+        totalCostUsd,
+        turns,
+        durationMs: Date.now() - startedAtMs
+    };
+}
+function buildSubRunSnapshot(subResult, durationMs) {
+    const nestedCosts = nestedSubRunCosts(subResult);
+    const budgetStopEvent = subResult.trace.events.find((event) => event.type === "budget-stop");
+    const outcome = budgetStopEvent !== undefined ? "budget-stopped" : "completed";
+    const totalInputTokens = subResult.cost.inputTokens;
+    const totalOutputTokens = subResult.cost.outputTokens;
+    const totalCostUsd = subResult.cost.usd;
+    const ownInputTokens = totalInputTokens - nestedCosts.reduce((sum, cost) => sum + cost.inputTokens, 0);
+    const ownOutputTokens = totalOutputTokens - nestedCosts.reduce((sum, cost) => sum + cost.outputTokens, 0);
+    const ownCostUsd = totalCostUsd - nestedCosts.reduce((sum, cost) => sum + cost.usd, 0);
+    const turns = subResult.trace.events.filter((event) => event.type === "agent-turn").length;
+    return {
+        outcome,
+        inputTokens: ownInputTokens,
+        outputTokens: ownOutputTokens,
+        costUsd: ownCostUsd,
+        totalInputTokens,
+        totalOutputTokens,
+        totalCostUsd,
+        turns,
+        durationMs
+    };
+}
+function nestedSubRunCosts(result) {
+    return result.trace.events.flatMap((event) => {
+        if (event.type === "sub-run-completed") {
+            return [event.subResult.cost];
+        }
+        if (event.type === "sub-run-failed") {
+            return [event.partialCost];
+        }
+        return [];
+    });
+}
+function subtractCost(total, nested) {
+    return {
+        usd: total.usd - nested.usd,
+        inputTokens: total.inputTokens - nested.inputTokens,
+        outputTokens: total.outputTokens - nested.outputTokens,
+        totalTokens: total.totalTokens - nested.totalTokens
+    };
+}
+function handleMetricsEvent(state, event) {
+    const parentRunIds = event.parentRunIds;
+    if (parentRunIds !== undefined) {
+        return;
+    }
+    switch (event.type) {
+        case "agent-turn": {
+            state.totalCost = event.cost;
+            state.turns += 1;
+            break;
+        }
+        case "broadcast":
+        case "budget-stop":
+        case "final": {
+            state.totalCost = event.cost;
+            break;
+        }
+        case "sub-run-started": {
+            state.subRunStartTimes.set(event.childRunId, Date.now());
+            break;
+        }
+        case "sub-run-completed": {
+            state.totalCost = addCost(state.totalCost, event.subResult.cost);
+            state.nestedCost = addCost(state.nestedCost, event.subResult.cost);
+            const startMs = state.subRunStartTimes.get(event.childRunId);
+            const durationMs = startMs !== undefined ? Date.now() - startMs : 0;
+            state.subRunStartTimes.delete(event.childRunId);
+            const snapshot = buildSubRunSnapshot(event.subResult, durationMs);
+            fireHook(state.metricsHook.onSubRunComplete, snapshot, state.logger);
+            break;
+        }
+        case "sub-run-failed": {
+            state.totalCost = addCost(state.totalCost, event.partialCost);
+            state.nestedCost = addCost(state.nestedCost, event.partialCost);
+            state.subRunStartTimes.delete(event.childRunId);
+            break;
+        }
+        default:
+            break;
+    }
+}
+function closeRunMetrics(state, result) {
+    if (result !== undefined) {
+        const snapshot = buildRunSnapshot(result, state.startedAtMs);
+        fireHook(state.metricsHook.onRunComplete, snapshot, state.logger);
+        return;
+    }
+    const ownCost = subtractCost(state.totalCost, state.nestedCost);
+    const snapshot = {
+        outcome: "aborted",
+        inputTokens: ownCost.inputTokens,
+        outputTokens: ownCost.outputTokens,
+        costUsd: ownCost.usd,
+        totalInputTokens: state.totalCost.inputTokens,
+        totalOutputTokens: state.totalCost.outputTokens,
+        totalCostUsd: state.totalCost.usd,
+        turns: state.turns,
+        durationMs: Date.now() - state.startedAtMs
+    };
+    fireHook(state.metricsHook.onRunComplete, snapshot, state.logger);
+}
+function handleTracingEvent(state, event) {
+    const parentRunIds = event.parentRunIds;
+    if (parentRunIds !== undefined) {
+        return;
+    }
+    if (state.runId === undefined) {
+        state.runId = event.runId;
+        state.runSpan.setAttribute("dogpile.run.id", event.runId);
+    }
+    switch (event.type) {
+        case "model-request": {
+            state.pendingModelRequests.set(event.callId, event);
+            state.agentIds.add(event.agentId);
+            if (!state.agentTurnSpans.has(event.agentId)) {
+                const turnNumber = (state.agentTurnCounters.get(event.agentId) ?? 0) + 1;
+                state.agentTurnCounters.set(event.agentId, turnNumber);
+                const turnParent = state.subRunSpans.get(event.runId) ?? state.runSpan;
+                const turnSpan = state.tracer.startSpan(DOGPILE_SPAN_NAMES.AGENT_TURN, {
+                    parent: turnParent,
+                    attributes: {
+                        "dogpile.agent.id": event.agentId,
+                        "dogpile.agent.role": event.role,
+                        "dogpile.turn.number": turnNumber,
+                        "dogpile.model.id": event.modelId
+                    }
+                });
+                state.agentTurnSpans.set(event.agentId, turnSpan);
+            }
+            const callParent = state.agentTurnSpans.get(event.agentId) ??
+                state.subRunSpans.get(event.runId) ??
+                state.runSpan;
+            const callSpan = state.tracer.startSpan(DOGPILE_SPAN_NAMES.MODEL_CALL, {
+                parent: callParent,
+                attributes: {
+                    "dogpile.model.id": event.modelId,
+                    "dogpile.call.id": event.callId,
+                    "dogpile.provider.id": event.providerId
+                }
+            });
+            state.modelCallSpans.set(event.callId, callSpan);
+            break;
+        }
+        case "model-response": {
+            const span = state.modelCallSpans.get(event.callId);
+            if (span) {
+                const inputTokens = event.response.usage?.inputTokens ?? 0;
+                const outputTokens = event.response.usage?.outputTokens ?? 0;
+                const responseCost = {
+                    usd: event.response.costUsd ?? 0,
+                    inputTokens,
+                    outputTokens,
+                    totalTokens: event.response.usage?.totalTokens ?? inputTokens + outputTokens
+                };
+                span.setAttribute("dogpile.model.input_tokens", inputTokens);
+                span.setAttribute("dogpile.model.output_tokens", outputTokens);
+                if (event.response.costUsd !== undefined) {
+                    span.setAttribute("dogpile.model.cost_usd", event.response.costUsd);
+                }
+                span.setStatus("ok");
+                span.end();
+                state.modelCallSpans.delete(event.callId);
+                const accum = state.turnAccumByAgent.get(event.agentId) ?? {
+                    inputTokens: 0,
+                    outputTokens: 0,
+                    costUsd: 0
+                };
+                accum.inputTokens += inputTokens;
+                accum.outputTokens += outputTokens;
+                accum.costUsd += responseCost.usd;
+                state.turnAccumByAgent.set(event.agentId, accum);
+                state.lastCost = addCost(state.lastCost, responseCost);
+            }
+            state.pendingModelRequests.delete(event.callId);
+            break;
+        }
+        case "agent-turn": {
+            state.agentIds.add(event.agentId);
+            state.turnCount += 1;
+            state.lastCost = event.cost;
+            const turnSpan = state.agentTurnSpans.get(event.agentId);
+            if (turnSpan) {
+                turnSpan.setAttribute("dogpile.agent.role", event.role);
+                const accum = state.turnAccumByAgent.get(event.agentId);
+                turnSpan.setAttribute("dogpile.turn.cost_usd", accum?.costUsd ?? 0);
+                turnSpan.setAttribute("dogpile.turn.input_tokens", accum?.inputTokens ?? 0);
+                turnSpan.setAttribute("dogpile.turn.output_tokens", accum?.outputTokens ?? 0);
+                turnSpan.setStatus("ok");
+                turnSpan.end();
+                state.agentTurnSpans.delete(event.agentId);
+            }
+            state.turnAccumByAgent.delete(event.agentId);
+            break;
+        }
+        case "broadcast":
+        case "budget-stop":
+        case "final": {
+            state.lastCost = event.cost;
+            break;
+        }
+        case "sub-run-started": {
+            const span = state.tracer.startSpan(DOGPILE_SPAN_NAMES.SUB_RUN, {
+                parent: state.runSpan,
+                attributes: {
+                    "dogpile.sub_run.child_run_id": event.childRunId,
+                    "dogpile.sub_run.parent_run_id": event.parentRunId,
+                    "dogpile.sub_run.depth": event.depth
+                }
+            });
+            state.subRunSpans.set(event.childRunId, span);
+            break;
+        }
+        case "sub-run-completed": {
+            const span = state.subRunSpans.get(event.childRunId);
+            if (span) {
+                span.setStatus("ok");
+                span.end();
+                state.subRunSpans.delete(event.childRunId);
+            }
+            break;
+        }
+        case "sub-run-failed": {
+            const span = state.subRunSpans.get(event.childRunId);
+            if (span) {
+                span.setStatus("error", event.error.message);
+                span.end();
+                state.subRunSpans.delete(event.childRunId);
+            }
+            break;
+        }
+        default:
+            break;
+    }
+}
+function closeRunTracing(state, result, error) {
+    if (error !== undefined) {
+        if (state.runId !== undefined) {
+            state.runSpan.setAttribute("dogpile.run.id", state.runId);
+        }
+        state.runSpan.setAttribute("dogpile.run.agent_count", state.agentIds.size);
+        state.runSpan.setAttribute("dogpile.run.turn_count", state.turnCount);
+        state.runSpan.setAttribute("dogpile.run.cost_usd", state.lastCost.usd);
+        state.runSpan.setAttribute("dogpile.run.input_tokens", state.lastCost.inputTokens);
+        state.runSpan.setAttribute("dogpile.run.output_tokens", state.lastCost.outputTokens);
+        state.runSpan.setAttribute("dogpile.run.outcome", "aborted");
+        state.runSpan.setStatus("error", error instanceof Error ? error.message : String(error));
+        closeOpenTracingSpans(state);
+        state.runSpan.end();
+        return;
+    }
+    if (result === undefined) {
+        closeOpenTracingSpans(state);
+        state.runSpan.end();
+        return;
+    }
+    const budgetStopEvent = result.trace.events.find((event) => event.type === "budget-stop");
+    const terminationReason = budgetStopEvent?.reason;
+    const outcome = terminationReason !== undefined ? "budget-stopped" : "completed";
+    state.runSpan.setAttribute("dogpile.run.id", result.trace.runId);
+    state.runSpan.setAttribute("dogpile.run.agent_count", result.trace.agentsUsed.length);
+    state.runSpan.setAttribute("dogpile.run.turn_count", result.trace.events.filter((event) => event.type === "agent-turn").length);
+    state.runSpan.setAttribute("dogpile.run.cost_usd", result.cost.usd);
+    state.runSpan.setAttribute("dogpile.run.input_tokens", result.cost.inputTokens);
+    state.runSpan.setAttribute("dogpile.run.output_tokens", result.cost.outputTokens);
+    state.runSpan.setAttribute("dogpile.run.outcome", outcome);
+    if (terminationReason !== undefined) {
+        state.runSpan.setAttribute("dogpile.run.termination_reason", terminationReason);
+    }
+    state.runSpan.setStatus("ok");
+    closeOpenTracingSpans(state);
+    state.runSpan.end();
+}
+function closeOpenTracingSpans(state) {
+    for (const span of state.modelCallSpans.values()) {
+        span.end();
+    }
+    state.modelCallSpans.clear();
+    for (const span of state.agentTurnSpans.values()) {
+        span.end();
+    }
+    state.agentTurnSpans.clear();
+    for (const span of state.subRunSpans.values()) {
+        span.end();
+    }
+    state.subRunSpans.clear();
+}
 async function runNonStreamingProtocol(options) {
+    const failureInstancesByChildRunId = new Map();
     const abortLifecycle = createNonStreamingAbortLifecycle({
         callerSignal: options.signal,
         timeoutMs: runtimeTimeoutMs(options),
-        providerId: options.model.id
+        providerId: options.model.id,
+        timeoutErrorSource: options.currentDepth !== undefined &&
+            options.currentDepth > 0 &&
+            options.parentDeadlineMs === undefined
+            ? "engine"
+            : "runtime"
     });
     try {
         const emittedEvents = [];
@@ -412,7 +893,8 @@ async function runNonStreamingProtocol(options) {
             ...(abortLifecycle.signal !== undefined ? { signal: abortLifecycle.signal } : {}),
             emit(event) {
                 emittedEvents.push(event);
-            }
+            },
+            failureInstancesByChildRunId
         }));
         const events = emittedEvents.length > 0 ? emittedEvents : result.trace.events;
         const trace = {
@@ -431,14 +913,20 @@ async function runNonStreamingProtocol(options) {
                 events
             }),
             eventLog: createRunEventLog(trace.runId, trace.protocol, events),
-            trace
+            trace,
+            health: computeHealth(trace, DEFAULT_HEALTH_THRESHOLDS)
         };
+        const terminalThrow = resolveRuntimeTerminalThrow(runResult.trace, failureInstancesByChildRunId);
+        if (terminalThrow) {
+            throw terminalThrow;
+        }
         return canonicalizeRunResult(await abortLifecycle.run(applyRunEvaluation(runResult, options.evaluate)));
     }
     catch (error) {
         throw abortLifecycle.translateError(error);
     }
     finally {
+        failureInstancesByChildRunId.clear();
         abortLifecycle.cleanup();
     }
 }
@@ -472,7 +960,56 @@ function finalEventWithEvaluation(event, evaluation) {
         evaluation
     };
 }
-function runProtocol(options) {
+async function runProtocol(options) {
+    const tracing = openRunTracing({
+        ...(options.tracer ? { tracer: options.tracer } : {}),
+        ...(options.parentSpan ? { parentSpan: options.parentSpan } : {}),
+        intent: options.intent,
+        protocolKind: options.protocol.kind,
+        tier: options.tier
+    });
+    const metrics = openRunMetrics({
+        ...(options.metricsHook ? { metricsHook: options.metricsHook } : {}),
+        ...(options.logger ? { logger: options.logger } : {})
+    });
+    const emitForProtocol = tracing || metrics || options.emit
+        ? (event) => {
+            if (tracing) {
+                handleTracingEvent(tracing, event);
+            }
+            if (metrics) {
+                handleMetricsEvent(metrics, event);
+            }
+            options.emit?.(event);
+        }
+        : undefined;
+    const protocolOptions = tracing
+        ? {
+            ...options,
+            subRunSpansByChildId: tracing.subRunSpans
+        }
+        : options;
+    try {
+        const result = await runProtocolInner(protocolOptions, emitForProtocol);
+        if (tracing) {
+            closeRunTracing(tracing, result);
+        }
+        if (metrics && (options.currentDepth === 0 || options.currentDepth === undefined)) {
+            closeRunMetrics(metrics, result);
+        }
+        return result;
+    }
+    catch (error) {
+        if (tracing) {
+            closeRunTracing(tracing, undefined, error);
+        }
+        if (metrics && (options.currentDepth === 0 || options.currentDepth === undefined)) {
+            closeRunMetrics(metrics, undefined);
+        }
+        throw error;
+    }
+}
+function runProtocolInner(options, emitForProtocol) {
     switch (options.protocol.kind) {
         case "sequential":
             return runSequential({
@@ -488,7 +1025,7 @@ function runProtocol(options) {
                 ...(options.signal !== undefined ? { signal: options.signal } : {}),
                 ...(options.terminate ? { terminate: options.terminate } : {}),
                 ...(options.wrapUpHint ? { wrapUpHint: options.wrapUpHint } : {}),
-                ...(options.emit ? { emit: options.emit } : {})
+                ...(emitForProtocol ? { emit: emitForProtocol } : {})
             });
         case "broadcast":
             return runBroadcast({
@@ -504,7 +1041,7 @@ function runProtocol(options) {
                 ...(options.signal !== undefined ? { signal: options.signal } : {}),
                 ...(options.terminate ? { terminate: options.terminate } : {}),
                 ...(options.wrapUpHint ? { wrapUpHint: options.wrapUpHint } : {}),
-                ...(options.emit ? { emit: options.emit } : {})
+                ...(emitForProtocol ? { emit: emitForProtocol } : {})
             });
         case "coordinator":
             return runCoordinator({
@@ -520,7 +1057,31 @@ function runProtocol(options) {
                 ...(options.signal !== undefined ? { signal: options.signal } : {}),
                 ...(options.terminate ? { terminate: options.terminate } : {}),
                 ...(options.wrapUpHint ? { wrapUpHint: options.wrapUpHint } : {}),
-                ...(options.emit ? { emit: options.emit } : {})
+                ...(emitForProtocol ? { emit: emitForProtocol } : {}),
+                ...(options.streamEvents !== undefined ? { streamEvents: options.streamEvents } : {}),
+                currentDepth: options.currentDepth ?? 0,
+                effectiveMaxDepth: options.effectiveMaxDepth ?? Infinity,
+                effectiveMaxConcurrentChildren: options.effectiveMaxConcurrentChildren ?? DEFAULT_MAX_CONCURRENT_CHILDREN,
+                onChildFailure: options.onChildFailure ?? "continue",
+                ...(options.parentDeadlineMs !== undefined ? { parentDeadlineMs: options.parentDeadlineMs } : {}),
+                ...(options.defaultSubRunTimeoutMs !== undefined
+                    ? { defaultSubRunTimeoutMs: options.defaultSubRunTimeoutMs }
+                    : {}),
+                ...(options.registerAbortDrain !== undefined ? { registerAbortDrain: options.registerAbortDrain } : {}),
+                ...(options.failureInstancesByChildRunId !== undefined
+                    ? { failureInstancesByChildRunId: options.failureInstancesByChildRunId }
+                    : {}),
+                runProtocol: (childInput) => {
+                    const { runId: childRunId, ...childProtocolInput } = childInput;
+                    const childParent = options.subRunSpansByChildId?.get(childRunId) ?? options.parentSpan;
+                    return runProtocol({
+                        ...childProtocolInput,
+                        protocol: normalizeProtocol(childProtocolInput.protocol),
+                        ...(options.tracer ? { tracer: options.tracer } : {}),
+                        ...(childParent ? { parentSpan: childParent } : {}),
+                        ...(options.logger ? { logger: options.logger } : {})
+                    });
+                }
             });
         case "shared":
             return runShared({
@@ -536,7 +1097,7 @@ function runProtocol(options) {
                 ...(options.signal !== undefined ? { signal: options.signal } : {}),
                 ...(options.terminate ? { terminate: options.terminate } : {}),
                 ...(options.wrapUpHint ? { wrapUpHint: options.wrapUpHint } : {}),
-                ...(options.emit ? { emit: options.emit } : {})
+                ...(emitForProtocol ? { emit: emitForProtocol } : {})
             });
     }
 }
@@ -584,13 +1145,29 @@ export function stream(options) {
  * the ergonomic {@link RunResult} wrapper from the JSON-serializable
  * {@link Trace} returned by a previous `run()`, `stream()`, or
  * `Dogpile.pile()` call.
+ *
+ * Tracing and metrics: replay is intentionally tracing-free and metrics-free.
+ * Even when an engine instance has been configured with a `tracer` or
+ * `metricsHook` on its `EngineOptions`, calling this function emits no spans
+ * or callbacks — replaying historical events with current timestamps would
+ * confuse observability backends. See `docs/developer-usage.md`.
  */
+// Tracing/metrics-free: replay never uses EngineOptions tracer or metricsHook.
 export function replay(trace) {
     const cost = trace.finalOutput.cost;
     const lastEvent = trace.events.at(-1);
+    // D-08 / D-10: rebuild accounting recursively from the saved trace and
+    // verify every embedded sub-run's recorded accounting matches what the
+    // child trace recomputes. Mismatches throw `invalid-configuration` with
+    // `detail.reason: "trace-accounting-mismatch"`. No provider invocation.
+    const accounting = recomputeAccountingFromTrace(trace);
+    const replayThrow = resolveReplayTerminalThrow(trace);
+    if (replayThrow) {
+        throw replayThrow;
+    }
     const baseResult = {
         output: trace.finalOutput.output,
-        eventLog: createRunEventLog(trace.runId, trace.protocol, trace.events),
+        eventLog: createRunEventLog(trace.runId, trace.protocol, synthesizeProviderEvents(trace, trace.providerCalls)),
         trace,
         transcript: trace.transcript,
         usage: createRunUsage(cost),
@@ -602,14 +1179,9 @@ export function replay(trace) {
             agentsUsed: trace.agentsUsed,
             events: trace.events
         }),
-        accounting: createRunAccounting({
-            tier: trace.tier,
-            ...(trace.budget.caps ? { budget: trace.budget.caps } : {}),
-            ...(trace.budget.termination ? { termination: trace.budget.termination } : {}),
-            cost,
-            events: trace.events
-        }),
-        cost
+        accounting,
+        cost,
+        health: computeHealth(trace, DEFAULT_HEALTH_THRESHOLDS)
     };
     if (lastEvent?.type !== "final") {
         return baseResult;
@@ -620,17 +1192,143 @@ export function replay(trace) {
         ...(lastEvent.evaluation !== undefined ? { evaluation: lastEvent.evaluation } : {})
     };
 }
+function synthesizeProviderEvents(trace, providerCalls) {
+    const hasLiveProvenance = trace.events.some((event) => event.type === "model-request" || event.type === "model-response");
+    if (hasLiveProvenance) {
+        return trace.events;
+    }
+    const baseEvents = trace.events.filter((event) => event.type !== "model-request" && event.type !== "model-response");
+    const result = [];
+    let turnCount = 0;
+    for (const event of baseEvents) {
+        if (event.type === "agent-turn") {
+            const call = providerCalls[turnCount];
+            if (call !== undefined) {
+                const modelId = typeof call.modelId === "string" && call.modelId.length > 0 ? call.modelId : call.providerId;
+                result.push({
+                    type: "model-request",
+                    runId: trace.runId,
+                    callId: call.callId,
+                    providerId: call.providerId,
+                    modelId,
+                    startedAt: call.startedAt,
+                    agentId: call.agentId,
+                    role: call.role,
+                    request: call.request
+                });
+                result.push({
+                    type: "model-response",
+                    runId: trace.runId,
+                    callId: call.callId,
+                    providerId: call.providerId,
+                    modelId,
+                    startedAt: call.startedAt,
+                    completedAt: call.completedAt,
+                    agentId: call.agentId,
+                    role: call.role,
+                    response: call.response
+                });
+            }
+            turnCount += 1;
+        }
+        result.push(event);
+    }
+    return result;
+}
+function resolveRuntimeTerminalThrow(trace, failureInstancesByChildRunId) {
+    if (trace.triggeringFailureForAbortMode !== undefined) {
+        return failureInstancesByChildRunId.get(trace.triggeringFailureForAbortMode.childRunId) ?? null;
+    }
+    const finalEvent = trace.events.at(-1);
+    if (finalEvent?.type !== "final" || finalEvent.termination === undefined) {
+        return null;
+    }
+    const lastFailure = findLastRealFailure(trace.events, failureInstancesByChildRunId);
+    if (lastFailure === null) {
+        return null;
+    }
+    if (hasFinalSynthesisAfterEvent(trace, lastFailure.eventIndex)) {
+        return null;
+    }
+    return lastFailure.error;
+}
+function findLastRealFailure(events, failureInstancesByChildRunId) {
+    for (let index = events.length - 1; index >= 0; index -= 1) {
+        const event = events[index];
+        if (event?.type !== "sub-run-failed") {
+            continue;
+        }
+        const instance = failureInstancesByChildRunId.get(event.childRunId);
+        if (instance) {
+            return { error: instance, eventIndex: index };
+        }
+    }
+    return null;
+}
+function resolveReplayTerminalThrow(trace) {
+    if (trace.triggeringFailureForAbortMode !== undefined) {
+        return dogpileErrorFromSerializedPayload(trace.triggeringFailureForAbortMode.error);
+    }
+    const finalEvent = trace.events.at(-1);
+    if (finalEvent?.type !== "final" || finalEvent.termination === undefined) {
+        return null;
+    }
+    const lastFailure = reconstructLastRealFailure(trace.events);
+    if (lastFailure === null) {
+        return null;
+    }
+    if (hasFinalSynthesisAfterEvent(trace, lastFailure.eventIndex)) {
+        return null;
+    }
+    return lastFailure.error;
+}
+function reconstructLastRealFailure(events) {
+    for (let index = events.length - 1; index >= 0; index -= 1) {
+        const event = events[index];
+        if (event?.type !== "sub-run-failed" || isSyntheticSubRunFailure(event)) {
+            continue;
+        }
+        return { error: dogpileErrorFromSerializedPayload(event.error), eventIndex: index };
+    }
+    return null;
+}
+function hasFinalSynthesisAfterEvent(trace, eventIndex) {
+    return trace.protocolDecisions.some((decision) => {
+        return decision.phase === "final-synthesis" && decision.eventIndex > eventIndex;
+    });
+}
+function isSyntheticSubRunFailure(event) {
+    const reason = event.error.detail?.["reason"];
+    return reason === "sibling-failed" || reason === "parent-aborted";
+}
+function dogpileErrorFromSerializedPayload(input) {
+    return new DogpileError({
+        code: input.code,
+        message: input.message,
+        ...(input.providerId !== undefined ? { providerId: input.providerId } : {}),
+        ...(input.detail !== undefined ? { detail: input.detail } : {})
+    });
+}
 /**
  * Replay a saved completed trace as a stream without invoking a model provider.
  *
  * @remarks
- * This is the streaming counterpart to {@link replay}. It yields the exact
- * saved {@link Trace.events} in order and resolves {@link StreamHandle.result}
- * to the rehydrated {@link RunResult}. Since all data comes from the trace,
- * replay remains storage-free and provider-free.
+ * This is the streaming counterpart to {@link replay}. It yields the same
+ * event sequence exposed by the replayed result event log, including legacy
+ * provenance synthesis when a saved trace predates model request/response
+ * events. Since all data comes from the trace, replay remains storage-free and
+ * provider-free.
+ *
+ * Tracing and metrics: replayStream is intentionally tracing-free and
+ * metrics-free. Even when an engine instance has been configured with a
+ * `tracer` or `metricsHook` on its `EngineOptions`, calling this function
+ * emits no spans or callbacks — replaying historical events with current
+ * timestamps would confuse observability backends. See `docs/developer-usage.md`.
  */
+// Tracing/metrics-free: replayStream never uses EngineOptions tracer or metricsHook.
 export function replayStream(trace) {
     const result = Promise.resolve(replay(trace));
+    const replayEvents = replayStreamEvents(trace);
     return {
         get status() {
             return "completed";
@@ -640,7 +1338,7 @@ export function replayStream(trace) {
             // Replay streams are already completed snapshots, so cancellation is a no-op.
         },
         subscribe(subscriber) {
-            for (const event of trace.events) {
+            for (const event of replayEvents) {
                 subscriber(event);
             }
             return {
@@ -653,7 +1351,7 @@ export function replayStream(trace) {
             let index = 0;
             return {
                 next() {
-                    const event = trace.events[index];
+                    const event = replayEvents[index];
                     if (event) {
                         index += 1;
                         return Promise.resolve({ done: false, value: event });
@@ -664,6 +1362,26 @@ export function replayStream(trace) {
         }
     };
 }
+function replayStreamEvents(trace, parentRunIds = []) {
+    const events = [];
+    for (const event of synthesizeProviderEvents(trace, trace.providerCalls)) {
+        if (event.type === "sub-run-completed") {
+            events.push(...replayStreamEvents(event.subResult.trace, [...parentRunIds, trace.runId]));
+        }
+        events.push(wrapReplayStreamEvent(event, parentRunIds));
+    }
+    return events;
+}
+function wrapReplayStreamEvent(event, parentRunIds) {
+    if (parentRunIds.length === 0) {
+        return event;
+    }
+    const inbound = event.parentRunIds;
+    return {
+        ...event,
+        parentRunIds: [...parentRunIds, ...(inbound ?? [])]
+    };
+}
 function wireCallerAbortSignal(callerSignal, abortController, cancelRun) {
     if (!callerSignal) {
         return () => { };
@@ -690,7 +1408,8 @@ function createStreamCancellationError(providerId, cause) {
         providerId,
         ...(cause !== undefined ? { cause } : {}),
         detail: {
-            status: "cancelled"
+            status: "cancelled",
+            reason: "parent-aborted"
         }
     });
 }
@@ -707,6 +1426,22 @@ function withHighLevelDefaults(options) {
         tier: options.tier ?? defaultHighLevelTier
     };
 }
+function assertRunDoesNotRaiseEngineMax(path, runValue, engineValue) {
+    if (runValue === undefined || runValue <= engineValue) {
+        return;
+    }
+    throw new DogpileError({
+        code: "invalid-configuration",
+        message: `${path} cannot raise the engine ceiling (${engineValue}).`,
+        retryable: false,
+        detail: {
+            kind: "configuration-validation",
+            path,
+            expected: `integer <= ${engineValue}`,
+            actual: runValue
+        }
+    });
+}
 /**
  * Branded high-level SDK namespace.
  *