@just-every/ensemble 0.2.211 → 0.2.213

package/dist/core/ensemble_request.js

@@ -9,9 +9,13 @@ import { waitWhilePaused } from '../utils/pause_controller.js';
 import { emitEvent } from '../utils/event_controller.js';
 import { createTraceContext } from '../utils/trace_context.js';
 import { convertToThinkingMessage, convertToOutputMessage, convertToFunctionCall, convertToFunctionCallOutput, } from '../utils/message_converter.js';
-import { truncateLargeValues } from '../utils/truncate_utils.js';
-const MAX_ERROR_ATTEMPTS = 5;
+import { createOperationGuard, normalizeFailure, RequestLifecycleController, selectMoreSevereFailure, streamWithAbortAndTimeout, toErrorEvent, } from '../utils/failure_detection.js';
+import { validateJsonResponseContent } from '../utils/json_schema.js';
+import { runningToolTracker } from '../utils/running_tool_tracker.js';
+import { calculateDelay } from '../utils/retry_handler.js';
+const DEFAULT_MAX_ERROR_RETRIES = 4;
 const DEFAULT_TERMINAL_TOOL_NAMES = new Set(['task_complete', 'task_fatal_error']);
+const TOOL_FAILURE_FINALIZATION_TIMEOUT_MS = 50;
 const getTerminalToolNames = (agent) => {
     const toolNames = new Set(DEFAULT_TERMINAL_TOOL_NAMES);
     for (const name of agent.terminalToolNames ?? []) {
@@ -21,9 +25,81 @@ const getTerminalToolNames = (agent) => {
     }
     return toolNames;
 };
+const hasTerminalTextContent = (content, expectsStructuredOutput) => {
+    if (typeof content !== 'string') {
+        return false;
+    }
+    return expectsStructuredOutput ? content.trim().length > 0 : content.length > 0;
+};
+const getMaxErrorRetries = (agent) => {
+    const configuredMaxRetries = agent.retryOptions?.maxRetries;
+    if (typeof configuredMaxRetries !== 'number' || Number.isNaN(configuredMaxRetries)) {
+        return DEFAULT_MAX_ERROR_RETRIES;
+    }
+    return Math.max(0, Math.floor(configuredMaxRetries));
+};
+const waitForRetryDelay = async (delayMs, abortSignal) => {
+    if (delayMs <= 0) {
+        return;
+    }
+    await new Promise((resolve, reject) => {
+        if (abortSignal?.aborted) {
+            reject(abortSignal.reason ?? new Error('Retry wait aborted'));
+            return;
+        }
+        const timeoutId = setTimeout(() => {
+            if (abortSignal && abortListener) {
+                abortSignal.removeEventListener('abort', abortListener);
+            }
+            resolve();
+        }, delayMs);
+        const abortListener = abortSignal
+            ? () => {
+                clearTimeout(timeoutId);
+                abortSignal.removeEventListener('abort', abortListener);
+                reject(abortSignal.reason ?? new Error('Retry wait aborted'));
+            }
+            : undefined;
+        if (abortSignal && abortListener) {
+            abortSignal.addEventListener('abort', abortListener, { once: true });
+        }
+    });
+};
+const getOuterRequestTimeoutMs = (agent) => {
+    const timeoutMs = agent.modelSettings?.timeout_ms;
+    if (typeof timeoutMs !== 'number' || Number.isNaN(timeoutMs) || timeoutMs <= 0) {
+        return undefined;
+    }
+    return Math.floor(timeoutMs);
+};
+const getRemainingRequestTimeoutMs = (requestTimeoutMs, requestStartedAt) => {
+    if (requestTimeoutMs === undefined || requestStartedAt === undefined) {
+        return undefined;
+    }
+    return Math.max(0, requestTimeoutMs - (Date.now() - requestStartedAt));
+};
+const createRequestTimeoutError = (model, timeoutMs) => {
+    const error = new Error(`Request generation for ${model} timed out after ${timeoutMs}ms`);
+    error.code = 'ETIMEDOUT';
+    error.recoverable = false;
+    return error;
+};
+const getFailureRetryOverrides = (agent) => ({
+    retryableErrors: agent.retryOptions?.additionalRetryableErrors,
+    retryableStatusCodes: agent.retryOptions?.additionalRetryableStatusCodes,
+});
 setEnsembleRequestFunction(ensembleRequest);
 setImageToTextFunction(ensembleRequest);
 export async function* ensembleRequest(messages, agent = {}) {
+    if (agent.jsonSchema && !agent.modelSettings?.json_schema) {
+        agent = {
+            ...agent,
+            modelSettings: {
+                ...agent.modelSettings,
+                json_schema: agent.jsonSchema,
+            },
+        };
+    }
     const conversationHistory = agent?.historyThread || messages;
     if (agent.instructions) {
         const alreadyHasInstructions = conversationHistory.some(msg => {
@@ -55,187 +131,228 @@ export async function* ensembleRequest(messages, agent = {}) {
         compactionThreshold: 0.7,
     });
     const trace = createTraceContext(agent, 'chat');
+    const lifecycle = new RequestLifecycleController();
+    const maxToolCalls = agent?.maxToolCalls ?? 200;
+    const maxRounds = agent?.maxToolCallRoundsPerTurn ?? Infinity;
+    const maxErrorRetries = getMaxErrorRetries(agent);
+    const maxErrorAttempts = maxErrorRetries + 1;
+    const outerRequestTimeoutMs = getOuterRequestTimeoutMs(agent);
+    const outerRequestStartedAt = outerRequestTimeoutMs !== undefined ? Date.now() : undefined;
+    const modelHistory = [];
+    let lastModelUsed;
     let totalToolCalls = 0;
     let toolCallRounds = 0;
     let errorRounds = 0;
+    let lastMessageContent = '';
     let turnStatus = 'completed';
     let turnEndReason = 'completed';
     let turnError;
-    const maxToolCalls = agent?.maxToolCalls ?? 200;
-    const maxRounds = agent?.maxToolCallRoundsPerTurn ?? Infinity;
-    let hasToolCalls = false;
-    let hasError = false;
-    let lastMessageContent = '';
-    const modelHistory = [];
+    let terminalFailure;
+    let terminalFailureEventEmitted = false;
+    let finalRound;
     await trace.emitTurnStart({
         input_messages: conversationHistory,
     });
     try {
-        do {
-            hasToolCalls = false;
-            hasError = false;
-            let terminalToolSucceededThisRound = false;
-            let currentRoundRequestId;
-            const currentRoundMessages = [];
-            const currentRoundErrors = [];
-            let currentRoundToolCalls = 0;
-            let currentRoundRequestDuration;
-            let currentRoundDurationWithTools;
-            let currentRoundRequestCost;
-            const terminalToolNames = getTerminalToolNames(agent);
+        const emitRoundAgentDone = async function* (round, model) {
+            if (!round.agentDoneEvent) {
+                return;
+            }
+            yield round.agentDoneEvent;
+            await emitEvent(round.agentDoneEvent, round.agentDoneAgent ?? agent, model);
+        };
+        while (!terminalFailure) {
             const model = await getModelFromAgent(agent, 'reasoning_mini', modelHistory);
+            const roundRequestId = randomUUID();
+            const startedStatusEvent = lifecycle.begin(roundRequestId);
             modelHistory.push(model);
-            const stream = executeRound(model, agent, history, totalToolCalls, maxToolCalls, trace);
-            try {
-                for await (const event of stream) {
-                    yield event;
-                    switch (event.type) {
-                        case 'agent_start': {
-                            currentRoundRequestId = event.request_id;
-                            break;
-                        }
-                        case 'message_complete': {
-                            const messageEvent = event;
-                            if (messageEvent.content) {
-                                lastMessageContent = messageEvent.content;
-                                currentRoundMessages.push(messageEvent.content);
-                            }
-                            break;
-                        }
-                        case 'tool_start': {
-                            const toolEvent = event;
-                            if (toolEvent.tool_call) {
-                                const toolName = toolEvent.tool_call.function.name;
-                                currentRoundToolCalls += 1;
-                                await trace.emitToolStart(event.request_id, toolEvent.tool_call.id, {
-                                    tool_name: toolName,
-                                    arguments: toolEvent.tool_call.function.arguments,
-                                    arguments_formatted: toolEvent.tool_call.function.arguments_formatted,
-                                });
-                                if (!terminalToolNames.has(toolName)) {
-                                    hasToolCalls = true;
-                                }
-                            }
-                            ++totalToolCalls;
-                            break;
-                        }
-                        case 'tool_done': {
-                            const toolEvent = event;
-                            if (toolEvent.tool_call) {
-                                const toolName = toolEvent.tool_call.function.name;
-                                if (terminalToolNames.has(toolName) && !toolEvent.result?.error) {
-                                    terminalToolSucceededThisRound = true;
-                                }
-                                await trace.emitToolDone(event.request_id, toolEvent.tool_call.id, {
-                                    tool_name: toolName,
-                                    call_id: toolEvent.result?.call_id,
-                                    output: toolEvent.result?.output,
-                                    error: toolEvent.result?.error,
-                                });
-                            }
-                            break;
-                        }
-                        case 'agent_done': {
-                            const agentDoneEvent = event;
-                            currentRoundRequestDuration = agentDoneEvent.request_duration;
-                            currentRoundDurationWithTools = agentDoneEvent.duration_with_tools;
-                            currentRoundRequestCost = agentDoneEvent.request_cost;
-                            break;
-                        }
-                        case 'error': {
-                            hasError = true;
-                            const errorEvent = event;
-                            if (errorEvent.error) {
-                                currentRoundErrors.push(String(errorEvent.error));
-                            }
-                            break;
-                        }
-                    }
+            lastModelUsed = model;
+            const round = yield* executeRound({
+                roundRequestId,
+                model,
+                agent,
+                history,
+                currentToolCalls: totalToolCalls,
+                maxToolCalls,
+                trace,
+                startedStatusEvent,
+                requestTimeoutMs: outerRequestTimeoutMs,
+                requestStartedAt: outerRequestStartedAt,
+            });
+            totalToolCalls += round.toolCallsStarted;
+            if (round.messages.length > 0) {
+                lastMessageContent = round.messages.at(-1) || lastMessageContent;
+            }
+            if (round.hasFollowupToolCalls) {
+                ++toolCallRounds;
+            }
+            const willRetryForError = (() => {
+                if (!round.failure) {
+                    return false;
                 }
+                ++errorRounds;
+                return !round.emittedTerminalOutput && round.failure.recoverable && errorRounds <= maxErrorRetries;
+            })();
+            const willContinueForTools = !round.failure &&
+                !round.terminalToolSucceeded &&
+                round.hasFollowupToolCalls &&
+                toolCallRounds < maxRounds &&
+                totalToolCalls < maxToolCalls;
+            let requestStatus = 'completed';
+            if (round.failure) {
+                requestStatus = willRetryForError ? 'error_retrying' : 'error';
             }
-            catch (roundError) {
-                hasError = true;
-                const errorMessage = roundError instanceof Error ? roundError.message : String(roundError);
-                currentRoundErrors.push(errorMessage);
-                yield {
-                    type: 'error',
-                    request_id: currentRoundRequestId,
-                    error: errorMessage,
-                    recoverable: true,
-                    timestamp: new Date().toISOString(),
-                };
+            else if (round.hasFollowupToolCalls && !round.terminalToolSucceeded) {
+                requestStatus = willContinueForTools ? 'waiting_for_followup_request' : 'tool_limit_reached';
+            }
+            await trace.emitRequestEnd(round.requestId, {
+                status: requestStatus,
+                will_continue: willRetryForError || willContinueForTools,
+                tool_calls: round.toolCallsStarted,
+                final_response: round.messages.length > 0 ? round.messages.join('\n') : undefined,
+                errors: round.errors.length > 0 ? round.errors : undefined,
+                request_duration_ms: round.requestDuration,
+                duration_with_tools_ms: round.durationWithTools,
+                request_cost: round.requestCost,
+            });
+            if (round.failure) {
+                const terminalRoundFailure = willRetryForError
+                    ? round.failure
+                    : {
+                        ...round.failure,
+                        recoverable: false,
+                        terminal: true,
+                    };
+                const errorEvent = toErrorEvent(terminalRoundFailure, {
+                    request_id: round.requestId,
+                });
+                yield errorEvent;
+                await emitEvent(errorEvent, agent, model);
+                if (willRetryForError) {
+                    agent.retryOptions?.onRetry?.({
+                        message: round.failure.error,
+                        code: round.failure.code,
+                        details: round.failure.details,
+                        recoverable: round.failure.recoverable,
+                    }, errorRounds);
+                    const retryingEvent = lifecycle.retrying(round.failure, errorRounds, maxErrorAttempts);
+                    if (retryingEvent) {
+                        yield retryingEvent;
+                        await emitEvent(retryingEvent, agent, model);
+                    }
+                    const retryDelayMs = calculateDelay(errorRounds, agent.retryOptions);
+                    const remainingTimeoutMs = getRemainingRequestTimeoutMs(outerRequestTimeoutMs, outerRequestStartedAt);
+                    const boundedRetryDelayMs = remainingTimeoutMs === undefined
+                        ? retryDelayMs
+                        : remainingTimeoutMs < retryDelayMs
+                            ? 0
+                            : retryDelayMs;
+                    yield* emitRoundAgentDone(round, model);
+                    await waitForRetryDelay(boundedRetryDelayMs, agent.abortSignal);
+                    continue;
+                }
+                terminalFailure = terminalRoundFailure;
+                terminalFailureEventEmitted = true;
+                finalRound = { round, model };
+                break;
             }
-            if (terminalToolSucceededThisRound) {
-                hasToolCalls = false;
-                hasError = false;
+            if (round.terminalToolSucceeded) {
+                finalRound = { round, model };
+                break;
             }
-            if (hasToolCalls) {
-                ++toolCallRounds;
+            if (willContinueForTools) {
                 if (agent.modelSettings?.tool_choice) {
+                    agent = {
+                        ...agent,
+                        modelSettings: {
+                            ...agent.modelSettings,
+                        },
+                    };
                     delete agent.modelSettings.tool_choice;
                 }
+                yield* emitRoundAgentDone(round, model);
+                continue;
             }
-            if (hasError) {
-                ++errorRounds;
-            }
-            const willRetryForError = hasError && errorRounds < MAX_ERROR_ATTEMPTS;
-            const willContinueForTools = hasToolCalls && toolCallRounds < maxRounds && totalToolCalls < maxToolCalls;
-            const willContinue = willRetryForError || willContinueForTools;
-            let requestStatus = 'completed';
-            if (hasError) {
-                requestStatus = willContinue ? 'error_retrying' : 'error';
-            }
-            else if (hasToolCalls) {
-                requestStatus = willContinue ? 'waiting_for_followup_request' : 'tool_limit_reached';
-            }
-            if (currentRoundRequestId) {
-                await trace.emitRequestEnd(currentRoundRequestId, {
-                    status: requestStatus,
-                    will_continue: willContinue,
-                    tool_calls: currentRoundToolCalls,
-                    final_response: currentRoundMessages.length > 0 ? currentRoundMessages.join('\n') : undefined,
-                    errors: currentRoundErrors.length > 0 ? currentRoundErrors : undefined,
-                    request_duration_ms: currentRoundRequestDuration,
-                    duration_with_tools_ms: currentRoundDurationWithTools,
-                    request_cost: currentRoundRequestCost,
+            if (round.hasFollowupToolCalls && !round.terminalToolSucceeded) {
+                terminalFailure = normalizeFailure(new Error(toolCallRounds >= maxRounds
+                    ? `Tool call rounds limit reached (${maxRounds}).`
+                    : `Tool call limit reached (${maxToolCalls}).`), {
+                    recoverable: false,
+                    reason: toolCallRounds >= maxRounds
+                        ? 'max_tool_call_rounds_reached'
+                        : 'max_tool_calls_reached',
+                    ...getFailureRetryOverrides(agent),
                 });
+                finalRound = { round, model };
+                break;
             }
-        } while ((hasError && errorRounds < MAX_ERROR_ATTEMPTS) ||
-            (hasToolCalls && toolCallRounds < maxRounds && totalToolCalls < maxToolCalls));
-        if (hasToolCalls && toolCallRounds >= maxRounds) {
-            console.log('[ensembleRequest] Tool call rounds limit reached');
-            turnEndReason = 'max_tool_call_rounds_reached';
+            finalRound = { round, model };
+            break;
         }
-        else if (hasToolCalls && totalToolCalls >= maxToolCalls) {
-            console.log('[ensembleRequest] Total tool calls limit reached');
-            turnEndReason = 'max_tool_calls_reached';
+        if (!terminalFailure && agent.verifier && lastMessageContent) {
+            const verification = yield* performVerification(agent, lastMessageContent, await history.getMessages());
+            if (!verification.passed) {
+                terminalFailure = normalizeFailure(new Error(verification.error || 'Verification failed'), {
+                    recoverable: false,
+                    reason: 'verification_failed',
+                    ...getFailureRetryOverrides(agent),
+                });
+            }
         }
-        else if (hasError && errorRounds >= MAX_ERROR_ATTEMPTS) {
+        if (terminalFailure) {
             turnStatus = 'error';
-            turnEndReason = 'max_error_attempts_reached';
+            turnEndReason = terminalFailure.reason || 'terminal_failure';
+            turnError = terminalFailure.error;
+            if (!terminalFailureEventEmitted) {
+                const errorEvent = toErrorEvent(terminalFailure, {
+                    request_id: lifecycle.getRequestId(),
+                });
+                yield errorEvent;
+                await emitEvent(errorEvent, agent, lastModelUsed);
+            }
+            const failedEvent = lifecycle.fail(terminalFailure, errorRounds || 1, maxErrorAttempts);
+            if (failedEvent) {
+                yield failedEvent;
+                await emitEvent(failedEvent, agent, lastModelUsed);
+            }
         }
-        if (agent?.verifier && lastMessageContent) {
-            const verificationResult = await performVerification(agent, lastMessageContent, await history.getMessages());
-            if (verificationResult) {
-                for await (const event of verificationResult) {
-                    yield event;
-                }
+        else {
+            const completedEvent = lifecycle.complete();
+            if (completedEvent) {
+                yield completedEvent;
+                await emitEvent(completedEvent, agent, lastModelUsed);
             }
         }
+        if (finalRound) {
+            yield* emitRoundAgentDone(finalRound.round, finalRound.model);
+        }
     }
     catch (err) {
-        const error = err;
+        if (!lifecycle.getRequestId()) {
+            const startedEvent = lifecycle.begin(randomUUID());
+            if (startedEvent) {
+                yield startedEvent;
+                await emitEvent(startedEvent, agent, lastModelUsed);
+            }
+        }
+        const failure = normalizeFailure(err, {
+            recoverable: false,
+            reason: 'exception',
+            ...getFailureRetryOverrides(agent),
+        });
         turnStatus = 'error';
         turnEndReason = 'exception';
-        turnError = error.message || 'Unknown error';
-        yield {
-            type: 'error',
-            error: error.message || 'Unknown error',
-            code: error.code,
-            details: error.details,
-            recoverable: error.recoverable,
-            timestamp: new Date().toISOString(),
-        };
+        turnError = failure.error;
+        const errorEvent = toErrorEvent(failure, {
+            request_id: lifecycle.getRequestId(),
+        });
+        yield errorEvent;
+        await emitEvent(errorEvent, agent, lastModelUsed);
+        const failedEvent = lifecycle.fail(failure, errorRounds || 1, maxErrorAttempts);
+        if (failedEvent) {
+            yield failedEvent;
+            await emitEvent(failedEvent, agent, lastModelUsed);
+        }
     }
     finally {
         await trace.emitTurnEnd(turnStatus, turnEndReason, {
@@ -250,14 +367,29 @@ export async function* ensembleRequest(messages, agent = {}) {
         };
     }
 }
-async function* executeRound(model, agent, history, currentToolCalls, maxToolCalls, trace) {
-    const requestId = randomUUID();
+async function* executeRound(options) {
+    const { roundRequestId, model, agent, history, currentToolCalls, maxToolCalls, trace, startedStatusEvent } = options;
     const startTime = Date.now();
     let totalCost = 0;
     let messages = await history.getMessages(model);
+    let roundAgentDefinition = agent;
+    let requestGuard;
+    let toolExecutionGuard;
+    let roundAgent = agent;
+    let provider;
+    let stream;
+    const roundSummary = {
+        requestId: roundRequestId,
+        messages: [],
+        errors: [],
+        toolCallsStarted: 0,
+        hasFollowupToolCalls: false,
+        emittedTerminalOutput: false,
+        terminalToolSucceeded: false,
+    };
     const agentStartEvent = {
         type: 'agent_start',
-        request_id: requestId,
+        request_id: roundRequestId,
         input: 'content' in messages[0] && typeof messages[0].content === 'string' ? messages[0].content : undefined,
         timestamp: new Date().toISOString(),
         agent: {
@@ -274,199 +406,447 @@ async function* executeRound(model, agent, history, currentToolCalls, maxToolCal
     };
     yield agentStartEvent;
     await emitEvent(agentStartEvent, agent, model);
-    if (agent.onRequest) {
-        [agent, messages] = await agent.onRequest(agent, messages);
+    try {
+        if (roundAgentDefinition.onRequest) {
+            const [nextAgent, nextMessages] = await roundAgentDefinition.onRequest(roundAgentDefinition, messages);
+            roundAgentDefinition = nextAgent;
+            messages = nextMessages;
+        }
+        const remainingTimeoutMs = getRemainingRequestTimeoutMs(options.requestTimeoutMs, options.requestStartedAt);
+        const needsRequestGuard = Boolean(roundAgentDefinition.abortSignal || remainingTimeoutMs !== undefined);
+        if (needsRequestGuard) {
+            if (options.requestTimeoutMs !== undefined && remainingTimeoutMs !== undefined && remainingTimeoutMs <= 0) {
+                throw createRequestTimeoutError(model, options.requestTimeoutMs);
+            }
+            requestGuard = createOperationGuard({
+                operationName: `Request generation for ${model}`,
+                abortSignal: roundAgentDefinition.abortSignal,
+                timeoutMs: remainingTimeoutMs,
+            });
+            roundAgent = {
+                ...roundAgentDefinition,
+                abortSignal: requestGuard.signal,
+            };
+        }
+        else {
+            roundAgent = roundAgentDefinition;
+        }
+        await waitWhilePaused(100, roundAgent.abortSignal);
+        toolExecutionGuard = createOperationGuard({
+            operationName: `Tool execution for ${model}`,
+            abortSignal: roundAgent.abortSignal,
+        });
+        if (startedStatusEvent) {
+            yield startedStatusEvent;
+            await emitEvent(startedStatusEvent, roundAgent, model);
+        }
+        provider = getModelProvider(model);
+        await trace.emitRequestStart(roundRequestId, {
+            agent_id: roundAgent.agent_id,
+            provider: provider.provider_id,
+            model,
+            payload: {
+                messages,
+                model_settings: roundAgent.modelSettings,
+                tool_names: roundAgent.tools?.map(tool => tool.definition.function.name) || [],
+            },
+        });
+        const rawStream = provider.createResponseStream(messages, model, roundAgent, roundRequestId);
+        stream = streamWithAbortAndTimeout(rawStream, {
+            abortSignal: requestGuard?.signal,
+        });
     }
-    await waitWhilePaused(100, agent.abortSignal);
-    const provider = getModelProvider(model);
-    await trace.emitRequestStart(requestId, {
-        agent_id: agent.agent_id,
-        provider: provider.provider_id,
-        model,
-        payload: {
-            messages,
-            model_settings: agent.modelSettings,
-            tool_names: agent.tools?.map(tool => tool.definition.function.name) || [],
-        },
-    });
-    const stream = 'createResponseStreamWithRetry' in provider
-        ? provider.createResponseStreamWithRetry(messages, model, agent, requestId)
-        : provider.createResponseStream(messages, model, agent, requestId);
-    const toolPromises = [];
+    catch (error) {
+        requestGuard?.cleanup();
+        toolExecutionGuard?.cleanup();
+        const failure = normalizeFailure(error, {
+            reason: 'request_setup_failed',
+            ...getFailureRetryOverrides(agent),
+        });
+        roundSummary.failure = failure;
+        roundSummary.errors.push(failure.error);
+        roundSummary.requestDuration = Date.now() - startTime;
+        roundSummary.durationWithTools = roundSummary.requestDuration;
+        roundSummary.agentDoneEvent = {
+            type: 'agent_done',
+            request_id: roundRequestId,
+            request_duration: roundSummary.requestDuration,
+            duration_with_tools: roundSummary.durationWithTools,
+            timestamp: new Date().toISOString(),
+        };
+        roundSummary.agentDoneAgent = roundAgentDefinition;
+        return roundSummary;
+    }
+    const terminalToolNames = getTerminalToolNames(roundAgent);
+    const expectsStructuredOutput = Boolean(roundAgent.modelSettings?.json_schema?.schema);
+    const structuredOutputSchema = roundAgent.modelSettings?.json_schema?.strict === true
+        ? roundAgent.modelSettings.json_schema.schema
+        : undefined;
+    const toolExecutions = [];
     const toolCallFormattedArgs = new Map();
     const toolEventBuffer = [];
     let sawToolCallThisRound = false;
-    agent.onToolEvent = async (event) => {
+    let sawTerminalProviderOutcome = false;
+    roundAgent.onToolEvent = async (event) => {
         toolEventBuffer.push(event);
     };
-    for await (let event of stream) {
-        event = { ...event, request_id: requestId };
-        if (event.type === 'tool_start') {
-            const toolEvent = event;
-            if (toolEvent.tool_call) {
-                const toolCall = toolEvent.tool_call;
-                let argumentsFormatted;
-                try {
-                    const tool = agent.tools?.find(t => t.definition.function.name === toolCall.function.name);
-                    if (tool && 'definition' in tool && tool.definition.function.parameters.properties) {
-                        const parsedArgs = JSON.parse(toolCall.function.arguments || '{}');
-                        if (typeof parsedArgs === 'object' && parsedArgs !== null && !Array.isArray(parsedArgs)) {
-                            const paramNames = Object.keys(tool.definition.function.parameters.properties);
-                            const orderedArgs = {};
-                            for (const param of paramNames) {
-                                if (param in parsedArgs) {
-                                    orderedArgs[param] = parsedArgs[param];
-                                }
-                            }
-                            argumentsFormatted = JSON.stringify(orderedArgs, null, 2);
-                        }
-                    }
-                }
-                catch (error) {
-                    console.debug('Failed to format tool arguments:', error);
+    const finalizeToolResults = async function* (mode) {
+        const waitForPendingExecutions = async (executions, timeoutMs) => {
+            if (executions.length === 0) {
+                return;
+            }
+            const completionPromise = Promise.all(executions.map(execution => execution.promise.then(() => undefined)));
+            if (timeoutMs === undefined) {
+                await completionPromise;
+                return;
+            }
+            await Promise.race([
+                completionPromise,
+                new Promise(resolve => setTimeout(resolve, timeoutMs)),
+            ]);
+        };
+        const waitForAllExecutions = async (executions, abortSignal) => {
+            if (executions.length === 0) {
+                return true;
+            }
+            const completionPromise = Promise.all(executions.map(execution => execution.promise.then(() => undefined))).then(() => true);
+            if (!abortSignal) {
+                return completionPromise;
+            }
+            if (abortSignal.aborted) {
+                return false;
+            }
+            return new Promise(resolve => {
+                const abortListener = () => {
+                    abortSignal.removeEventListener('abort', abortListener);
+                    resolve(false);
+                };
+                completionPromise.then(completed => {
+                    abortSignal.removeEventListener('abort', abortListener);
+                    resolve(completed);
+                });
+                abortSignal.addEventListener('abort', abortListener, { once: true });
+            });
+        };
+        let finalizationMode = mode;
+        if (finalizationMode === 'wait_all') {
+            const completedAllExecutions = await waitForAllExecutions(toolExecutions.filter(execution => !execution.settled), requestGuard?.signal);
+            if (!completedAllExecutions) {
+                finalizationMode = 'bounded_failure';
+            }
+        }
+        if (finalizationMode === 'bounded_failure') {
+            toolExecutionGuard?.abort(roundSummary.failure?.error
+                ? new Error(roundSummary.failure.error)
+                : new Error('Request finalized after terminal provider failure.'));
+            await waitForPendingExecutions(toolExecutions.filter(execution => !execution.settled), TOOL_FAILURE_FINALIZATION_TIMEOUT_MS);
+            for (const execution of toolExecutions) {
+                if (!execution.settled) {
+                    runningToolTracker.abortRunningTool(execution.toolCall.id || execution.toolCall.call_id || '');
                 }
-                if (argumentsFormatted) {
-                    toolCallFormattedArgs.set(toolCall.id, argumentsFormatted);
+            }
+            await waitForPendingExecutions(toolExecutions.filter(execution => !execution.settled), TOOL_FAILURE_FINALIZATION_TIMEOUT_MS);
+            for (const execution of toolExecutions) {
+                const runningToolId = execution.toolCall.id || execution.toolCall.call_id || '';
+                if (execution.settled) {
+                    const leakedRunningTool = runningToolId
+                        ? runningToolTracker.getRunningTool(runningToolId)
+                        : undefined;
+                    if (leakedRunningTool) {
+                        const failureResult = execution.result ?? createToolFinalizationFailureResult(execution.toolCall);
+                        await runningToolTracker.failRunningTool(runningToolId, failureResult.error || 'Tool execution failed during bounded finalization.');
+                    }
                 }
-                const modifiedEvent = {
-                    ...event,
-                    tool_call: {
-                        ...toolCall,
-                        function: {
-                            ...toolCall.function,
-                            arguments_formatted: argumentsFormatted,
-                        },
+            }
+        }
+        const toolResults = finalizationMode === 'wait_all'
+            ? await Promise.all(toolExecutions.map(execution => execution.promise))
+            : toolExecutions.flatMap(execution => (execution.settled && execution.result ? [execution.result] : []));
+        for (const toolResult of toolResults) {
+            const toolName = toolResult.toolCall.function.name;
+            const isTerminalTool = terminalToolNames.has(toolName);
+            const formattedArgs = toolCallFormattedArgs.get(toolResult.toolCall.id);
+            const toolCallWithFormattedArgs = formattedArgs
+                ? {
+                    ...toolResult.toolCall,
+                    function: {
+                        ...toolResult.toolCall.function,
+                        arguments_formatted: formattedArgs,
                     },
+                }
+                : toolResult.toolCall;
+            const toolDoneEvent = {
+                type: 'tool_done',
+                request_id: roundRequestId,
+                tool_call: toolCallWithFormattedArgs,
+                result: {
+                    call_id: toolResult.call_id || toolResult.id,
+                    output: toolResult.output,
+                    error: toolResult.error,
+                },
+            };
+            if (isTerminalTool && !toolResult.error) {
+                roundSummary.terminalToolSucceeded = true;
+            }
+            yield toolDoneEvent;
+            await emitEvent(toolDoneEvent, roundAgent, model);
+            await trace.emitToolDone(roundRequestId, toolResult.toolCall.id, {
+                tool_name: toolName,
+                call_id: toolResult.call_id,
+                output: toolResult.output,
+                error: toolResult.error,
+            });
+            if (!isTerminalTool) {
+                const functionOutput = convertToFunctionCallOutput(toolResult, model, 'completed');
+                history.add(functionOutput);
+                yield {
+                    type: 'response_output',
+                    message: functionOutput,
+                    request_id: roundRequestId,
                 };
-                event = modifiedEvent;
             }
         }
-        yield event;
-        await emitEvent(event, agent, model);
-        switch (event.type) {
-            case 'cost_update': {
-                const costEvent = event;
-                if (costEvent.usage?.cost) {
-                    totalCost += costEvent.usage.cost;
+        for (const bufferedEvent of toolEventBuffer) {
+            yield { ...bufferedEvent, request_id: roundRequestId };
+        }
+    };
+    try {
+        for await (let event of stream) {
+            event = { ...event, request_id: roundRequestId };
+            if (event.type === 'error') {
+                const failure = normalizeFailure(event, {
+                    error: event.error,
+                    recoverable: event.recoverable,
+                    code: event.code,
+                    details: event.details,
+                    ...getFailureRetryOverrides(agent),
+                });
+                roundSummary.failure = selectMoreSevereFailure(roundSummary.failure, failure);
+                roundSummary.errors.push(failure.error);
+                continue;
+            }
+            if (event.type === 'message_complete' && structuredOutputSchema) {
+                const messageEvent = event;
+                if (hasTerminalTextContent(messageEvent.content, true)) {
+                    const validationResult = validateJsonResponseContent(messageEvent.content, structuredOutputSchema);
+                    if (!validationResult.ok && 'error' in validationResult) {
+                        const failure = normalizeFailure(new Error(validationResult.error), {
+                            recoverable: false,
+                            reason: 'structured_output_validation_failed',
+                        });
+                        roundSummary.failure = selectMoreSevereFailure(roundSummary.failure, failure);
+                        roundSummary.errors.push(failure.error);
+                        continue;
+                    }
                 }
-                break;
             }
-            case 'message_complete': {
+            if (event.type === 'tool_start') {
+                const toolEvent = event;
+                if (toolEvent.tool_call) {
+                    const toolCall = toolEvent.tool_call;
+                    let argumentsFormatted;
+                    try {
+                        const tool = roundAgent.tools?.find(t => t.definition.function.name === toolCall.function.name);
+                        if (tool?.definition.function.parameters.properties) {
+                            const parsedArgs = JSON.parse(toolCall.function.arguments || '{}');
+                            if (typeof parsedArgs === 'object' && parsedArgs !== null && !Array.isArray(parsedArgs)) {
+                                const paramNames = Object.keys(tool.definition.function.parameters.properties);
+                                const orderedArgs = {};
+                                for (const param of paramNames) {
+                                    if (param in parsedArgs) {
+                                        orderedArgs[param] = parsedArgs[param];
+                                    }
+                                }
+                                argumentsFormatted = JSON.stringify(orderedArgs, null, 2);
+                            }
+                        }
+                    }
+                    catch (error) {
+                        console.debug('Failed to format tool arguments:', error);
+                    }
+                    if (argumentsFormatted) {
+                        toolCallFormattedArgs.set(toolCall.id, argumentsFormatted);
+                    }
+                    event = {
+                        ...event,
+                        tool_call: {
+                            ...toolCall,
+                            function: {
+                                ...toolCall.function,
+                                arguments_formatted: argumentsFormatted,
+                            },
+                        },
+                    };
+                }
+            }
+            if (event.type === 'message_complete') {
                 const messageEvent = event;
-                if (sawToolCallThisRound) {
+                if (hasTerminalTextContent(messageEvent.content, expectsStructuredOutput)) {
+                    sawTerminalProviderOutcome = true;
+                }
+            }
+            else if (event.type === 'tool_start' || event.type === 'file_complete') {
+                sawTerminalProviderOutcome = true;
+            }
+            yield event;
+            await emitEvent(event, roundAgent, model);
+            switch (event.type) {
+                case 'cost_update': {
+                    const costEvent = event;
+                    if (costEvent.usage?.cost) {
+                        totalCost += costEvent.usage.cost;
+                    }
                     break;
                 }
-                if (messageEvent.thinking_content ||
-                    (!messageEvent.content && messageEvent.message_id)) {
-                    const thinkingMessage = convertToThinkingMessage(messageEvent, model);
-                    if (agent.onThinking) {
-                        await agent.onThinking(thinkingMessage);
+                case 'message_complete': {
+                    const messageEvent = event;
+                    if (sawToolCallThisRound) {
+                        break;
                     }
-                    history.add(thinkingMessage);
-                    yield {
-                        type: 'response_output',
-                        message: thinkingMessage,
-                        request_id: requestId,
-                    };
+                    if (messageEvent.thinking_content ||
+                        (!messageEvent.content && messageEvent.message_id)) {
+                        const thinkingMessage = convertToThinkingMessage(messageEvent, model);
+                        if (roundAgent.onThinking) {
+                            await roundAgent.onThinking(thinkingMessage);
+                        }
+                        history.add(thinkingMessage);
+                        yield {
+                            type: 'response_output',
+                            message: thinkingMessage,
+                            request_id: roundRequestId,
+                        };
+                    }
+                    if (hasTerminalTextContent(messageEvent.content, expectsStructuredOutput)) {
+                        roundSummary.emittedTerminalOutput = true;
+                        roundSummary.messages.push(messageEvent.content);
+                        const contentMessage = convertToOutputMessage(messageEvent, model, 'completed');
+                        if (roundAgent.onResponse) {
+                            await roundAgent.onResponse(contentMessage);
+                        }
+                        history.add(contentMessage);
+                        yield {
+                            type: 'response_output',
+                            message: contentMessage,
+                            request_id: roundRequestId,
+                        };
+                    }
+                    break;
+                }
+                case 'file_complete': {
+                    roundSummary.emittedTerminalOutput = true;
+                    break;
                 }
-                if (messageEvent.content) {
-                    const contentMessage = convertToOutputMessage(messageEvent, model, 'completed');
-                    if (agent.onResponse) {
-                        await agent.onResponse(contentMessage);
+                case 'tool_start': {
+                    const toolEvent = event;
+                    if (!toolEvent.tool_call) {
+                        break;
+                    }
+                    if (!sawToolCallThisRound) {
+                        roundSummary.emittedTerminalOutput = false;
+                        roundSummary.messages = [];
                     }
-                    history.add(contentMessage);
+                    sawToolCallThisRound = true;
+                    const remainingCalls = maxToolCalls - currentToolCalls - roundSummary.toolCallsStarted;
+                    if (remainingCalls <= 0) {
+                        console.warn(`Tool call limit reached (${maxToolCalls}). Skipping tool calls.`);
+                        const failure = normalizeFailure(new Error(`Tool call limit reached (${maxToolCalls}). Cannot execute tool ${toolEvent.tool_call.function.name}.`), {
+                            recoverable: false,
+                            reason: 'max_tool_calls_reached',
+                            ...getFailureRetryOverrides(agent),
+                        });
+                        roundSummary.failure = selectMoreSevereFailure(roundSummary.failure, failure);
+                        if (!roundSummary.errors.includes(failure.error)) {
+                            roundSummary.errors.push(failure.error);
+                        }
+                        break;
+                    }
+                    const toolCall = toolEvent.tool_call;
+                    const functionCall = convertToFunctionCall(toolCall, model, 'completed');
+                    history.add(functionCall);
                     yield {
                         type: 'response_output',
-                        message: contentMessage,
-                        request_id: requestId,
+                        message: functionCall,
+                        request_id: roundRequestId,
                     };
-                }
-                break;
-            }
-            case 'tool_start': {
-                const toolEvent = event;
-                if (!toolEvent.tool_call) {
-                    break;
-                }
-                sawToolCallThisRound = true;
-                const remainingCalls = maxToolCalls - currentToolCalls;
-                if (remainingCalls <= 0) {
-                    console.warn(`Tool call limit reached (${maxToolCalls}). Skipping tool calls.`);
+                    ++roundSummary.toolCallsStarted;
+                    if (!terminalToolNames.has(toolCall.function.name)) {
+                        roundSummary.hasFollowupToolCalls = true;
+                    }
+                    await trace.emitToolStart(roundRequestId, toolCall.id, {
+                        tool_name: toolCall.function.name,
+                        arguments: toolCall.function.arguments,
+                        arguments_formatted: toolCall.function.arguments_formatted,
+                    });
+                    const trackedExecution = {
+                        toolCall,
+                        promise: processToolCall(toolCall, {
+                            ...roundAgent,
+                            abortSignal: toolExecutionGuard?.signal ?? roundAgent.abortSignal,
+                        }),
+                        settled: false,
+                    };
+                    trackedExecution.promise = trackedExecution.promise.then(result => {
+                        if (!trackedExecution.settled) {
+                            trackedExecution.settled = true;
+                            trackedExecution.result = result;
+                        }
+                        return trackedExecution.result ?? result;
+                    });
+                    toolExecutions.push(trackedExecution);
                     break;
                 }
-                const toolCall = toolEvent.tool_call;
-                const functionCall = convertToFunctionCall(toolCall, model, 'completed');
-                toolPromises.push(processToolCall(toolCall, agent));
-                history.add(functionCall);
-                yield {
-                    type: 'response_output',
-                    message: functionCall,
-                    request_id: requestId,
-                };
-                break;
-            }
-            case 'error': {
-                console.error('[executeRound] Error event:', truncateLargeValues(event.error));
-                break;
             }
         }
     }
-    const request_duration = Date.now() - startTime;
-    const toolResults = await Promise.all(toolPromises);
-    const terminalToolNames = getTerminalToolNames(agent);
-    for (const toolResult of toolResults) {
-        const toolName = toolResult.toolCall.function.name;
-        const isTerminalTool = terminalToolNames.has(toolName);
-        const formattedArgs = toolCallFormattedArgs.get(toolResult.toolCall.id);
-        const toolCallWithFormattedArgs = formattedArgs
-            ? {
-                ...toolResult.toolCall,
-                function: {
-                    ...toolResult.toolCall.function,
-                    arguments_formatted: formattedArgs,
-                },
-            }
-            : toolResult.toolCall;
-        const toolDoneEvent = {
-            type: 'tool_done',
-            request_id: requestId,
-            tool_call: toolCallWithFormattedArgs,
-            result: {
-                call_id: toolResult.call_id || toolResult.id,
-                output: toolResult.output,
-                error: toolResult.error,
-            },
-        };
-        yield toolDoneEvent;
-        await emitEvent(toolDoneEvent, agent, model);
-        if (!isTerminalTool) {
-            const functionOutput = convertToFunctionCallOutput(toolResult, model, 'completed');
-            history.add(functionOutput);
-            yield {
-                type: 'response_output',
-                message: functionOutput,
-                request_id: requestId,
-            };
+    catch (error) {
+        const streamFailure = normalizeFailure(error, {
+            reason: 'request_stream_failed',
+            ...getFailureRetryOverrides(agent),
+        });
+        roundSummary.failure = selectMoreSevereFailure(roundSummary.failure, streamFailure);
+        roundSummary.errors.push(streamFailure.error);
+    }
+    if (!sawTerminalProviderOutcome && !roundSummary.failure) {
+        const emptyResponseFailure = normalizeFailure(new Error(`Provider ${provider.provider_id} ended the stream without any terminal content, tool calls, files, or errors.`), {
+            recoverable: false,
+            reason: 'empty_provider_response',
+            ...getFailureRetryOverrides(agent),
+        });
+        roundSummary.failure = emptyResponseFailure;
+        roundSummary.errors.push(emptyResponseFailure.error);
+    }
+    roundSummary.requestDuration = Date.now() - startTime;
+    const shouldUseBoundedFailureFinalization = Boolean(roundSummary.failure?.terminal);
+    yield* finalizeToolResults(shouldUseBoundedFailureFinalization ? 'bounded_failure' : 'wait_all');
+    if (requestGuard?.signal.aborted) {
+        const abortFailure = normalizeFailure(requestGuard.signal.reason, {
+            reason: 'request_stream_failed',
+            ...getFailureRetryOverrides(agent),
+        });
+        roundSummary.failure = selectMoreSevereFailure(roundSummary.failure, abortFailure);
+        if (!roundSummary.errors.includes(abortFailure.error)) {
+            roundSummary.errors.push(abortFailure.error);
         }
     }
-    const duration_with_tools = Date.now() - startTime;
-    const agentDoneEvent = {
+    roundSummary.durationWithTools = Date.now() - startTime;
+    roundSummary.requestCost = totalCost > 0 ? totalCost : undefined;
+    roundSummary.agentDoneEvent = {
         type: 'agent_done',
-        request_id: requestId,
-        request_cost: totalCost > 0 ? totalCost : undefined,
-        request_duration,
-        duration_with_tools,
+        request_id: roundRequestId,
+        request_cost: roundSummary.requestCost,
+        request_duration: roundSummary.requestDuration,
+        duration_with_tools: roundSummary.durationWithTools,
         timestamp: new Date().toISOString(),
     };
-    yield agentDoneEvent;
-    await emitEvent(agentDoneEvent, agent, model);
-    for (const bufferedEvent of toolEventBuffer) {
-        yield { ...bufferedEvent, request_id: requestId };
-    }
+    roundSummary.agentDoneAgent = roundAgent;
+    requestGuard?.cleanup();
+    toolExecutionGuard?.cleanup();
+    return roundSummary;
 }
 async function* performVerification(agent, output, messages, attempt = 0) {
-    if (!agent.verifier)
-        return;
+    if (!agent.verifier) {
+        return { passed: true };
+    }
     const maxAttempts = agent.maxVerificationAttempts || 2;
     const verification = await verifyOutput(agent.verifier, output, messages);
     if (verification.status === 'pass') {
@@ -474,7 +854,7 @@ async function* performVerification(agent, output, messages, attempt = 0) {
             type: 'message_delta',
             content: '\n\n✓ Output verified',
         };
-        return;
+        return { passed: true };
     }
     if (attempt < maxAttempts - 1) {
         yield {
@@ -503,27 +883,37 @@ async function* performVerification(agent, output, messages, attempt = 0) {
         const retryStream = ensembleRequest(retryMessages, retryAgent);
         let retryOutput = '';
         for await (const event of retryStream) {
+            if (event.type === 'operation_status' || event.type === 'error') {
+                continue;
+            }
             yield event;
             if (event.type === 'message_complete' && 'content' in event) {
                 retryOutput = event.content;
             }
         }
         if (retryOutput) {
-            yield* performVerification(agent, retryOutput, messages, attempt + 1);
+            return yield* performVerification(agent, retryOutput, messages, attempt + 1);
         }
-    }
-    else {
-        yield {
-            type: 'message_delta',
-            content: `\n\n❌ Verification failed after ${maxAttempts} attempts: ${verification.reason}`,
+        return {
+            passed: false,
+            error: 'Verification retry did not produce a final response.',
         };
     }
+    const failureMessage = `Verification failed after ${maxAttempts} attempts: ${verification.reason}`;
+    yield {
+        type: 'message_delta',
+        content: `\n\n❌ ${failureMessage}`,
+    };
+    return {
+        passed: false,
+        error: failureMessage,
+    };
 }
 async function processToolCall(toolCall, agent) {
-    if (agent.onToolCall) {
-        await agent.onToolCall(toolCall);
-    }
     try {
+        if (agent.onToolCall) {
+            await agent.onToolCall(toolCall);
+        }
         if (!agent.tools) {
             throw new Error('No tools available for agent');
         }
@@ -531,7 +921,7 @@ async function processToolCall(toolCall, agent) {
         if (!tool || !('function' in tool)) {
             throw new Error(`Tool ${toolCall.function.name} not found`);
         }
-        const rawResult = await handleToolCall(toolCall, tool, agent);
+        const rawResult = await handleToolCall(toolCall, tool, agent, agent.abortSignal);
         const processedResult = await processToolResult(toolCall, rawResult, agent, tool.allowSummary);
         const toolCallResult = {
             toolCall,
@@ -545,21 +935,32 @@ async function processToolCall(toolCall, agent) {
         return toolCallResult;
     }
     catch (error) {
-        const errorOutput = error instanceof Error
-            ? `Tool execution failed: ${error.message}`
-            : `Tool execution failed: ${String(error)}`;
-        const toolCallResult = {
-            toolCall,
-            id: toolCall.id,
-            call_id: toolCall.call_id || toolCall.id,
-            error: errorOutput,
-        };
+        const toolCallResult = createToolFailureResult(toolCall, error);
         if (agent.onToolError) {
-            await agent.onToolError(toolCallResult);
+            try {
+                await agent.onToolError(toolCallResult);
+            }
+            catch (hookError) {
+                console.error('[processToolCall] onToolError hook failed:', hookError);
+            }
         }
         return toolCallResult;
     }
 }
+function createToolFailureResult(toolCall, error) {
+    const errorOutput = error instanceof Error
+        ? `Tool execution failed: ${error.message}`
+        : `Tool execution failed: ${String(error)}`;
+    return {
+        toolCall,
+        id: toolCall.id,
+        call_id: toolCall.call_id || toolCall.id,
+        error: errorOutput,
+    };
+}
+function createToolFinalizationFailureResult(toolCall) {
+    return createToolFailureResult(toolCall, 'Tool did not finish before request finalization after a terminal provider failure.');
+}
 export function mergeHistoryThread(mainHistory, thread, startIndex) {
     const newMessages = thread.slice(startIndex);
     mainHistory.push(...newMessages);