@qduc/term2 0.15.0 → 0.17.0

package/dist/services/agent-runtime/application-run-loop.js

@@ -9,7 +9,8 @@ import { RunBudget, isRunBudgetInteraction, } from './run-budget.js';
 import { addTokenUsage, normalizeUsage } from '../../utils/ai/token-usage.js';
 import { computeModelCost } from '../../services/cost/model-cost.js';
 import { getCatalogPricingVersion, getModelPricing } from '../../services/cost/pricing.js';
-import { GenerationDeadline, GenerationGuard, GenerationGuardError, } from './generation-guard.js';
+import { GenerationGuard, GenerationGuardError, GenerationStreamDeadlines, } from './generation-guard.js';
+import { classifyInLoopModelRetry, sleepWithAbort } from '../retry/in-loop-model-retry.js';
 /**
  * Compatibility error for legacy callers without a staged run budget. The loop
  * owns turn counting — the SDK-era `callModelInputFilter` hook is deliberately
@@ -657,150 +658,220 @@ export class ApplicationRunLoop {
                 state.requestBoundaryReady = undefined;
             }
             const model = await this.#deps.resolveModel(state.agent.model);
-            let sawToolCall = false;
-            const streamedToolCalls = [];
+            const maxRetries = state.agent.modelSettings?.retry?.maxRetries ?? 0;
+            let attempt = 0;
+            let streamedToolCalls = [];
             let completion;
+            let sawToolCall = false;
+            let activeRequestId = '';
+            let activeRequest;
             let pendingNativeReasoning;
-            // Stable process-unique request id, allocated immediately before dispatch
-            // so a cost record can be settled exactly once (success or failure).
-            const requestId = this.#nextRequestId();
-            const criticalWrapUp = state.criticalWrapUpPending === true && state.criticalWrapUpDispatched !== true;
-            if (criticalWrapUp)
-                state.criticalWrapUpDispatched = true;
-            const disableChaining = state.disableChainingForAttempt === true;
-            if (disableChaining) {
-                state.disableChainingForAttempt = false;
-                state.responseId = undefined;
-                state.responseProviderId = undefined;
-            }
-            const request = {
-                instructions: criticalWrapUp
-                    ? `${state.agent.instructions}\n\nBudget containment is terminal. Do not call tools. In this one final response, summarize what you completed, the evidence you have, and what remains.`
-                    : state.agent.instructions,
-                ...(state.supportsConversationChaining &&
-                    !disableChaining &&
-                    state.responseId &&
-                    state.responseProviderId !== undefined &&
-                    state.responseProviderId === state.currentProviderId
-                    ? { previousResponseId: state.responseId }
-                    : {}),
-                ...(disableChaining ? { disableChaining: true } : {}),
-                input: state.input,
-                tools: toModelTools(criticalWrapUp ? [] : state.agent.tools),
-                applicationTools: criticalWrapUp ? [] : state.agent.tools,
-                ...(state.agent.modelSettings?.temperature !== undefined
-                    ? { temperature: state.agent.modelSettings.temperature }
-                    : {}),
-                ...(state.agent.modelSettings?.reasoning ? { reasoning: state.agent.modelSettings.reasoning } : {}),
-                ...(state.agent.modelSettings?.maxTokens !== undefined
-                    ? { maxTokens: state.agent.modelSettings.maxTokens }
-                    : {}),
-                ...(state.agent.outputType !== undefined ? { outputType: state.agent.outputType } : {}),
-                ...(state.agent.modelSettings?.codex ? { codex: state.agent.modelSettings.codex } : {}),
-                ...(state.agent.modelSettings?.providerData
-                    ? {
-                        providerOptions: this.#contextCompactionSessionState.disabled
-                            ? Object.fromEntries(Object.entries(state.agent.modelSettings.providerData).filter(([key]) => key !== 'contextCompaction'))
-                            : state.agent.modelSettings.providerData,
-                    }
-                    : {}),
-                ...(options.signal ? { signal: options.signal } : {}),
-            };
-            const generationGuard = new GenerationGuard({
-                maxOutputCharacters: state.agent.modelSettings?.maxStreamOutputChars,
-                maxTextCharacters: state.agent.modelSettings?.maxStreamOutputChars,
-                maxReasoningCharacters: state.agent.modelSettings?.maxStreamOutputChars,
-                maxToolArgumentCharacters: state.agent.modelSettings?.maxStreamOutputChars,
-                maxCumulativeToolArgumentCharacters: state.agent.modelSettings?.maxStreamOutputChars,
-                requestDeadlineMs: state.agent.modelSettings?.maxModelRequestDurationMs,
-                ...options.generationGuard,
-            });
-            const consume = async () => {
-                const deadline = new GenerationDeadline(generationGuard.requestDeadlineMs, () => requestAbortController.abort(), () => generationGuard.progress);
-                let iterator;
-                let iteratorFinished = false;
-                try {
-                    iterator = model.stream(request)[Symbol.asyncIterator]();
-                    while (true) {
-                        const next = await deadline.wait(iterator.next());
-                        if (next.done) {
-                            iteratorFinished = true;
-                            return;
-                        }
-                        const event = next.value;
-                        if (event.type === 'completion') {
-                            generationGuard.observeCompletion(event.output);
-                            completion = event;
-                            continue;
-                        }
-                        if (event.type === 'text_delta') {
-                            generationGuard.observeText(event.text);
-                            outputPush(stream, queue, { type: 'text_delta', text: event.text });
-                            continue;
-                        }
-                        if (event.type === 'codex_rate_limits') {
-                            outputPush(stream, queue, { type: 'codex_rate_limits', rateLimits: event.rateLimits });
-                            continue;
+            let criticalWrapUp = false;
+            while (true) {
+                streamedToolCalls = [];
+                completion = undefined;
+                sawToolCall = false;
+                pendingNativeReasoning = undefined;
+                // Stable process-unique request id, allocated immediately before dispatch
+                // so a cost record can be settled exactly once (success or failure).
+                activeRequestId = this.#nextRequestId();
+                const requestId = activeRequestId;
+                const outputLengthBefore = stream.output.length;
+                const newItemsLengthBefore = stream.newItems.length;
+                const inputLengthBefore = state.input.length;
+                const historyLengthBefore = state.history.length;
+                const criticalWrapUpDispatchedBefore = state.criticalWrapUpDispatched;
+                let provisionalTextCharacters = 0;
+                let provisionalReasoningCharacters = 0;
+                let provisionalTextDeltas = 0;
+                let provisionalReasoningDeltas = 0;
+                criticalWrapUp = state.criticalWrapUpPending === true && state.criticalWrapUpDispatched !== true;
+                if (criticalWrapUp)
+                    state.criticalWrapUpDispatched = true;
+                const disableChaining = state.disableChainingForAttempt === true;
+                if (disableChaining) {
+                    state.disableChainingForAttempt = false;
+                    state.responseId = undefined;
+                    state.responseProviderId = undefined;
+                }
+                const request = {
+                    instructions: criticalWrapUp
+                        ? `${state.agent.instructions}\n\nBudget containment is terminal. Do not call tools. In this one final response, summarize what you completed, the evidence you have, and what remains.`
+                        : state.agent.instructions,
+                    ...(state.supportsConversationChaining &&
+                        !disableChaining &&
+                        state.responseId &&
+                        state.responseProviderId !== undefined &&
+                        state.responseProviderId === state.currentProviderId
+                        ? { previousResponseId: state.responseId }
+                        : {}),
+                    ...(disableChaining ? { disableChaining: true } : {}),
+                    input: state.input,
+                    tools: toModelTools(criticalWrapUp ? [] : state.agent.tools),
+                    applicationTools: criticalWrapUp ? [] : state.agent.tools,
+                    ...(state.agent.modelSettings?.temperature !== undefined
+                        ? { temperature: state.agent.modelSettings.temperature }
+                        : {}),
+                    ...(state.agent.modelSettings?.reasoning ? { reasoning: state.agent.modelSettings.reasoning } : {}),
+                    ...(state.agent.modelSettings?.maxTokens !== undefined
+                        ? { maxTokens: state.agent.modelSettings.maxTokens }
+                        : {}),
+                    ...(state.agent.outputType !== undefined ? { outputType: state.agent.outputType } : {}),
+                    ...(state.agent.modelSettings?.codex ? { codex: state.agent.modelSettings.codex } : {}),
+                    ...(state.agent.modelSettings?.providerData
+                        ? {
+                            providerOptions: this.#contextCompactionSessionState.disabled
+                                ? Object.fromEntries(Object.entries(state.agent.modelSettings.providerData).filter(([key]) => key !== 'contextCompaction'))
+                                : state.agent.modelSettings.providerData,
                         }
-                        if (event.type === 'reasoning_delta') {
-                            generationGuard.observeReasoning(event.text);
-                            pendingNativeReasoning = appendNativeReasoning(pendingNativeReasoning, event);
-                            outputPush(stream, queue, { type: 'reasoning_delta', text: event.text });
-                            continue;
+                        : {}),
+                    ...(options.signal ? { signal: options.signal } : {}),
+                };
+                activeRequest = request;
+                const generationGuard = new GenerationGuard({
+                    maxOutputCharacters: state.agent.modelSettings?.maxStreamOutputChars,
+                    maxTextCharacters: state.agent.modelSettings?.maxStreamOutputChars,
+                    maxReasoningCharacters: state.agent.modelSettings?.maxStreamOutputChars,
+                    maxToolArgumentCharacters: state.agent.modelSettings?.maxStreamOutputChars,
+                    maxCumulativeToolArgumentCharacters: state.agent.modelSettings?.maxStreamOutputChars,
+                    requestDeadlineMs: state.agent.modelSettings?.maxModelRequestDurationMs,
+                    maxStreamIdleMs: state.agent.modelSettings?.maxModelStreamIdleMs,
+                    ...options.generationGuard,
+                });
+                const consume = async () => {
+                    const deadline = new GenerationStreamDeadlines({ totalMs: generationGuard.requestDeadlineMs, idleMs: generationGuard.maxStreamIdleMs }, () => requestAbortController.abort(), () => generationGuard.progress);
+                    let iterator;
+                    let iteratorFinished = false;
+                    try {
+                        iterator = model.stream(request)[Symbol.asyncIterator]();
+                        while (true) {
+                            const next = await deadline.wait(iterator.next());
+                            if (next.done) {
+                                iteratorFinished = true;
+                                return;
+                            }
+                            // Any streamed event is transport activity: re-arm the inactivity
+                            // window so long legitimate reasoning is never mistaken for a stall.
+                            deadline.recordActivity();
+                            const event = next.value;
+                            if (event.type === 'completion') {
+                                generationGuard.observeCompletion(event.output);
+                                completion = event;
+                                continue;
+                            }
+                            if (event.type === 'text_delta') {
+                                generationGuard.observeText(event.text);
+                                provisionalTextCharacters += event.text.length;
+                                provisionalTextDeltas++;
+                                outputPush(stream, queue, { type: 'text_delta', text: event.text });
+                                continue;
+                            }
+                            if (event.type === 'codex_rate_limits') {
+                                outputPush(stream, queue, { type: 'codex_rate_limits', rateLimits: event.rateLimits });
+                                continue;
+                            }
+                            if (event.type === 'reasoning_delta') {
+                                const accepted = generationGuard.observeReasoning(event.text);
+                                if (!accepted)
+                                    continue;
+                                const forwarded = accepted === event.text ? event : { ...event, text: accepted };
+                                pendingNativeReasoning = appendNativeReasoning(pendingNativeReasoning, forwarded);
+                                provisionalReasoningCharacters += accepted.length;
+                                provisionalReasoningDeltas++;
+                                outputPush(stream, queue, { type: 'reasoning_delta', text: accepted });
+                                continue;
+                            }
+                            if (event.type === 'tool_call_streaming_delta') {
+                                generationGuard.observeToolArgumentProgress(event.argumentCharCount);
+                                outputPush(stream, queue, event);
+                                continue;
+                            }
+                            if (event.type === 'context_compaction_started' || event.type === 'context_compaction_completed') {
+                                outputPush(stream, queue, event);
+                                continue;
+                            }
+                            if (event.type === 'tool_call') {
+                                generationGuard.observeToolCall(event.arguments);
+                                pendingNativeReasoning = commitPendingNativeReasoning(state, stream, queue, pendingNativeReasoning);
+                                sawToolCall = true;
+                                streamedToolCalls.push(event);
+                            }
                         }
-                        if (event.type === 'tool_call_streaming_delta') {
-                            generationGuard.observeToolArgumentProgress(event.argumentCharCount);
-                            outputPush(stream, queue, event);
-                            continue;
+                    }
+                    finally {
+                        deadline.dispose();
+                        if (iterator && !iteratorFinished)
+                            void iterator.return?.().catch(() => undefined);
+                    }
+                };
+                const dispatch = async () => {
+                    state.requestPreparation?.prepare(request);
+                    await consume();
+                };
+                try {
+                    if (state.requestPreparation)
+                        await state.requestPreparation.run(dispatch);
+                    else
+                        await dispatch();
+                    break;
+                }
+                catch (error) {
+                    if (error instanceof GenerationGuardError)
+                        requestAbortController.abort();
+                    const decision = classifyInLoopModelRetry(error, attempt, maxRetries, Math.random, {
+                        previousResponseId: activeRequest?.previousResponseId,
+                    });
+                    if (decision.retryable && !options.signal?.aborted) {
+                        attempt++;
+                        state.input.splice(inputLengthBefore);
+                        state.history.splice(historyLengthBefore);
+                        state.criticalWrapUpDispatched = criticalWrapUpDispatchedBefore;
+                        stream.output.splice(outputLengthBefore);
+                        if (stream.newItems !== stream.output) {
+                            stream.newItems.splice(newItemsLengthBefore);
                         }
-                        if (event.type === 'context_compaction_started' || event.type === 'context_compaction_completed') {
-                            outputPush(stream, queue, event);
-                            continue;
+                        if (provisionalTextDeltas > 0 || provisionalReasoningDeltas > 0) {
+                            queue.push({
+                                type: 'model_attempt_rollback',
+                                textCharacters: provisionalTextCharacters,
+                                reasoningCharacters: provisionalReasoningCharacters,
+                                textDeltas: provisionalTextDeltas,
+                                reasoningDeltas: provisionalReasoningDeltas,
+                            });
                         }
-                        if (event.type === 'tool_call') {
-                            generationGuard.observeToolCall(event.arguments);
-                            pendingNativeReasoning = commitPendingNativeReasoning(state, stream, queue, pendingNativeReasoning);
-                            sawToolCall = true;
-                            streamedToolCalls.push(event);
+                        if (decision.kind === 'chain_recovery') {
+                            state.disableChainingForAttempt = true;
+                            state.responseId = undefined;
+                            state.responseProviderId = undefined;
                         }
+                        this.#deps.logDiagnostic?.('Retrying model request in run loop', {
+                            attempt,
+                            maxRetries,
+                            kind: decision.kind,
+                            delayMs: decision.delayMs,
+                            error: error instanceof Error ? error.message : String(error),
+                        });
+                        await (this.#deps.waitBeforeModelRetry ?? sleepWithAbort)(decision.delayMs, options.signal);
+                        continue;
                     }
+                    // Dispatch began, but no terminal completion was accepted. Record an
+                    // unpriced marker so the summary stays honest (partial) rather than
+                    // appearing exact. Observational only: the error still propagates.
+                    const cancelled = error instanceof Error && error.name === 'AbortError';
+                    const record = this.#appendCostRecord(state, {
+                        requestId,
+                        provider: state.currentProviderId,
+                        model: state.agent.model,
+                        tier: resolveServiceTier(request),
+                        outcome: cancelled ? 'cancelled' : 'failed',
+                    });
+                    // Only the live queue sees cost records: `output`/`newItems` feed
+                    // persistence and replay, and a cost event must never become a
+                    // restored history item.
+                    queue.push({ type: 'cost_update', record });
+                    this.#evaluateRunBudget(state, stream, queue);
+                    throw error;
                 }
-                finally {
-                    deadline.dispose();
-                    if (iterator && !iteratorFinished)
-                        void iterator.return?.().catch(() => undefined);
-                }
-            };
-            const dispatch = async () => {
-                state.requestPreparation?.prepare(request);
-                await consume();
-            };
-            try {
-                if (state.requestPreparation)
-                    await state.requestPreparation.run(dispatch);
-                else
-                    await dispatch();
-            }
-            catch (error) {
-                if (error instanceof GenerationGuardError)
-                    requestAbortController.abort();
-                // Dispatch began, but no terminal completion was accepted. Record an
-                // unpriced marker so the summary stays honest (partial) rather than
-                // appearing exact. Observational only: the error still propagates.
-                const cancelled = error instanceof Error && error.name === 'AbortError';
-                const record = this.#appendCostRecord(state, {
-                    requestId,
-                    provider: state.currentProviderId,
-                    model: state.agent.model,
-                    tier: resolveServiceTier(request),
-                    outcome: cancelled ? 'cancelled' : 'failed',
-                });
-                // Only the live queue sees cost records: `output`/`newItems` feed
-                // persistence and replay, and a cost event must never become a
-                // restored history item.
-                queue.push({ type: 'cost_update', record });
-                this.#evaluateRunBudget(state, stream, queue);
-                throw error;
             }
             if (!completion) {
                 // A few application-owned model fixtures (and legacy adapters) end a
@@ -820,6 +891,7 @@ export class ApplicationRunLoop {
                 }
                 continue;
             }
+            const finalCompletion = completion;
             // Commit the authoritative terminal state before handling terminal-only
             // tool calls. Approval may pause immediately after this point, and the
             // continuation must still carry the response that produced the calls.
@@ -828,7 +900,7 @@ export class ApplicationRunLoop {
             // chaining, while still exposing the completed provider response on the
             // current stream for diagnostics.
             if (state.supportsConversationChaining && state.currentProviderId !== undefined) {
-                state.responseId = completion.responseId;
+                state.responseId = finalCompletion.responseId;
                 state.responseProviderId = state.currentProviderId;
             }
             else {
@@ -836,8 +908,8 @@ export class ApplicationRunLoop {
                 state.responseProviderId = undefined;
             }
             let normalizedCompletionUsage;
-            if (completion.usage !== undefined) {
-                normalizedCompletionUsage = normalizeModelUsage(completion.usage);
+            if (finalCompletion.usage !== undefined) {
+                normalizedCompletionUsage = normalizeModelUsage(finalCompletion.usage);
                 if (normalizedCompletionUsage) {
                     const accumulated = addTokenUsage(normalizeModelUsage(state.usage), normalizedCompletionUsage);
                     state.usage = {
@@ -858,7 +930,7 @@ export class ApplicationRunLoop {
                 else {
                     // Providers may carry a future/opaque usage shape. Preserve the old
                     // pass-through behavior when normalization cannot recognize it.
-                    state.usage = completion.usage;
+                    state.usage = finalCompletion.usage;
                 }
             }
             else {
@@ -872,13 +944,13 @@ export class ApplicationRunLoop {
             // persistence and replay, and a cost event must never become a restored
             // history item.
             const record = this.#appendCostRecord(state, {
-                requestId,
+                requestId: activeRequestId,
                 provider: state.currentProviderId,
                 model: state.agent.model,
-                tier: resolveServiceTier(request),
+                tier: activeRequest ? resolveServiceTier(activeRequest) : 'standard',
                 outcome: 'completed',
                 usage: normalizedCompletionUsage,
-                providerUsd: completion.costUsd,
+                providerUsd: finalCompletion.costUsd,
             });
             queue.push({ type: 'cost_update', record });
             // Cost and usage arrive in the same completion metadata, so surface both
@@ -887,18 +959,18 @@ export class ApplicationRunLoop {
             if (normalizedCompletionUsage)
                 queue.push({ type: 'usage_update', usage: normalizedCompletionUsage });
             this.#evaluateRunBudget(state, stream, queue);
-            stream.lastResponseId = completion.responseId;
-            stream.rawResponses?.push(completion);
+            stream.lastResponseId = finalCompletion.responseId;
+            stream.rawResponses?.push(finalCompletion);
             // Some provider adapters report function calls only in the terminal
             // completion rather than as separate stream events. Their reasoning may
             // likewise be terminal-only, so associate it before replaying calls.
             const toolCalls = [...streamedToolCalls];
             if (!sawToolCall) {
-                for (const item of completion.output) {
+                for (const item of finalCompletion.output) {
                     if (item.type === 'reasoning')
                         pendingNativeReasoning = appendNativeReasoning(pendingNativeReasoning, item);
                 }
-                for (const item of completion.output) {
+                for (const item of finalCompletion.output) {
                     if (item.type !== 'tool_call')
                         continue;
                     pendingNativeReasoning = commitPendingNativeReasoning(state, stream, queue, pendingNativeReasoning);
@@ -916,7 +988,7 @@ export class ApplicationRunLoop {
             // applies compaction's replacement rule; keeping the item here also
             // makes it available to an in-flight tool continuation before that
             // terminal history commit occurs.
-            for (const item of completion.output) {
+            for (const item of finalCompletion.output) {
                 if (item.type !== 'provider_opaque')
                     continue;
                 const historyItem = {
@@ -927,7 +999,7 @@ export class ApplicationRunLoop {
                 state.input.push(item);
                 outputPush(stream, queue, { type: 'item', item });
             }
-            const assistantText = completion.output
+            const assistantText = finalCompletion.output
                 .filter((item) => item.type === 'message')
                 .flatMap((item) => item.content)
                 .map((part) => part.text)
@@ -1092,6 +1164,13 @@ export class ApplicationRunLoop {
                 maxParallelToolCalls,
                 dispatchOrder: group.map((entry) => entry.event.id),
             });
+            for (const entry of group) {
+                outputPush(stream, queue, {
+                    type: 'tool_call_dispatched',
+                    callId: entry.event.id,
+                    toolName: entry.event.name,
+                });
+            }
             if (group.length > 1) {
                 const results = await Promise.allSettled(group.map((entry) => this.#invokePlannedTool(entry, toolContext, state)));
                 const rejected = results.find((result) => result.status === 'rejected');
@@ -1211,6 +1290,9 @@ export class ApplicationRunLoop {
         }
     }
     #emitRunBudgetEvent(state, event, message, stream, queue) {
+        if (state.runBudgetEscalation === 'disabled') {
+            return;
+        }
         if (event.type === 'budget_stage' && event.stage === 'critical' && state.wrapUpOnCriticalRunBudget) {
             state.criticalWrapUpPending = true;
         }