@pentoshi/clai 3.9.1 → 3.9.2

package/dist/agent/runner.js

@@ -1,6 +1,7 @@
 import chalk from "chalk";
 import { join } from "node:path";
-import { streamWithProvider, completeWithProvider, isEmptyCompletionError } from "../llm/router.js";
+import { streamWithProvider, completeWithProvider } from "../llm/router.js";
+import { classifyStreamFailure, planStreamRecovery, recordRecoveryAttempt, createStreamRecoveryState, resetStreamRecoveryState, } from "./stream-recovery.js";
 import { resolveToolDialect } from "../llm/capabilities.js";
 import { syntheticToolCallId, isTextOnlyModel, fromWireName, } from "../llm/tool-protocol.js";
 import { sanitizeAssistantText } from "../ui/ansi-box.js";
@@ -102,6 +103,28 @@ export function shouldYieldForResponderBeforeReport(plan, runningJobs, notificat
             !notification.readAt &&
             !notification.analyzedAt));
 }
+/**
+ * Cancellable backoff. Resolves after `ms`, or rejects immediately if the
+ * signal aborts (double-Esc) so recovery waits never trap a cancelled turn.
+ */
+function delay(ms, signal) {
+    if (ms <= 0)
+        return Promise.resolve();
+    if (signal?.aborted) {
+        return Promise.reject(signal.reason ?? new Error("Aborted"));
+    }
+    return new Promise((resolve, reject) => {
+        const onAbort = () => {
+            clearTimeout(timer);
+            reject(signal?.reason ?? new Error("Aborted"));
+        };
+        const timer = setTimeout(() => {
+            signal?.removeEventListener("abort", onAbort);
+            resolve();
+        }, ms);
+        signal?.addEventListener("abort", onAbort, { once: true });
+    });
+}
 export async function runAgentTurn(prompt, options = {}) {
     const agentMode = options.mode === "plan" || options.mode === "agent" || options.mode === "ask"
         ? options.mode
@@ -696,6 +719,13 @@ export async function runAgentTurn(prompt, options = {}) {
         // to actually act instead of silently returning an empty answer.
         let emptyVisibleRetries = 0;
         let retryWithoutThinking = false;
+        // Robust stream-failure recovery. When a provider stream/complete fails we
+        // try working approaches (backoff, compaction, thinking-off, provider
+        // fallback) before surrendering the turn — see ./stream-recovery. Both are
+        // reset on any successful stream so each failure episode gets a fresh
+        // budget and we only give up in the worst case.
+        let allowModelFallback = false;
+        const recoveryState = createStreamRecoveryState();
         // Track tool calls truncated by the token limit so we can ask the model
         // to retry in smaller pieces instead of leaking broken JSON as an answer.
         let truncatedToolRetries = 0;
@@ -2254,6 +2284,10 @@ export async function runAgentTurn(prompt, options = {}) {
         /** E4: consecutive free-tier stream failures this turn. */
         let freeTierConsecutiveFailures = 0;
         let freeTierLargeContextWarned = false;
+        // Surface the free-tier "failed N times / switch provider" advisory at most
+        // once per turn — the recovery planner already narrates each retry, so
+        // repeating this on every failure just adds noise.
+        let freeTierAdvisoryShown = false;
         const summarizeForCompaction = async (summaryPrompt) => {
             const response = await completeWithProvider({
                 provider,
@@ -2522,7 +2556,7 @@ export async function runAgentTurn(prompt, options = {}) {
                         completion = await streamWithProvider({
                             provider,
                             model,
-                            allowModelFallback: false,
+                            allowModelFallback,
                             messages,
                             temperature: /minimax-m3/i.test(model) ? 1.0 : 0.2,
                             maxTokens: stepMaxTokens,
@@ -2663,36 +2697,67 @@ export async function runAgentTurn(prompt, options = {}) {
                             }
                         });
                         freeTierConsecutiveFailures = 0;
+                        // Stream succeeded → the failure episode is over. Reset the recovery
+                        // budget and the one-shot fallback flag so a later, unrelated failure
+                        // starts fresh (and we never give up while making progress).
+                        resetStreamRecoveryState(recoveryState);
+                        allowModelFallback = false;
                     }
                     catch (streamError) {
+                        // User cancelled (double-Esc) — never try to recover, just stop.
+                        if (options.signal?.aborted)
+                            throw streamError;
                         // E4: track free-tier failures for advisory notices (never blocks).
                         freeTierConsecutiveFailures += 1;
-                        for (const notice of freeTierGuardNotices({
-                            provider,
-                            estimatedInputTokens: contextBreakdown.estimatedTotalTokens,
-                            consecutiveFailures: freeTierConsecutiveFailures,
-                        })) {
-                            if (notice.includes("Large context"))
-                                continue; // already shown above
-                            writeNotice("warn", notice, chalk.yellow(`  ⚠ ${notice}\n`));
-                        }
-                        // A fully empty model completion (no text, no tool calls) must not
-                        // kill the turn. It shows up most after auto-compaction, when the
-                        // tail ends on re-injected system context and the model has nothing
-                        // to answer. Append a trailing user nudge (so the turn no longer
-                        // ends on system messages) and retry, bounded like the
-                        // successful-but-empty path below.
-                        if (!options.signal?.aborted &&
-                            isEmptyCompletionError(streamError) &&
-                            emptyVisibleRetries < 3) {
-                            emptyVisibleRetries += 1;
-                            writeNotice("warn", "model streamed an empty response — nudging it to continue", chalk.yellow("  ⚠ model streamed an empty response — nudging it to continue\n"));
-                            messages.push(recoveryUserMessage("Your previous response was empty. Continue the task now: emit your next tool call, " +
-                                "or give your final answer if every required step is already complete and verified. " +
-                                "Do not reply with an empty message."));
-                            continue;
+                        if (!freeTierAdvisoryShown) {
+                            for (const notice of freeTierGuardNotices({
+                                provider,
+                                estimatedInputTokens: contextBreakdown.estimatedTotalTokens,
+                                consecutiveFailures: freeTierConsecutiveFailures,
+                            })) {
+                                if (notice.includes("Large context"))
+                                    continue; // already shown above
+                                writeNotice("warn", notice, chalk.yellow(`  ⚠ ${notice}\n`));
+                                freeTierAdvisoryShown = true;
+                            }
+                        }
+                        // Robust recovery: a single flaky provider/model (empty admission,
+                        // connection glitch, capacity 5xx, rate limit, or an oversized
+                        // request) must not kill the turn. Classify the failure and take a
+                        // bounded, escalating recovery step — back off, compact, drop
+                        // thinking, or let the router fall back to another provider/model.
+                        // We only rethrow (stop the turn) in the worst case: every approach
+                        // for that failure class is exhausted or the total budget is spent.
+                        const failureKind = classifyStreamFailure(streamError);
+                        const plan = planStreamRecovery({
+                            kind: failureKind,
+                            state: recoveryState,
+                        });
+                        if (plan.action === "give-up") {
+                            throw streamError;
+                        }
+                        recordRecoveryAttempt(recoveryState, failureKind);
+                        if (plan.notice) {
+                            writeNotice("warn", plan.notice, chalk.yellow(`  ⚠ ${plan.notice}\n`));
                         }
-                        throw streamError;
+                        if (plan.disableThinking)
+                            retryWithoutThinking = true;
+                        if (plan.allowModelFallback)
+                            allowModelFallback = true;
+                        if (plan.forceCompact) {
+                            await maybeAutoCompact(`stream-recovery:${failureKind}`, true);
+                        }
+                        if (plan.nudge) {
+                            messages.push(recoveryUserMessage(plan.nudge));
+                        }
+                        if (plan.delayMs > 0) {
+                            emit({
+                                type: "status",
+                                text: `retrying in ${Math.ceil(plan.delayMs / 1000)}s (${failureKind})`,
+                            });
+                            await delay(plan.delayMs, options.signal);
+                        }
+                        continue;
                     }
                 }
                 finally {