@monotykamary/pi-retry 0.6.9 → 0.7.0

package/README.md

@@ -23,7 +23,8 @@ This extension automatically detects and retries **all** errors by default, with
 |------------|----------------|----------|
 | **Any retryable error** (catch-all) | **Indefinite** with capped backoff | Everything else — provider hiccups, stream exhaustion, credit issues, unknown errors |
 | HTTP 400/413 | **Indefinite** with capped backoff, NO compaction | Transient context overflow that might resolve |
-| Credit / payment errors | **Indefinite** with capped backoff | "Not Enough Credits", insufficient balance, 402 |
+| Credit / payment errors | **Indefinite** with capped backoff | "Not Enough Credits", insufficient balance, 402 — top up and the retry loop auto-resumes |
+| **Quota / session-limit / budget exhaustion** | **Not retried** — notify + stop | "You've hit your limit", `insufficient_quota`, "out of budget", suspended accounts |
 | Connection errors | **Indefinite** with capped backoff | Network hiccups, connection drops, socket errors, stream exhaustion |
 | Max tokens (`stopReason: "length"`) | **Auto-continue** indefinitely with hidden continuation turns | Model hits output token limit mid-generation |
 
@@ -50,6 +51,7 @@ This extension provides **automatic** infinite retry with sensible exponential b
 - Automatic detection of 400/413, connection, credit, and stream exhaustion errors
 - **Auto-continuation** when the model hits its max output tokens (`stopReason: "length"`) — indefinite, no cap, hidden from the TUI
 - **Indefinite retry** — Keeps retrying until success
+- **Auto-stop on quota/budget exhaustion** — Session limits, plan quotas, and budget caps ("You've hit your limit", "out of budget", `insufficient_quota`, suspended accounts) are detected and **not** retried, with a notification explaining why
 - Exponential backoff with cap: max 60s between retries
 - **Hidden triggers** — provider-valid custom messages use `display: false`, so retries do not add TUI clutter
 - Manual controls via unified `/retry` command
@@ -133,7 +135,7 @@ const BACKOFF_MULTIPLIER = 2;      // Double each time
 
 1. **Listen to `agent_end` event** — Fires after each agent turn completes
 2. **Check for any error** — Examine the last assistant message for `stopReason === "error"`
-3. **Blacklist check** — Skip known permanent failures (invalid API key, model not found, etc.)
+3. **Blacklist check** — Skip known permanent failures (invalid API key, model not found, quota/session-limit/budget exhaustion, suspended accounts, etc.)
 4. **Categorize for messaging** — Classify into 400/413, credit, connection, or other for nice UI notifications
 5. **Retry or continue with hidden turns** — Wait with exponential backoff, then trigger a provider-valid custom user turn via `pi.sendMessage()` with `display: false` and `triggerTurn: true`
 6. **Valid provider context** — Hidden retry and continuation messages remain in context so providers never receive a trailing assistant message
@@ -158,6 +160,16 @@ These are explicitly **not** retried:
 - Model not found / unknown model / no such model / model does not exist
 - Unsupported model
 
+### Non-Retryable (Quota / Session Limit / Budget)
+Exhausted quotas, session limits, and budgets are auto-detected and stop the retry loop (with an explanatory notification), because retrying is pointless until you act or the reset window passes:
+- **Usage / session limits with reset windows** — "You've hit your limit · resets …" and "5-hour limit reached" (Claude Code), "You've hit your usage limit" / "You've exceeded your usage limit" (Codex)
+- **Plan / billing quotas** — OpenAI `insufficient_quota`, "You exceeded your current quota, please check your plan and billing details" (OpenAI, Gemini — reached only after pi's built-in 429 retry gives up)
+- **Hard allotments** — OpenRouter `free-models-per-day`, Alibaba "Allocated quota exceeded" (`Throttling.AllocationQuota`), GitHub Copilot "premium request allowance"
+- **Budget exhaustion** — "out of budget", "Budget has been exceeded" (LiteLLM-style proxies), max/spending/monthly limits
+- **Suspended accounts** — "Your account … is suspended" (Kimi `exceeded_current_quota_error`)
+
+Deliberate distinction: plain pay-as-you-go **balance** errors stay retryable — DeepSeek 402 "Insufficient Balance", OpenRouter 402 "Insufficient credits", Kimi "exceeded your current token quota". A mid-session top-up lets the retry loop auto-resume, whereas session limits and budgets do not self-resolve for hours.
+
 ### Max Tokens (stopReason: "length")
 - The model hit its `max_tokens` / output token limit
 - The model's response was truncated mid-generation
@@ -291,7 +303,7 @@ pi install npm:@georgebashi/pi-retry
 - Error messages remain in the session history (but are invisible to the LLM)
 - May hit the same error repeatedly if the issue is persistent (use `Ctrl+C` to abort)
 - **Warning**: Retrying 400/413 without reducing context may fail repeatedly if the payload is genuinely too large
-- Non-retryable errors (invalid API key, missing model) are logged but not retried — you'll need to fix the underlying issue
+- Non-retryable errors (invalid API key, missing model, quota/session-limit/budget exhaustion) are logged but not retried — you'll need to fix the underlying issue, then use `/retry`
 
 ---
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@monotykamary/pi-retry",
-  "version": "0.6.9",
+  "version": "0.7.0",
   "description": "Extension suite for pi coding agent that handles 400/413 errors and connection errors with automatic retry",
   "type": "module",
   "author": "Tom X Nguyen",
@@ -32,9 +32,15 @@
     "src/",
     "README.md"
   ],
+  "peerDependencies": {
+    "@earendil-works/pi-agent-core": "*",
+    "@earendil-works/pi-coding-agent": "*",
+    "@earendil-works/pi-tui": "*"
+  },
   "devDependencies": {
     "@earendil-works/pi-agent-core": "^0.84.0",
     "@earendil-works/pi-coding-agent": "^0.84.0",
+    "@earendil-works/pi-tui": "^0.84.0",
     "@types/node": "25.9.1",
     "@vitest/coverage-v8": "4.1.7",
     "knip": "6.14.1",

package/retry.ts

@@ -1,4 +1,5 @@
 import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { matchesKey } from "@earendil-works/pi-tui";
 import { Agent } from "@earendil-works/pi-agent-core";
 import { AgentSession } from "@earendil-works/pi-coding-agent";
 import {
@@ -8,6 +9,7 @@ import {
   hasRetryableError,
   isNonRetryableError,
   isSilencedError,
+  hasQuotaExhaustedError,
   hasMaxTokensStop,
   isContextOverflowError,
   isAssistantMessage,
@@ -29,8 +31,9 @@ const RETRY_CANCELLED_EVENT = "pi-retry:cancelled";
  * Unified retry extension — retries EVERY error by default.
  *
  * Philosophy: any assistant message with stopReason === "error" is retried
- * indefinitely with exponential backoff, except a tiny blacklist of known
- * permanent failures (invalid API key, model not found, etc.).
+ * indefinitely with exponential backoff, except a small blacklist of known
+ * permanent failures and hard-stop conditions (invalid API key, model not
+ * found, quota/session-limit/budget exhaustion, suspended accounts, etc.).
  *
  * Specific categories (400/413, credit, connection, stream exhaustion, etc.)
  * are tracked for diagnostics but all share the same retry mechanism.
@@ -102,9 +105,10 @@ const stateOther = new RetryState();
 // Max_tokens continuation state (indefinite — no cap needed)
 const stateContinuation = new ContinuationState();
 
-// Abort flag: set when turn_end reports stopReason "aborted", cleared on
-// session_start and on fresh user activity.  Prevents triggerInvisibleContinue()
-// from starting a hidden retry turn after the user explicitly cancelled.
+// Abort flag: set when Pi's active signal is aborted or turn_end reports
+// stopReason "aborted", cleared on session_start and fresh user activity.
+// Prevents triggerInvisibleContinue() from starting a hidden retry turn after
+// the user explicitly cancelled, even if a tool/provider reports an error.
 let _userAborted = false;
 
 // Mutex: only one triggerInvisibleContinue may be in-flight at a time.
@@ -186,25 +190,32 @@ export default function (pi: ExtensionAPI) {
 
   pi.on("input", () => {
     _inputGeneration++;
+    // The generation change cancels any older loop; the new user request gets
+    // its own retry eligibility even if the previous request was aborted.
+    _userAborted = false;
   });
 
   // Reset retry counters on successful completion (not max_tokens, not error)
   pi.on("turn_end", async (event, ctx) => {
     const msg = event.message as any;
+    if (
+      ctx.signal?.aborted ||
+      (msg.role === "assistant" && msg.stopReason === "aborted")
+    ) {
+      // User cancelled — reset retry state so it doesn't leak into other
+      // branches of the session tree. The signal check matters for tool calls:
+      // some tools/providers finish with an error-shaped result after abort.
+      state400.reset();
+      stateCredit.reset();
+      stateConnection.reset();
+      stateOther.reset();
+      stateContinuation.endContinuation();
+      // Signal to any in-flight triggerInvisibleContinue or pending retry
+      // that the user has cancelled — do not queue another retry turn.
+      _userAborted = true;
+      return;
+    }
     if (msg.role === "assistant" && msg.stopReason !== "error") {
-      if (msg.stopReason === "aborted") {
-        // User cancelled — reset retry state so it doesn't leak into other
-        // branches of the session tree.
-        state400.reset();
-        stateCredit.reset();
-        stateConnection.reset();
-        stateOther.reset();
-        stateContinuation.endContinuation();
-        // Signal to any in-flight triggerInvisibleContinue or pending retry
-        // that the user has cancelled — do not queue another retry turn.
-        _userAborted = true;
-        return;
-      }
       if (msg.stopReason !== "length") {
         // Normal completion — reset everything including continuation count
         state400.succeed();
@@ -230,6 +241,14 @@ export default function (pi: ExtensionAPI) {
   // triggerInvisibleContinue(), which owns the retry loop with backoff
   // sleeps that happen AFTER processEvents returns (outside the agent run).
   pi.on("agent_end", async (event, ctx) => {
+    // Prefer Pi's run signal over provider-specific stop-reason mapping. An
+    // Escape during a Fabric/core tool may settle as an error-shaped result,
+    // but it is still a user cancellation and must never schedule a retry.
+    if (ctx.signal?.aborted) {
+      _userAborted = true;
+      return;
+    }
+
     const entries = ctx.sessionManager.getEntries();
     const lastAssistant = getLastAssistantMessage(entries);
 
@@ -309,7 +328,12 @@ export default function (pi: ExtensionAPI) {
     // (silenced errors are neither retried nor shown)
     if (isNonRetryableError(lastAssistant) && !isSilencedError(lastAssistant)) {
       const errorMsg = lastAssistant.errorMessage || "Unknown error";
-      ctx.ui.notify(`Non-retryable error (not retried): ${errorMsg.substring(0, 100)}`, "error");
+      ctx.ui.notify(
+        hasQuotaExhaustedError(lastAssistant)
+          ? `Quota/limit exhausted — not retrying (fix plan/billing or wait for the reset window, then /retry): ${errorMsg.substring(0, 100)}`
+          : `Non-retryable error (not retried): ${errorMsg.substring(0, 100)}`,
+        "error",
+      );
     }
   });
 
@@ -422,6 +446,19 @@ export default function (pi: ExtensionAPI) {
         return;
       }
 
+      // Non-retryable errors (permanent failures + quota/budget
+      // exhaustion): report clearly instead of the generic fallback.
+      if (isNonRetryableError(lastAssistant)) {
+        const errorMsg = lastAssistant.errorMessage || "Unknown error";
+        ctx.ui.notify(
+          hasQuotaExhaustedError(lastAssistant)
+            ? `Quota/limit exhausted — resolve the plan/billing issue or wait for the reset window first: ${errorMsg.substring(0, 100)}`
+            : `Non-retryable error (fix the underlying issue first, then /retry): ${errorMsg.substring(0, 100)}`,
+          "warning",
+        );
+        return;
+      }
+
       // Auto-detect error type and trigger appropriate retry
       if (has400or413Error(lastAssistant)) {
         ctx.ui.notify("Manually retrying 400/413 error...", "info");
@@ -480,16 +517,20 @@ export default function (pi: ExtensionAPI) {
     if (ctx.mode === "tui") {
       _terminalInputUnsubscribe = ctx.ui.onTerminalInput(data => {
         if (
-          data !== "\x1b" ||
+          !matchesKey(data, "escape") ||
           !_continueInProgress ||
           _continueGeneration !== _sessionGeneration
         ) {
           return undefined;
         }
 
+        // Cancel pi-retry's out-of-band loop, but let Pi's native interrupt
+        // handler receive the same key. Pi owns the active turn and queue: its
+        // handler aborts the current tool/model call and clears any queued
+        // steer/follow-up messages. Consuming Escape here bypasses that cleanup
+        // and can let AgentSession continue after the user asked it to stop.
         _userAborted = true;
-        ctx.abort();
-        return { consume: true };
+        return undefined;
       });
     }
   });

package/src/error-patterns.ts

@@ -131,6 +131,56 @@ const SILENCED_PATTERNS = [
   /cannot continue from message role/i,
 ];
 
+// Quota / session-limit / budget exhaustion — hard stops:
+// retrying is pointless until the user upgrades, tops up a budget, or waits
+// out a reset window measured in hours/days. Distinct from per-minute rate
+// limits and pay-as-you-go balance errors, which stay retryable.
+//
+// Evidence (real provider messages):
+// - Claude Code:  "You've hit your limit · resets 4pm (Asia/Kuala_Lumpur)"
+//                 "Claude usage limit reached. Your limit will reset at 3pm"
+//                 "5-hour limit reached · resets 12pm"
+// - Codex:        "You've hit your usage limit. Upgrade to Plus"
+//                 "You've exceeded your usage limit."
+// - OpenAI:       code "insufficient_quota" — "You exceeded your current
+//                 quota, please check your plan and billing details"
+// - Gemini:       same sentence in 429 RESOURCE_EXHAUSTED responses; only
+//                 reaches us after pi's built-in 429 retry gives up
+// - OpenRouter:   "Rate limit exceeded: free-models-per-day. ..."
+// - Alibaba:      "Allocated quota exceeded, please increase your quota limit"
+//                 (Throttling.AllocationQuota — hard cap; RateQuota is
+//                 transient and stays retryable)
+// - Copilot:      "You have exceeded your premium request allowance"
+// - LiteLLM:      "Budget has been exceeded! Current cost: …, Max budget: …"
+// - Kimi:         "Your account {org}<{ak}> is suspended, please check your
+//                 plan and billing details" (exceeded_current_quota_error)
+export const QUOTA_EXHAUSTED_PATTERNS = [
+  // Session / usage limits with reset windows (Claude, Codex)
+  /hit your (usage )?limit/i,
+  /usage\s*limit\s*(has\s*been\s*)?reached/i,
+  /hour\s*limit\s*reached/i, // "5-hour limit reached" — must NOT hit DeepSeek 429 "Rate Limit Reached"
+  /limit\s*will\s*reset\s*at/i,
+  /session\s*(limit|quota)/i,
+  /exceeded your usage limit/i,
+  // Billing / plan quotas (OpenAI insufficient_quota, Gemini RESOURCE_EXHAUSTED)
+  /insufficient[_\s]quota/i,
+  /exceeded your current quota/i, // Kimi's "...current token quota" (balance, retryable) intentionally not matched
+  // Hard allotments
+  /free.models.per.day/i, // OpenRouter free-tier daily pool
+  /allocated\s*quota/i, // Alibaba Throttling.AllocationQuota
+  /premium\s*request\s*allowance/i, // GitHub Copilot monthly allowance
+  /monthly\s*(limit|quota|budget|allowance)/i,
+  // Budget exhaustion (LiteLLM and similar proxies/gateways)
+  /out of budget/i,
+  /budget\s*(has\s*been\s*)?(exceeded|exhausted|limit)/i,
+  /max(imum)?\s*budget\s*(exceeded|reached|limit)/i,
+  /spending\s*limit/i,
+  // Suspended accounts (Kimi exceeded_current_quota_error suspended form)
+  /account\b[^.]*\bis\s*suspended/i,
+  // Generic
+  /quota\s*(exhausted|depleted)/i,
+];
+
 // ── Type guard ──
 
 export function isAssistantMessage(message: AgentMessage): message is Extract<AgentMessage, { role: "assistant" }> {
@@ -184,7 +234,7 @@ export function isContextOverflowError(message: AgentMessage): boolean {
 export function hasRetryableError(message: AgentMessage): boolean {
   if (!isAssistantMessage(message)) return false;
   if (message.stopReason !== "error" || !message.errorMessage) return false;
-  return !NON_RETRYABLE_PATTERNS.some(p => p.test(message.errorMessage!));
+  return !isNonRetryableError(message);
 }
 
 /**
@@ -193,7 +243,26 @@ export function hasRetryableError(message: AgentMessage): boolean {
 export function isNonRetryableError(message: AgentMessage): boolean {
   if (!isAssistantMessage(message)) return false;
   if (message.stopReason !== "error" || !message.errorMessage) return false;
-  return NON_RETRYABLE_PATTERNS.some(p => p.test(message.errorMessage!));
+  return (
+    NON_RETRYABLE_PATTERNS.some(p => p.test(message.errorMessage!)) ||
+    QUOTA_EXHAUSTED_PATTERNS.some(p => p.test(message.errorMessage!))
+  );
+}
+
+/**
+ * Returns true for quota / session-limit / budget exhaustion errors where
+ * retrying is pointless until the user acts (upgrade, top up a budget) or a
+ * long reset window passes (hours/days). Treated as non-retryable.
+ *
+ * Deliberately NOT matched: per-minute rate limits (429s) and pay-as-you-go
+ * balance errors (DeepSeek "Insufficient Balance", OpenRouter "Insufficient
+ * credits", Kimi "exceeded your current token quota") — those stay retryable
+ * so a mid-session top-up auto-resumes.
+ */
+export function hasQuotaExhaustedError(message: AgentMessage): boolean {
+  if (!isAssistantMessage(message)) return false;
+  if (message.stopReason !== "error" || !message.errorMessage) return false;
+  return QUOTA_EXHAUSTED_PATTERNS.some(p => p.test(message.errorMessage!));
 }
 
 /**
@@ -209,7 +278,8 @@ export function isSilencedError(message: AgentMessage): boolean {
 
 // ── Categorisation (for UI messages) ──
 
-export function getErrorCategory(errorMessage: string): '400-413' | 'credit' | 'connection' | 'builtin' | 'other' {
+export function getErrorCategory(errorMessage: string): '400-413' | 'credit' | 'connection' | 'builtin' | 'quota' | 'other' {
+  if (QUOTA_EXHAUSTED_PATTERNS.some(p => p.test(errorMessage))) return 'quota';
   if (ERROR_400_413_PATTERNS.some(p => p.test(errorMessage))) return '400-413';
   if (CREDIT_ERROR_PATTERNS.some(p => p.test(errorMessage))) return 'credit';
   if (CONNECTION_ERROR_PATTERNS.some(p => p.test(errorMessage))) return 'connection';