@crouton-kit/crouter 0.3.64 → 0.3.66

package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/__tests__/provider-rotation.test.ts

@@ -1074,10 +1074,10 @@ test("treats a request timeout as transient: retries the SAME provider in place,
   }
 });
 
-// Regression: when a transient blip never recovers, the turn must end with a clear,
-// actionable error instead of spinning forever or silently rotating a healthy credential
-// away to a fallback provider that was never the actual problem.
-test("ends the turn without rotating once the transient retry budget is exhausted", async () => {
+// Once the in-place transient retry budget is exhausted, the failure is surfaced to the
+// user AND the credential is rotated away: the struggling credential cools down briefly and
+// the turn continues on the fallback provider, rather than hard-failing.
+test("surfaces the transient error and rotates to the fallback once the in-place retry budget is exhausted", async () => {
   resetRotationState();
   writeLadderConfig();
   helpers.writeSubscriptionPool("anthropic", [credential("primary")]);
@@ -1085,6 +1085,10 @@ test("ends the turn without rotating once the transient retry budget is exhauste
 
   const { ctx, setModelCalls, providers, events, pi } = makeRotationCtx();
   ctx.model = { provider: "anthropic", id: "claude-opus-4-8", api: "anthropic-messages" };
+  const notifications: Array<{ message: string; level: string }> = [];
+  ctx.ui.notify = (message: string, level: string) => {
+    notifications.push({ message, level });
+  };
   const streamedModels: any[] = [];
   const sleepCalls: number[] = [];
   rotationModule.__setSleepForTest(async (ms) => {
@@ -1093,8 +1097,12 @@ test("ends the turn without rotating once the transient retry budget is exhauste
   rotationModule.__setStreamForProviderForTest((model) => {
     streamedModels.push(model);
     return (async function* () {
-      // 503 with no retry-after header is transient by STATUS, not just by message.
-      yield { type: "error", error: Object.assign(new Error("Service Unavailable"), { status: 503 }) };
+      if (model.provider === "anthropic") {
+        // 503 with no retry-after header is transient by STATUS, not just by message.
+        yield { type: "error", error: Object.assign(new Error("Service Unavailable"), { status: 503 }) };
+        return;
+      }
+      yield { type: "text_delta", text: "hello from codex fallback" };
     })() as any;
   });
 
@@ -1108,13 +1116,17 @@ test("ends the turn without rotating once the transient retry budget is exhauste
     const emitted = [] as any[];
     for await (const event of stream) emitted.push(event);
 
-    assert.deepEqual(setModelCalls, []);
-    assert.deepEqual(streamedModels.map((m) => m.provider), ["anthropic", "anthropic", "anthropic"]);
+    // Retried the same anthropic credential in place (3 attempts) then rotated to codex.
+    assert.equal(streamedModels.filter((m) => m.provider === "anthropic").length, 3);
     assert.deepEqual(sleepCalls, [250, 750]);
-    assert.equal(emitted.length, 1);
-    assert.equal(emitted[0].type, "error");
-    assert.match(emitted[0].error.errorMessage, /transient error after 3 attempts/);
-    assert.equal(helpers.readSubscriptionPool("anthropic")[0].rateLimitedUntil, 0);
+    assert.deepEqual(setModelCalls.map((m) => m.provider), ["openai-codex"]);
+    // The turn was served on the fallback, never failed with a terminal error.
+    assert.ok(emitted.some((e) => e.type === "text_delta"));
+    assert.ok(!emitted.some((e) => e.type === "error"));
+    // The user was told about the transient failure AND that it rotated.
+    assert.ok(notifications.some((n) => /transient error after 3 attempts/.test(n.message) && /rotating/i.test(n.message)));
+    // The struggling anthropic credential was cooled down so rotation moved on.
+    assert.ok(helpers.readSubscriptionPool("anthropic")[0].rateLimitedUntil > Date.now());
   } finally {
     rotationModule.__setStreamForProviderForTest(undefined);
     rotationModule.__setSleepForTest(undefined);

package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts

@@ -47,17 +47,23 @@ const INVALID_REFRESH_TOKEN_BACKOFF_MS = 30 * 24 * 60 * 60 * 1000;
 // not a wait. Cap at the same window as our own default rate-limit backoff.
 const SOLE_PROVIDER_MAX_WAIT_MS = DEFAULT_RATE_LIMIT_BACKOFF_MS;
 // Short in-place retry budget for a transient network/5xx blip on the SAME credential
-// (#122): a couple of quick retries, no cooldown, no rotation to another credential or
-// provider. Once this budget is exhausted the turn ends with an actionable error.
+// (#122): a couple of quick retries, no cooldown. Once this budget is exhausted we no
+// longer hard-fail the turn -- we surface the error and rotate (see TRANSIENT_ROTATE below).
 const TRANSIENT_RETRY_BACKOFFS_MS = [250, 750];
+// After the in-place transient retry budget is exhausted, rotate instead of failing: the
+// credential itself is probably healthy (this is a provider-side blip like "overloaded"),
+// so cool it down only briefly -- long enough to rotate to another credential / fallback
+// provider, short enough that it comes back quickly if it was the only option.
+const TRANSIENT_ROTATE_COOLDOWN_MS = 60 * 1000;
 
 let runtimeContext: ExtensionContext | undefined;
 let extensionAPI: ExtensionAPI | undefined;
 
 type AttemptResult = {
-  result: "success" | "rate_limited" | "fatal";
+  result: "success" | "rate_limited" | "transient_exhausted" | "fatal";
   retryAfterMs?: number;
   attemptAt: number;
+  attempts?: number;
   classification?: ProviderErrorClassification;
 };
 
@@ -466,21 +472,16 @@ async function streamProviderAttempt(
       return { result: "rate_limited", retryAfterMs: requestedRetryAfterMs, attemptAt, classification };
     }
 
-    // Transient: retry the SAME credential/provider in place -- no cooldown, no rotation.
+    // Transient: retry the SAME credential/provider in place first -- no cooldown for the
+    // quick in-place budget.
     const backoffMs = TRANSIENT_RETRY_BACKOFFS_MS[attempt - 1];
     if (backoffMs === undefined) {
-      // Exhausted the short in-place retry budget: surface a clear terminal error rather
-      // than spinning forever or silently rotating a healthy credential away.
-      bufferedEvents.push({
-        type: "error",
-        reason: "error",
-        error: toAssistantError(
-          model,
-          `${getProviderLabel(providerId)} "${credential.label}" hit a transient error after ${attempt} attempts (${classification.reason})`,
-        ),
-      });
-      flush();
-      return { result: "fatal", attemptAt, classification };
+      // In-place retry budget exhausted. Rather than hard-failing the turn, hand back a
+      // transient-exhausted result so the caller surfaces the error to the user AND rotates
+      // to another credential / fallback provider. Nothing is emitted into the stream here:
+      // the buffered (pre-content) events are discarded exactly like the rate-limit path, so
+      // the rotation target streams a clean turn.
+      return { result: "transient_exhausted", attemptAt, classification, attempts: attempt };
     }
     logRotation(
       `${getProviderLabel(providerId)} subscription "${credential.label}" hit a transient error (${classification.reason}, status=${classification.status ?? "n/a"}); retrying same credential in ${backoffMs}ms (attempt ${attempt + 1})`,
@@ -599,6 +600,30 @@ async function runManagedProvider(
       logRotation(
         `${getProviderLabel(providerId)} subscription "${freshCredential.label}" rate-limited (status=${result.classification?.status ?? "n/a"}, reason=${result.classification?.reason ?? "unclassified"}, ${retryAfterPart}); cooling down for ${Math.ceil(cooldownMs / 1000)}s, trying next credential`,
       );
+      // Surface the rotation to the user too (parity with the transient-exhausted path), so a
+      // rate-limit-driven credential/provider switch is visible rather than silent.
+      runtimeContext?.ui.notify?.(
+        `${getProviderLabel(providerId)} "${freshCredential.label}" rate-limited (cooling down ${Math.ceil(cooldownMs / 1000)}s); rotating to another credential/provider.`,
+        "warn",
+      );
+      continue;
+    }
+
+    if (result.result === "transient_exhausted") {
+      // The credential is likely healthy -- this was a provider-side transient blip (e.g.
+      // "overloaded") that survived the in-place retries. Surface it to the user and rotate:
+      // cool the credential down briefly so this loop (and the pool-exhaustion logic below)
+      // moves on to the next credential / fallback provider instead of failing the turn.
+      const reason = result.classification?.reason ?? "transient error";
+      markSubscriptionRateLimited(providerId, freshCredential.label, TRANSIENT_ROTATE_COOLDOWN_MS, result.attemptAt);
+      runtimeContext?.ui.notify?.(
+        `${getProviderLabel(providerId)} "${freshCredential.label}" hit a transient error after ${result.attempts ?? 1} attempts (${reason}); rotating to another credential/provider.`,
+        "warn",
+      );
+      logRotation(
+        `${getProviderLabel(providerId)} subscription "${freshCredential.label}" transient error exhausted (reason=${reason}, status=${result.classification?.status ?? "n/a"}); cooling down ${Math.ceil(TRANSIENT_ROTATE_COOLDOWN_MS / 1000)}s and rotating`,
+      );
+      setStatus(formatStatusLine(providerId));
       continue;
     }