@north-light/crouter 0.3.304 → 0.3.305

package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/__tests__/integration/provider-rotation.test.ts

@@ -1251,6 +1251,93 @@ test("surfaces the underlying transient error without reporting a rate limit whe
   }
 });
 
+test("carries an observed 503 into the terminal provider error after bounded recovery", async () => {
+  await resetRotationState();
+  writeLadderConfig();
+  await helpers.writeSubscriptionPool("openai-codex", [credential("codex")]);
+
+  const { ctx, providers, events, pi } = makeRotationCtx(undefined);
+  rotationModule.__setSleepForTest(async () => {});
+  rotationModule.__setStreamForProviderForTest((model, _context, options) => (async function* () {
+    await options?.onResponse?.({ status: 503, headers: {} }, model);
+    yield { type: "error", error: new Error("Request failed", { cause: Object.assign(new Error("socket reset"), { code: "ECONNRESET" }) }) };
+  })() as any);
+
+  try {
+    await providerRotationExtension(pi);
+    await events.session_start?.({}, ctx);
+    const emitted = [] as any[];
+    for await (const event of providers["openai-codex"].streamSimple(ctx.model, ctx)) emitted.push(event);
+
+    const terminal = emitted.find((event) => event.type === "error");
+    assert.ok(terminal, "the bounded attempts settle a terminal provider error");
+    assert.equal(terminal.error.provider, "openai-codex");
+    assert.equal(terminal.error.model, "gpt-5.6-sol");
+    const details = terminal.error.diagnostics?.find((diagnostic: any) => diagnostic.type === "crtr:provider-failure")?.details;
+    assert.equal(details?.status, 503, "the response status survives the stream error boundary");
+    assert.equal(details?.causes?.[0]?.code, "ECONNRESET", "the cause chain survives the stream error boundary");
+  } finally {
+    rotationModule.__setStreamForProviderForTest(undefined);
+    rotationModule.__setSleepForTest(undefined);
+  }
+});
+
+test("does not replay a partially exposed response, while retaining its 503 facts", async () => {
+  await resetRotationState();
+  writeLadderConfig();
+  await helpers.writeSubscriptionPool("anthropic", [credential("primary")]);
+
+  const { ctx, setModelCalls, providers, events, pi } = makeRotationCtx();
+  let attempts = 0;
+  rotationModule.__setStreamForProviderForTest((model, _context, options) => (async function* () {
+    attempts += 1;
+    yield { type: "text_delta", text: "already exposed" };
+    await options?.onResponse?.({ status: 503, headers: {} }, model);
+    yield { type: "error", error: new Error("Request failed") };
+  })() as any);
+
+  try {
+    await providerRotationExtension(pi);
+    await events.session_start?.({}, ctx);
+    const emitted = [] as any[];
+    for await (const event of providers["anthropic"].streamSimple(ctx.model, ctx)) emitted.push(event);
+
+    assert.equal(attempts, 1, "content commits the attempt before its failure");
+    assert.ok(setModelCalls.every((selected) => selected.provider === "anthropic"), "a partial response never rotates to another route");
+    const terminal = emitted.find((event) => event.type === "error");
+    assert.equal(terminal.error.diagnostics?.find((diagnostic: any) => diagnostic.type === "crtr:provider-failure")?.details?.status, 503);
+  } finally {
+    rotationModule.__setStreamForProviderForTest(undefined);
+  }
+});
+
+test("keeps a known authentication rejection fatal", async () => {
+  await resetRotationState();
+  writeLadderConfig();
+  await helpers.writeSubscriptionPool("openai-codex", [credential("codex")]);
+
+  const { ctx, providers, events, pi } = makeRotationCtx(undefined);
+  let attempts = 0;
+  rotationModule.__setStreamForProviderForTest((model, _context, options) => (async function* () {
+    attempts += 1;
+    await options?.onResponse?.({ status: 401, headers: {} }, model);
+    yield { type: "error", error: new Error("Request failed") };
+  })() as any);
+
+  try {
+    await providerRotationExtension(pi);
+    await events.session_start?.({}, ctx);
+    const emitted = [] as any[];
+    for await (const event of providers["openai-codex"].streamSimple(ctx.model, ctx)) emitted.push(event);
+
+    assert.equal(attempts, 1, "a protocol/auth failure never enters transient retries");
+    const terminal = emitted.find((event) => event.type === "error");
+    assert.equal(terminal.error.diagnostics?.find((diagnostic: any) => diagnostic.type === "crtr:provider-failure")?.details?.status, 401);
+  } finally {
+    rotationModule.__setStreamForProviderForTest(undefined);
+  }
+});
+
 // Regression: a transport-poisoned process (a corrupted TLS upload leaves undici's H2
 // session destroyed, so EVERY request fails with no HTTP response while fresh processes
 // succeed) must self-terminate after consecutive transport-only exhausted passes so the

package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js

@@ -8,6 +8,7 @@ import { envModelExact, envNodeId } from '../../../shared/env.js';
 import { ManagedProviderCoolingError } from '../../../core/runtime/managed-provider-cooling.js';
 import { recordTransportPoisonVerdict } from '../../../core/runtime/busy.js';
 import { recognizeProviderError } from '../../../core/provider-error-recognition.js';
+import { normalizeProviderFailure, providerFailureDiagnostic } from '../../../core/provider-failure.js';
 // A response without Retry-After proves only that THIS attempt was throttled; it
 // does not authorize crouter to invent a five-minute provider-wide outage. Keep
 // a short turn-local probe backoff to avoid an immediate retry loop, then let a
@@ -59,30 +60,31 @@ function classifyProviderSignal(signal) {
     }
     return { kind: 'fatal', status: signal.status, reason: 'unclassified', hadRetryAfterHeader };
 }
-function classifyProviderError(error, observed) {
-    const value = error;
-    const message = [value?.errorMessage, value?.message, error instanceof Error ? error.message : '']
+function classifyProviderFailure(failure, retryAfterMs) {
+    const nested = failure.causes ?? [];
+    const code = failure.code ?? nested.find((cause) => cause.code !== undefined)?.code;
+    const errno = failure.errno ?? nested.find((cause) => cause.errno !== undefined)?.errno;
+    const message = [failure.message, failure.body, ...nested.flatMap((cause) => [cause.message, cause.body])]
         .filter((part) => typeof part === 'string' && part.length > 0)
         .join(' ');
     return classifyProviderSignal(recognizeProviderError({
-        status: observed.status ?? value?.status,
-        retryAfterMs: observed.retryAfterMs,
+        status: failure.status,
+        retryAfterMs,
+        code,
+        errno,
         message,
     }));
 }
+function classifyProviderError(error, observed) {
+    return classifyProviderFailure(normalizeProviderFailure(error, { status: observed.status }), observed.retryAfterMs);
+}
 /** Test seam for the policy mapping over the shared recognition signal. */
 export function __classifyProviderErrorForTest(error, observed = {}) {
     return classifyProviderError(error, observed);
 }
-function providerErrorMessage(error) {
-    const value = error;
-    const message = typeof value?.errorMessage === "string" && value.errorMessage.trim()
-        ? value.errorMessage.trim()
-        : typeof value?.message === "string" && value.message.trim()
-            ? value.message.trim()
-            : error instanceof Error && error.message.trim()
-                ? error.message.trim()
-                : undefined;
+function providerFailureAssistantMessage(model, failure) {
+    const message = toAssistantError(model, failure.message);
+    message.diagnostics = [providerFailureDiagnostic(failure)];
     return message;
 }
 async function defaultSleep(ms) {
@@ -435,6 +437,7 @@ async function streamProviderAttempt(state, providerId, model, context, options,
         let sawContent = false;
         let committed = false;
         let classification;
+        let failure;
         let failureMessage;
         // Events are buffered ONLY until this attempt commits to the provider -- i.e. until the
         // first real content event (thinking/text/toolcall) arrives, after which rotating away is
@@ -462,12 +465,13 @@ async function streamProviderAttempt(state, providerId, model, context, options,
                         flush();
                         return { result: "fatal", attemptAt };
                     }
-                    failureMessage = providerErrorMessage(event.error);
-                    classification = classifyProviderError(event.error, { status: observedStatus, retryAfterMs: requestedRetryAfterMs });
+                    failure = normalizeProviderFailure(event.error, { status: observedStatus });
+                    failureMessage = failure.message;
+                    classification = classifyProviderFailure(failure, requestedRetryAfterMs);
                     if (classification.kind === "fatal" || sawContent) {
-                        emit(event);
+                        emit({ ...event, error: providerFailureAssistantMessage(model, failure) });
                         flush();
-                        return { result: "fatal", attemptAt, classification };
+                        return { result: "fatal", attemptAt, classification, failure };
                     }
                     break;
                 }
@@ -490,16 +494,17 @@ async function streamProviderAttempt(state, providerId, model, context, options,
                 flush();
                 return { result: "fatal", attemptAt };
             }
-            failureMessage = providerErrorMessage(error);
-            classification = classifyProviderError(error, { status: observedStatus, retryAfterMs: requestedRetryAfterMs });
+            failure = normalizeProviderFailure(error, { status: observedStatus });
+            failureMessage = failure.message;
+            classification = classifyProviderFailure(failure, requestedRetryAfterMs);
             if (classification.kind === "fatal" || sawContent) {
                 emit({
                     type: "error",
                     reason: "error",
-                    error: toAssistantError(model, error instanceof Error ? error.message : String(error)),
+                    error: providerFailureAssistantMessage(model, failure),
                 });
                 flush();
-                return { result: "fatal", attemptAt, classification };
+                return { result: "fatal", attemptAt, classification, failure };
             }
         }
         if (!classification && requestedRetryAfterMs !== undefined && !sawContent) {
@@ -523,14 +528,14 @@ async function streamProviderAttempt(state, providerId, model, context, options,
             // to another credential / fallback provider. Nothing is emitted into the stream here:
             // the buffered (pre-content) events are discarded exactly like the rate-limit path, so
             // the rotation target streams a clean turn.
-            return { result: "transient_exhausted", attemptAt, classification, attempts: attempt, failureMessage };
+            return { result: "transient_exhausted", attemptAt, classification, attempts: attempt, failure, failureMessage };
         }
         logRotation(`${credential ? `${getProviderLabel(providerId)} subscription "${credential.label}"` : providerId} hit a transient error (${classification.reason}, status=${classification.status ?? "n/a"}); retrying same route in ${backoffMs}ms (attempt ${attempt + 1})`);
         await activeSleep(backoffMs);
     }
 }
-function emitTerminalError(stream, model, message) {
-    stream.push({ type: "error", reason: "error", error: toAssistantError(model, message) });
+function emitTerminalError(stream, model, message, failure) {
+    stream.push({ type: "error", reason: "error", error: failure === undefined ? toAssistantError(model, message) : providerFailureAssistantMessage(model, failure) });
     stream.end();
 }
 function enqueueFreshRouteCandidates(candidates, queuedRouteIds, knownRoutes, seenReadyLabels, cooldownAttempts, localCoolingDeadlines) {
@@ -710,7 +715,7 @@ async function runManagedProvider(state, model, context, options, stream) {
             return;
         }
         const detail = lastTransientFailure.failureMessage ?? lastTransientFailure.classification?.reason ?? 'unknown connection failure';
-        emitTerminalError(stream, terminalModel, `Provider routes failed after transient retries: ${detail} (tried ${tried})`);
+        emitTerminalError(stream, terminalModel, `Provider routes failed after transient retries: ${detail} (tried ${tried})`, lastTransientFailure.failure);
         return;
     }
     state.transportOnlyExhaustedPasses = 0;

package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts

@@ -44,6 +44,7 @@ import { envModelExact, envNodeId } from '../../../shared/env.js';
 import { ManagedProviderCoolingError } from '../../../core/runtime/managed-provider-cooling.js';
 import { recordTransportPoisonVerdict } from '../../../core/runtime/busy.js';
 import { recognizeProviderError, type ProviderErrorSignal } from '../../../core/provider-error-recognition.js';
+import { normalizeProviderFailure, providerFailureDiagnostic, type NormalizedProviderFailure } from '../../../core/provider-failure.js';
 
 // A response without Retry-After proves only that THIS attempt was throttled; it
 // does not authorize crouter to invent a five-minute provider-wide outage. Keep
@@ -111,6 +112,7 @@ type AttemptResult = {
   attemptAt: number;
   attempts?: number;
   classification?: ProviderErrorClassification;
+  failure?: NormalizedProviderFailure;
   failureMessage?: string;
 };
 
@@ -153,32 +155,34 @@ function classifyProviderSignal(signal: ProviderErrorSignal): ProviderErrorClass
   return { kind: 'fatal', status: signal.status, reason: 'unclassified', hadRetryAfterHeader };
 }
 
-function classifyProviderError(error: unknown, observed: { status?: number; retryAfterMs?: number }): ProviderErrorClassification {
-  const value = error as { status?: unknown; errorMessage?: unknown; message?: unknown } | undefined;
-  const message = [value?.errorMessage, value?.message, error instanceof Error ? error.message : '']
+function classifyProviderFailure(failure: NormalizedProviderFailure, retryAfterMs: number | undefined): ProviderErrorClassification {
+  const nested = failure.causes ?? [];
+  const code = failure.code ?? nested.find((cause) => cause.code !== undefined)?.code;
+  const errno = failure.errno ?? nested.find((cause) => cause.errno !== undefined)?.errno;
+  const message = [failure.message, failure.body, ...nested.flatMap((cause) => [cause.message, cause.body])]
     .filter((part): part is string => typeof part === 'string' && part.length > 0)
     .join(' ');
   return classifyProviderSignal(recognizeProviderError({
-    status: observed.status ?? value?.status,
-    retryAfterMs: observed.retryAfterMs,
+    status: failure.status,
+    retryAfterMs,
+    code,
+    errno,
     message,
   }));
 }
 
+function classifyProviderError(error: unknown, observed: { status?: number; retryAfterMs?: number }): ProviderErrorClassification {
+  return classifyProviderFailure(normalizeProviderFailure(error, { status: observed.status }), observed.retryAfterMs);
+}
+
 /** Test seam for the policy mapping over the shared recognition signal. */
 export function __classifyProviderErrorForTest(error: unknown, observed: { status?: number; retryAfterMs?: number } = {}): ProviderErrorClassification {
   return classifyProviderError(error, observed);
 }
 
-function providerErrorMessage(error: unknown): string | undefined {
-  const value = error as { errorMessage?: unknown; message?: unknown } | undefined;
-  const message = typeof value?.errorMessage === "string" && value.errorMessage.trim()
-    ? value.errorMessage.trim()
-    : typeof value?.message === "string" && value.message.trim()
-      ? value.message.trim()
-      : error instanceof Error && error.message.trim()
-        ? error.message.trim()
-        : undefined;
+function providerFailureAssistantMessage(model: Model<any>, failure: NormalizedProviderFailure): AssistantMessage {
+  const message = toAssistantError(model, failure.message);
+  message.diagnostics = [providerFailureDiagnostic(failure)];
   return message;
 }
 
@@ -599,6 +603,7 @@ async function streamProviderAttempt(
     let sawContent = false;
     let committed = false;
     let classification: ProviderErrorClassification | undefined;
+    let failure: NormalizedProviderFailure | undefined;
     let failureMessage: string | undefined;
 
     // Events are buffered ONLY until this attempt commits to the provider -- i.e. until the
@@ -625,12 +630,13 @@ async function streamProviderAttempt(
             flush();
             return { result: "fatal", attemptAt };
           }
-          failureMessage = providerErrorMessage(event.error);
-          classification = classifyProviderError(event.error, { status: observedStatus, retryAfterMs: requestedRetryAfterMs });
+          failure = normalizeProviderFailure(event.error, { status: observedStatus });
+          failureMessage = failure.message;
+          classification = classifyProviderFailure(failure, requestedRetryAfterMs);
           if (classification.kind === "fatal" || sawContent) {
-            emit(event);
+            emit({ ...event, error: providerFailureAssistantMessage(model, failure) });
             flush();
-            return { result: "fatal", attemptAt, classification };
+            return { result: "fatal", attemptAt, classification, failure };
           }
           break;
         }
@@ -650,16 +656,17 @@ async function streamProviderAttempt(
         flush();
         return { result: "fatal", attemptAt };
       }
-      failureMessage = providerErrorMessage(error);
-      classification = classifyProviderError(error, { status: observedStatus, retryAfterMs: requestedRetryAfterMs });
+      failure = normalizeProviderFailure(error, { status: observedStatus });
+      failureMessage = failure.message;
+      classification = classifyProviderFailure(failure, requestedRetryAfterMs);
       if (classification.kind === "fatal" || sawContent) {
         emit({
           type: "error",
           reason: "error",
-          error: toAssistantError(model, error instanceof Error ? error.message : String(error)),
+          error: providerFailureAssistantMessage(model, failure),
         });
         flush();
-        return { result: "fatal", attemptAt, classification };
+        return { result: "fatal", attemptAt, classification, failure };
       }
     }
 
@@ -687,7 +694,7 @@ async function streamProviderAttempt(
       // to another credential / fallback provider. Nothing is emitted into the stream here:
       // the buffered (pre-content) events are discarded exactly like the rate-limit path, so
       // the rotation target streams a clean turn.
-      return { result: "transient_exhausted", attemptAt, classification, attempts: attempt, failureMessage };
+      return { result: "transient_exhausted", attemptAt, classification, attempts: attempt, failure, failureMessage };
     }
     logRotation(
       `${credential ? `${getProviderLabel(providerId as ManagedProviderId)} subscription "${credential.label}"` : providerId} hit a transient error (${classification.reason}, status=${classification.status ?? "n/a"}); retrying same route in ${backoffMs}ms (attempt ${attempt + 1})`,
@@ -696,8 +703,8 @@ async function streamProviderAttempt(
   }
 }
 
-function emitTerminalError(stream: AssistantMessageEventStream, model: Model<any>, message: string): void {
-  stream.push({ type: "error", reason: "error", error: toAssistantError(model, message) });
+function emitTerminalError(stream: AssistantMessageEventStream, model: Model<any>, message: string, failure?: NormalizedProviderFailure): void {
+  stream.push({ type: "error", reason: "error", error: failure === undefined ? toAssistantError(model, message) : providerFailureAssistantMessage(model, failure) });
   stream.end();
 }
 
@@ -895,7 +902,7 @@ async function runManagedProvider(
       return;
     }
     const detail = lastTransientFailure.failureMessage ?? lastTransientFailure.classification?.reason ?? 'unknown connection failure';
-    emitTerminalError(stream, terminalModel, `Provider routes failed after transient retries: ${detail} (tried ${tried})`);
+    emitTerminalError(stream, terminalModel, `Provider routes failed after transient retries: ${detail} (tried ${tried})`, lastTransientFailure.failure);
     return;
   }
   state.transportOnlyExhaustedPasses = 0;