@crewhaus/recovery-engine 0.2.3 → 0.3.0

package/dist/index.d.ts

@@ -5,6 +5,10 @@
  * the chosen action (sleeping, recompacting, injecting messages).
  *
  * Taxonomy:
+ *   billing (402 / credit balance / insufficient_quota)
+ *                        → halt      (terminal — retrying an empty account is futile)
+ *   auth (401/403)       → halt      (terminal — retrying bad credentials is futile)
+ *   rate_limit (429)     → retry     (honors Retry-After; exhaustion → halt, exit 32)
  *   prompt_too_long      → compact   (max once per turn → fail)
  *   max_output_tokens    → continue  (max 3 per turn   → fail)
  *   overloaded / 5xx     → retry     (exponential backoff, max 5 → fail)
@@ -23,7 +27,7 @@
  * References: claude-code/query.ts recovery branches; agent-framework
  * _runner.py retry; AI-Harness-Systems §recovery.
  */
-import { CrewhausError } from "@crewhaus/errors";
+import { CrewhausError, type FailureClass, type FailureReport } from "@crewhaus/errors";
 export type RecoveryAction = {
     readonly kind: "compact";
 } | {
@@ -40,8 +44,11 @@ export type RecoveryAction = {
 } | {
     readonly kind: "fail";
     readonly reason: string;
+} | {
+    readonly kind: "halt";
+    readonly report: FailureReport;
 };
-export type RecoveryErrorClass = "prompt_too_long" | "max_output_tokens" | "overloaded_or_5xx" | "invalid_request" | "user_aborted" | "unknown";
+export type RecoveryErrorClass = "billing" | "auth" | "rate_limit" | "prompt_too_long" | "max_output_tokens" | "overloaded_or_5xx" | "invalid_request" | "user_aborted" | "unknown";
 export type RecoveryState = {
     /** Number of `retry` actions already chosen in this turn. */
     readonly retryCount: number;
@@ -64,6 +71,17 @@ export declare class RecoveryEngineError extends CrewhausError {
  * 4 s, ..., capped at 30 s, plus 0–250 ms of jitter.
  */
 export declare function backoffMs(attempt: number, jitterFn?: () => number): number;
+/**
+ * v0.3.0 Goal 6 — extract a provider Retry-After delay from an error object,
+ * in milliseconds, or undefined when absent. Duck-typed against the shapes
+ * the SDKs / adapters surface:
+ *   · `retryAfterMs` (number, milliseconds) — adapter-normalized field;
+ *   · `retryAfter`   (number, seconds)      — SDK convenience field;
+ *   · `headers` — a Headers instance or plain object carrying `retry-after`
+ *     as delta-seconds or an HTTP-date.
+ * Results are clamped into [0, RETRY_AFTER_CAP_MS].
+ */
+export declare function retryAfterMs(error: unknown, now?: () => number): number | undefined;
 /**
  * Classify an unknown error value into a recovery taxonomy bucket. Duck-typed
  * against the Anthropic SDK error shape: `.status`, `.error.type`, `.message`,
@@ -74,6 +92,60 @@ export declare function backoffMs(attempt: number, jitterFn?: () => number): num
  * that shape after seeing `stop_reason: "max_tokens"`).
  */
 export declare function classify(error: unknown): RecoveryErrorClass;
+/**
+ * v0.3.0 Goal 6 — built-in named failure classes. Report metadata (title /
+ * remediation / exit code) for the terminal classes the engine can halt
+ * with. Consulted AFTER user `failure_taxonomy` entries (user overrides
+ * win) and BEFORE the generic classify buckets. `mcp_boot_failure` and
+ * `crewhaus_budget` are not produced by `classify()` — they exist so the
+ * runtime's MCP-boot and budget-cap paths build their reports from the
+ * same table (wired in the follow-up runtime PR).
+ */
+export type BuiltinFailureClass = {
+    readonly class: FailureClass;
+    readonly title: string;
+    readonly remediation: string;
+    readonly exitCode: number;
+    readonly docsUrl?: string;
+};
+export declare const BUILTIN_FAILURE_CLASSES: {
+    readonly billing_exhausted: {
+        readonly class: "billing";
+        readonly title: "provider account out of funding";
+        readonly remediation: "add credits to your provider account, then rerun.";
+        readonly exitCode: 31;
+    };
+    readonly auth_invalid: {
+        readonly class: "auth";
+        readonly title: "provider rejected the credentials";
+        readonly remediation: "check the provider API key env var (see .env.example), then rerun.";
+        readonly exitCode: 30;
+    };
+    readonly rate_limited: {
+        readonly class: "rate_limit";
+        readonly title: "provider rate limit still exceeded after retries";
+        readonly remediation: "wait for the rate-limit window to reset (or lower the request rate), then rerun.";
+        readonly exitCode: 32;
+    };
+    readonly mcp_boot_failure: {
+        readonly class: "mcp_boot";
+        readonly title: "an MCP server failed to boot";
+        readonly remediation: "check the MCP server command, URL, and credentials in the spec, then rerun.";
+        readonly exitCode: 40;
+    };
+    readonly crewhaus_budget: {
+        readonly class: "crewhaus_budget";
+        readonly title: "run stopped by the configured budget cap";
+        readonly remediation: "raise the spec's budget cap (or rerun with a higher cap).";
+        readonly exitCode: 33;
+    };
+};
+/**
+ * Build a `FailureReport` for a built-in class from the offending error.
+ * AdapterError wrappers carry `providerId`, which drives the "…said:"
+ * attribution in `detail` and the provider-specific remediation line.
+ */
+export declare function buildFailureReport(key: keyof typeof BUILTIN_FAILURE_CLASSES, error: unknown): FailureReport;
 /**
  * Section 55 (Track A) — a single entry from a spec's `failure_taxonomy`,
  * carried verbatim from the IR. The recovery engine consults the taxonomy
@@ -117,4 +189,5 @@ export declare const BUDGETS: {
     readonly BASE_BACKOFF_MS: 1000;
     readonly MAX_BACKOFF_MS: 30000;
     readonly MAX_JITTER_MS: 250;
+    readonly RETRY_AFTER_CAP_MS: 60000;
 };

package/dist/index.js

@@ -5,6 +5,10 @@
  * the chosen action (sleeping, recompacting, injecting messages).
  *
  * Taxonomy:
+ *   billing (402 / credit balance / insufficient_quota)
+ *                        → halt      (terminal — retrying an empty account is futile)
+ *   auth (401/403)       → halt      (terminal — retrying bad credentials is futile)
+ *   rate_limit (429)     → retry     (honors Retry-After; exhaustion → halt, exit 32)
  *   prompt_too_long      → compact   (max once per turn → fail)
  *   max_output_tokens    → continue  (max 3 per turn   → fail)
  *   overloaded / 5xx     → retry     (exponential backoff, max 5 → fail)
@@ -23,7 +27,7 @@
  * References: claude-code/query.ts recovery branches; agent-framework
  * _runner.py retry; AI-Harness-Systems §recovery.
  */
-import { CrewhausError } from "@crewhaus/errors";
+import { CrewhausError, EXIT_CODES, isRunFailedError, } from "@crewhaus/errors";
 export const initialRecoveryState = {
     retryCount: 0,
     compactCount: 0,
@@ -59,6 +63,51 @@ export function backoffMs(attempt, jitterFn = Math.random) {
     const jitter = Math.min(Math.floor(jitterFn() * (MAX_JITTER_MS + 1)), MAX_JITTER_MS);
     return exp + jitter;
 }
+// Honor a provider Retry-After up to this cap so a pathological header
+// (e.g. an HTTP-date hours away) can't stall the run indefinitely; past the
+// cap the normal capped exponential backoff applies.
+const RETRY_AFTER_CAP_MS = 60_000;
+/**
+ * v0.3.0 Goal 6 — extract a provider Retry-After delay from an error object,
+ * in milliseconds, or undefined when absent. Duck-typed against the shapes
+ * the SDKs / adapters surface:
+ *   · `retryAfterMs` (number, milliseconds) — adapter-normalized field;
+ *   · `retryAfter`   (number, seconds)      — SDK convenience field;
+ *   · `headers` — a Headers instance or plain object carrying `retry-after`
+ *     as delta-seconds or an HTTP-date.
+ * Results are clamped into [0, RETRY_AFTER_CAP_MS].
+ */
+export function retryAfterMs(error, now = Date.now) {
+    if (error === null || typeof error !== "object")
+        return undefined;
+    const errObj = error;
+    const clamp = (ms) => Number.isFinite(ms) ? Math.min(Math.max(Math.round(ms), 0), RETRY_AFTER_CAP_MS) : undefined;
+    if (typeof errObj.retryAfterMs === "number")
+        return clamp(errObj.retryAfterMs);
+    if (typeof errObj.retryAfter === "number")
+        return clamp(errObj.retryAfter * 1_000);
+    const headers = errObj.headers;
+    let raw;
+    if (headers !== null && typeof headers === "object") {
+        const maybeGet = headers.get;
+        if (typeof maybeGet === "function") {
+            raw = maybeGet.call(headers, "retry-after");
+        }
+        else {
+            const record = headers;
+            raw = record["retry-after"] ?? record["Retry-After"];
+        }
+    }
+    if (typeof raw !== "string" || raw.length === 0)
+        return undefined;
+    // Delta-seconds form ("30") first, then the HTTP-date form.
+    if (/^\d+$/.test(raw.trim()))
+        return clamp(Number(raw.trim()) * 1_000);
+    const at = Date.parse(raw);
+    if (Number.isNaN(at))
+        return undefined;
+    return clamp(at - now());
+}
 /**
  * Classify an unknown error value into a recovery taxonomy bucket. Duck-typed
  * against the Anthropic SDK error shape: `.status`, `.error.type`, `.message`,
@@ -79,8 +128,44 @@ export function classify(error) {
         }
     }
     const innerType = typeof errObj.error?.type === "string" ? errObj.error.type : undefined;
+    const innerMessage = typeof errObj.error?.message === "string" ? errObj.error.message : "";
     const message = typeof errObj.message === "string" ? errObj.message : "";
     const status = typeof errObj.status === "number" ? errObj.status : undefined;
+    // BOTH code slots are consulted independently (PR 2): a raw SDK error
+    // carries the provider code top-level, but adapter WRAPPERS keep
+    // CrewhausError's ErrorCode there ("adapter") and surface the provider
+    // code on the copied error envelope — a top-level string must not shadow
+    // the envelope's discriminator.
+    const code = typeof errObj.code === "string" ? errObj.code : undefined;
+    const innerCode = typeof errObj.error?.code === "string" ? errObj.error.code : undefined;
+    // v0.3.0 Goal 6 — billing / auth / rate_limit come BEFORE the pre-existing
+    // buckets so an out-of-funds 400 is not misrouted through a tombstone and
+    // an insufficient_quota 429 is not burned through backoff retries.
+    //
+    // billing — the provider account itself is out of funding:
+    //   · HTTP 402 anywhere;
+    //   · Anthropic: 400 invalid_request_error + "credit balance … too low";
+    //   · OpenAI: 429 + code "insufficient_quota" (distinct from rate limits);
+    //   · Bedrock: ServiceQuotaExceededException by name (a hard account
+    //     quota — retrying won't help; raise the quota instead).
+    if (status === 402)
+        return "billing";
+    if (code === "insufficient_quota" || innerCode === "insufficient_quota")
+        return "billing";
+    if (status === 400 && /credit balance/i.test(innerMessage.length > 0 ? innerMessage : message)) {
+        return "billing";
+    }
+    if (typeof errObj.name === "string" && /ServiceQuotaExceeded/.test(errObj.name)) {
+        return "billing";
+    }
+    // auth — runtime 401/403 (distinct from boot-time ProviderAuthError, which
+    // fires before any request when credentials are entirely missing).
+    if (status === 401 || status === 403)
+        return "auth";
+    // rate_limit — a genuine 429 (NOT insufficient_quota, handled above).
+    // Retried with Retry-After honored; budget exhaustion halts with exit 32.
+    if (status === 429 || innerType === "rate_limit_error")
+        return "rate_limit";
     if (innerType === "max_output_tokens")
         return "max_output_tokens";
     // Anthropic returns "Prompt is too long" inside invalid_request_error responses;
@@ -98,6 +183,100 @@ export function classify(error) {
         return "invalid_request";
     return "unknown";
 }
+export const BUILTIN_FAILURE_CLASSES = {
+    billing_exhausted: {
+        class: "billing",
+        title: "provider account out of funding",
+        remediation: "add credits to your provider account, then rerun.",
+        exitCode: EXIT_CODES.billing,
+    },
+    auth_invalid: {
+        class: "auth",
+        title: "provider rejected the credentials",
+        remediation: "check the provider API key env var (see .env.example), then rerun.",
+        exitCode: EXIT_CODES.auth,
+    },
+    rate_limited: {
+        class: "rate_limit",
+        title: "provider rate limit still exceeded after retries",
+        remediation: "wait for the rate-limit window to reset (or lower the request rate), then rerun.",
+        exitCode: EXIT_CODES.rate_limit,
+    },
+    mcp_boot_failure: {
+        class: "mcp_boot",
+        title: "an MCP server failed to boot",
+        remediation: "check the MCP server command, URL, and credentials in the spec, then rerun.",
+        exitCode: EXIT_CODES.tool,
+    },
+    crewhaus_budget: {
+        class: "crewhaus_budget",
+        title: "run stopped by the configured budget cap",
+        remediation: "raise the spec's budget cap (or rerun with a higher cap).",
+        exitCode: EXIT_CODES.crewhaus_budget,
+    },
+};
+/** Provider display names for `detail` attribution ("Anthropic said: …"). */
+const PROVIDER_DISPLAY_NAMES = {
+    anthropic: "Anthropic",
+    openai: "OpenAI",
+    gemini: "Gemini",
+    bedrock: "Bedrock",
+};
+/** Per-provider remediation overrides for the billing / auth classes. */
+const PROVIDER_REMEDIATIONS = {
+    anthropic: {
+        billing: "add credits at https://console.anthropic.com/settings/billing, then rerun.",
+        auth: "check ANTHROPIC_AUTH_TOKEN / ANTHROPIC_API_KEY (see .env.example), then rerun.",
+    },
+    openai: {
+        billing: "add credits at https://platform.openai.com/settings/organization/billing, then rerun.",
+        auth: "check OPENAI_API_KEY (see .env.example), then rerun.",
+    },
+    gemini: {
+        billing: "check your plan and billing at https://aistudio.google.com/, then rerun.",
+        auth: "check GEMINI_API_KEY / GOOGLE_API_KEY (see .env.example), then rerun.",
+    },
+    bedrock: {
+        billing: "request a service-quota increase in the AWS console, then rerun.",
+        auth: "check your AWS credentials and Bedrock model access, then rerun.",
+    },
+};
+/** Best raw provider text: the API envelope message, else the wrapper message. */
+function rawProviderText(error) {
+    const errObj = error;
+    const inner = typeof errObj?.error?.message === "string" ? errObj.error.message : "";
+    if (inner.length > 0)
+        return inner;
+    const outer = typeof errObj?.message === "string" ? errObj.message : "";
+    if (outer.length > 0)
+        return outer;
+    return String(error);
+}
+/**
+ * Build a `FailureReport` for a built-in class from the offending error.
+ * AdapterError wrappers carry `providerId`, which drives the "…said:"
+ * attribution in `detail` and the provider-specific remediation line.
+ */
+export function buildFailureReport(key, error) {
+    const builtin = BUILTIN_FAILURE_CLASSES[key];
+    const providerId = typeof error?.providerId === "string"
+        ? error.providerId
+        : undefined;
+    const raw = rawProviderText(error);
+    const detail = providerId !== undefined
+        ? `${PROVIDER_DISPLAY_NAMES[providerId] ?? providerId} said: ${JSON.stringify(raw)}`
+        : raw;
+    const remediation = (providerId !== undefined ? PROVIDER_REMEDIATIONS[providerId]?.[builtin.class] : undefined) ??
+        builtin.remediation;
+    return {
+        class: builtin.class,
+        title: builtin.title,
+        detail,
+        remediation,
+        exitCode: builtin.exitCode,
+        ...(builtin.docsUrl !== undefined ? { docsUrl: builtin.docsUrl } : {}),
+    };
+}
 /**
  * Match an unknown error against a taxonomy entry's `pattern`. The pattern
  * is either a `/regex/flags` literal or a case-insensitive substring of
@@ -142,16 +321,44 @@ export function matchNamedFailure(error, taxonomy) {
  * built-in classes). Unmatched errors fall through to the built-in flow.
  */
 export function recover(error, state, taxonomy) {
+    // v0.3.0 §7.1 — an error that ALREADY carries a classified terminal report
+    // (a child run's RunFailedError escalated through the sub-agent spawner)
+    // halts with that report verbatim: no re-classification, no retry ladder,
+    // and the user taxonomy cannot downgrade an already-terminal verdict —
+    // "a billing failure anywhere ends the run with the billing message".
+    if (isRunFailedError(error)) {
+        return { kind: "halt", report: error.report };
+    }
     if (taxonomy !== undefined && taxonomy.length > 0) {
         const named = matchNamedFailure(error, taxonomy);
         if (named !== undefined) {
-            return recoverNamed(named, state);
+            return recoverNamed(named, state, error);
         }
     }
     const klass = classify(error);
     const message = error?.message;
     const reasonStr = typeof message === "string" && message.length > 0 ? message : klass;
     switch (klass) {
+        // v0.3.0 Goal 6 — terminal classes halt immediately: no tombstone
+        // detour, no futile backoff retries against an empty account or a
+        // rejected key.
+        case "billing":
+            return { kind: "halt", report: buildFailureReport("billing_exhausted", error) };
+        case "auth":
+            return { kind: "halt", report: buildFailureReport("auth_invalid", error) };
+        case "rate_limit": {
+            // A genuine rate limit is transient: keep retrying, but honor the
+            // provider's Retry-After for the delay when one is present. Budget
+            // exhaustion is a classified halt (exit 32), not a generic fail.
+            if (state.retryCount >= MAX_RETRIES) {
+                return { kind: "halt", report: buildFailureReport("rate_limited", error) };
+            }
+            return {
+                kind: "retry",
+                delayMs: retryAfterMs(error) ?? backoffMs(state.retryCount),
+                attempt: state.retryCount + 1,
+            };
+        }
         case "prompt_too_long":
             if (state.compactCount >= MAX_COMPACTS) {
                 return { kind: "fail", reason: `compact budget exhausted: ${reasonStr}` };
@@ -185,15 +392,35 @@ export function recover(error, state, taxonomy) {
 /**
  * Convert a matched named-failure entry into the corresponding
  * RecoveryAction, respecting the same per-turn budgets as built-in
- * classes. Budget exhaustion always returns `fail` so the user can't
- * declare an infinite retry loop via taxonomy.
+ * classes. Budget exhaustion always returns a terminal action so the
+ * user can't declare an infinite retry loop via taxonomy.
+ *
+ * v0.3.0 Goal 6 — the entry's `hint` field (declared since §55, never
+ * consumed until now) finally reaches the user: when a matched entry
+ * resolves terminally (declared `fail` or budget exhaustion) AND carries a
+ * `hint`, the result is a `halt` whose report carries the hint as the
+ * remediation line. Entries without a hint keep the pre-0.3.0 `fail`.
  */
-function recoverNamed(named, state) {
+function recoverNamed(named, state, error) {
     const reason = `failure_taxonomy: ${named.class}`;
+    const terminal = (fullReason) => {
+        if (named.hint === undefined)
+            return { kind: "fail", reason: fullReason };
+        return {
+            kind: "halt",
+            report: {
+                class: "unknown",
+                title: fullReason,
+                detail: error === undefined ? "" : rawProviderText(error),
+                remediation: named.hint,
+                exitCode: EXIT_CODES.generic,
+            },
+        };
+    };
     switch (named.recovery) {
         case "retry":
             if (state.retryCount >= MAX_RETRIES) {
-                return { kind: "fail", reason: `retry budget exhausted: ${reason}` };
+                return terminal(`retry budget exhausted: ${reason}`);
             }
             return {
                 kind: "retry",
@@ -202,26 +429,26 @@ function recoverNamed(named, state) {
             };
         case "compact":
             if (state.compactCount >= MAX_COMPACTS) {
-                return { kind: "fail", reason: `compact budget exhausted: ${reason}` };
+                return terminal(`compact budget exhausted: ${reason}`);
             }
             return { kind: "compact" };
         case "continue":
             if (state.continueCount >= MAX_CONTINUES) {
-                return { kind: "fail", reason: `continue budget exhausted: ${reason}` };
+                return terminal(`continue budget exhausted: ${reason}`);
             }
             return { kind: "continue" };
         case "tombstone":
             if (state.tombstoneCount >= MAX_TOMBSTONES) {
-                return { kind: "fail", reason: `tombstone budget exhausted: ${reason}` };
+                return terminal(`tombstone budget exhausted: ${reason}`);
             }
             return { kind: "tombstone" };
         case "switch-model":
             if (state.switchModelCount >= MAX_SWITCH_MODELS) {
-                return { kind: "fail", reason: `switch-model budget exhausted: ${reason}` };
+                return terminal(`switch-model budget exhausted: ${reason}`);
             }
             return { kind: "switch-model" };
         case "fail":
-            return { kind: "fail", reason };
+            return terminal(reason);
     }
 }
 /**
@@ -241,6 +468,7 @@ export function advanceState(state, action) {
         case "switch-model":
             return { ...state, switchModelCount: state.switchModelCount + 1 };
         case "fail":
+        case "halt":
             return state;
     }
 }
@@ -254,4 +482,5 @@ export const BUDGETS = {
     BASE_BACKOFF_MS,
     MAX_BACKOFF_MS,
     MAX_JITTER_MS,
+    RETRY_AFTER_CAP_MS,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crewhaus/recovery-engine",
-  "version": "0.2.3",
+  "version": "0.3.0",
   "type": "module",
   "description": "Pure recovery taxonomy: classify Anthropic API errors into a RecoveryAction (compact/retry/continue/tombstone/fail)",
   "main": "dist/index.js",
@@ -15,7 +15,7 @@
     "test": "bun test src"
   },
   "dependencies": {
-    "@crewhaus/errors": "0.2.3"
+    "@crewhaus/errors": "0.3.0"
   },
   "license": "Apache-2.0",
   "author": {