cartethyia 2.7.3 → 2.7.5

package/src/network/pool/selector.ts

@@ -36,8 +36,7 @@ import {
   type ProxyCooldownEntry,
 } from "./pool-state";
 
-export type { PoolRotation, PoolUsageSnapshot, ProxyCooldownEntry };
-export { parseCooldownEntry };
+export type { PoolRotation, ProxyCooldownEntry };
 
 // Pool selector
 /**
@@ -51,7 +50,7 @@ export { parseCooldownEntry };
  */
 export const PROVIDER_BLOCK_MS = 10 * 365 * 24 * 60 * 60 * 1000;
 
-export const DEFAULT_PROXY_CONCURRENCY = 10;
+const DEFAULT_PROXY_CONCURRENCY = 10;
 const DEFAULT_PROXY_WEIGHT = 100;
 const MAX_PROXY_WEIGHT = 1000;
 // Crash-recovery bound, not a lease duration: normal releases DECR/DEL the
@@ -63,7 +62,7 @@ const MAX_PROXY_WEIGHT = 1000;
 function poolInflightTtlSeconds(): number {
   return resolveInflightTtlSeconds();
 }
-export function effectiveConcurrency(limit: number): number {
+function effectiveConcurrency(limit: number): number {
   if (!Number.isInteger(limit) || limit < 1) return 0;
   return limit;
 }
@@ -100,7 +99,7 @@ function decrementLocal(counter: Map<string, number>, key: string): void {
   else counter.set(key, value - 1);
 }
 
-export type PoolSelectionFailureReason =
+type PoolSelectionFailureReason =
   | "at_capacity"
   | "provider_at_capacity"
   | "cooldown"
@@ -109,7 +108,7 @@ export type PoolSelectionFailureReason =
   | "coordination_unavailable"
   | "no_active_pool";
 
-export interface PoolCapacitySnapshot {
+interface PoolCapacitySnapshot {
   readonly poolId: string;
   readonly maxInflight: number;
   readonly currentInflight: number;
@@ -206,67 +205,67 @@ export class NetworkPoolSelector {
   }
 
   /**
-   * Raise this provider's share of a pool, enforcing both the pool ceiling and
-   * the provider ceiling. The pool half is still checked here so a caller that
-   * already admitted the pool through `acquire` does not double-count it — use
-   * this instead of `acquire` whenever a provider ceiling applies.
+   * One local slot acquisition: checks every map's ceiling, bumps each counter,
+   * and returns a release that decrements all of them exactly once. Shared by
+   * `acquire` (single pool counter) and `acquireProvider` (pool + provider
+   * share) so the release closure and the local bookkeeping live in one place.
    */
-  acquireProvider(
-    poolId: string,
-    providerId: string,
-    poolCapacity: number,
-    providerCapacity: number,
+  private acquireLocalSlot(
+    increments: ReadonlyArray<{
+      readonly map: Map<string, number>;
+      readonly key: string;
+      readonly capacity: number;
+    }>,
   ): { acquired: boolean; release: () => void } {
-    const current = this.getInflight(poolId);
-    if (current >= poolCapacity) return { acquired: false, release: () => {} };
-    const key = providerInflightStateKey(poolId, providerId);
-    const providerCurrent = this.state.providerInflight.get(key) ?? 0;
-    if (providerCurrent >= providerCapacity) return { acquired: false, release: () => {} };
-
-    enforceInflightLimit(this.state.inflight);
-    enforceInflightLimit(this.state.providerInflight);
-    this.state.inflight.set(poolId, current + 1);
-    this.state.providerInflight.set(key, providerCurrent + 1);
+    for (const increment of increments) {
+      const current = increment.map.get(increment.key) ?? 0;
+      if (current >= increment.capacity) return { acquired: false, release: () => {} };
+    }
+    for (const increment of increments) {
+      enforceInflightLimit(increment.map);
+      increment.map.set(increment.key, (increment.map.get(increment.key) ?? 0) + 1);
+    }
     this.notifyPoolUsage();
 
     let released = false;
     const release = () => {
       if (released) return;
       released = true;
-      decrementLocal(this.state.inflight, poolId);
-      decrementLocal(this.state.providerInflight, key);
+      for (const increment of increments) decrementLocal(increment.map, increment.key);
       this.notifyPoolUsage();
     };
     return { acquired: true, release };
   }
 
+  /**
+   * Raise this provider's share of a pool, enforcing both the pool ceiling and
+   * the provider ceiling. The pool half is still checked here so a caller that
+   * already admitted the pool through `acquire` does not double-count it — use
+   * this instead of `acquire` whenever a provider ceiling applies.
+   */
+  acquireProvider(
+    poolId: string,
+    providerId: string,
+    poolCapacity: number,
+    providerCapacity: number,
+  ): { acquired: boolean; release: () => void } {
+    return this.acquireLocalSlot([
+      { map: this.state.inflight, key: poolId, capacity: poolCapacity },
+      {
+        map: this.state.providerInflight,
+        key: providerInflightStateKey(poolId, providerId),
+        capacity: providerCapacity,
+      },
+    ]);
+  }
+
   acquire(
     poolId: string,
     maxInflight = DEFAULT_PROXY_CONCURRENCY,
   ): { acquired: boolean; release: () => void } {
-    const current = this.getInflight(poolId);
-    if (current >= maxInflight) {
-      return { acquired: false, release: () => {} };
-    }
-
-    enforceInflightLimit(this.state.inflight);
-    this.state.inflight.set(poolId, current + 1);
-    this.notifyPoolUsage();
-    let released = false;
-
-    const release = () => {
-      if (released) return;
-      released = true;
-      const val = this.state.inflight.get(poolId) ?? 1;
-      if (val <= 1) {
-        this.state.inflight.delete(poolId);
-      } else {
-        this.state.inflight.set(poolId, val - 1);
-      }
-      this.notifyPoolUsage();
-    };
-
-    return { acquired: true, release };
+    return this.acquireLocalSlot([
+      { map: this.state.inflight, key: poolId, capacity: maxInflight },
+    ]);
   }
 
   private cooldownKey(poolId: string, providerId: string): string {
@@ -577,6 +576,98 @@ export class NetworkPoolSelector {
     await this.redis.del(indexKey);
   }
 
+  /**
+   * Tries one local candidate: bump the local counters (pool, and provider
+   * share when a ceiling is set) and return its release. `undefined` when the
+   * candidate is full or the provider share is exhausted.
+   */
+  private localAcquire(
+    candidate: { readonly poolId: string; readonly capacity: number },
+    providerId: string,
+    providerCapacity: number | undefined,
+  ): { poolId: string; release: () => void } | undefined {
+    const slot =
+      providerCapacity === undefined
+        ? this.acquire(candidate.poolId, candidate.capacity)
+        : this.acquireProvider(candidate.poolId, providerId, candidate.capacity, providerCapacity);
+    return slot.acquired ? { poolId: candidate.poolId, release: slot.release } : undefined;
+  }
+
+  /**
+   * One atomic Redis admit (the compare-and-increment script) for a candidate:
+   * bumps both the remote and local counters on success and returns a release
+   * that decrements both. `undefined` when the pool or provider share is full
+   * (1 admitted; 0 pool full; 2 provider full — either full result fails over
+   * to the next pool, which is the whole point of the ceiling).
+   */
+  private async redisAdmit(
+    redis: RedisClient,
+    candidate: { readonly poolId: string; readonly capacity: number },
+    providerId: string,
+    providerCapacity: number | undefined,
+  ): Promise<{ poolId: string; release: () => void } | undefined> {
+    const providerKey =
+      providerCapacity === undefined
+        ? undefined
+        : proxyProviderInflightKey(candidate.poolId, providerId);
+    const result =
+      providerKey === undefined
+        ? await redisEvalNumber(
+            redis,
+            POOL_ADMIT_SCRIPT,
+            1,
+            proxyInflightKey(candidate.poolId),
+            String(candidate.capacity),
+            String(poolInflightTtlSeconds()),
+          )
+        : await redisEvalNumber(
+            redis,
+            POOL_PROVIDER_ADMIT_SCRIPT,
+            2,
+            proxyInflightKey(candidate.poolId),
+            providerKey,
+            String(candidate.capacity),
+            String(providerCapacity),
+            String(poolInflightTtlSeconds()),
+          );
+    if (result !== 1) return undefined;
+    enforceInflightLimit(this.state.inflight);
+    this.state.inflight.set(candidate.poolId, this.getInflight(candidate.poolId) + 1);
+    if (providerKey !== undefined) {
+      enforceInflightLimit(this.state.providerInflight);
+      const localProviderKey = providerInflightStateKey(candidate.poolId, providerId);
+      this.state.providerInflight.set(
+        localProviderKey,
+        this.getProviderInflight(candidate.poolId, providerId) + 1,
+      );
+    }
+    this.notifyPoolUsage();
+    let released = false;
+    return {
+      poolId: candidate.poolId,
+      release: () => {
+        if (released) return;
+        released = true;
+        decrementLocal(this.state.inflight, candidate.poolId);
+        if (providerKey !== undefined) {
+          decrementLocal(
+            this.state.providerInflight,
+            providerInflightStateKey(candidate.poolId, providerId),
+          );
+        }
+        this.notifyPoolUsage();
+        const script = providerKey === undefined ? POOL_RELEASE_SCRIPT : POOL_PROVIDER_RELEASE_SCRIPT;
+        const keys =
+          providerKey === undefined
+            ? [proxyInflightKey(candidate.poolId)]
+            : [proxyInflightKey(candidate.poolId), providerKey];
+        void redisEvalNumber(redis, script, keys.length, ...keys).catch((error: unknown) => {
+          log.error("[pool-selector] failed to release distributed slot", error as Error);
+        });
+      },
+    };
+  }
+
   /**
    * Distributed pool admission — the selection algorithm:
    *
@@ -652,17 +743,8 @@ export class NetworkPoolSelector {
         shuffleArray(scored);
         for (const candidate of scored) {
           if (this.getInflight(candidate.poolId) >= candidate.capacity) continue;
-          const slot =
-            providerCapacity === undefined
-              ? this.acquire(candidate.poolId, candidate.capacity)
-              : this.acquireProvider(
-                  candidate.poolId,
-                  providerId,
-                  candidate.capacity,
-                  providerCapacity,
-                );
-          if (!slot.acquired) continue;
-          return { poolId: candidate.poolId, release: slot.release };
+          const slot = this.localAcquire(candidate, providerId, providerCapacity);
+          if (slot) return slot;
         }
         return undefined;
       }
@@ -677,18 +759,10 @@ export class NetworkPoolSelector {
           const candidate = scored[absolute];
           if (!candidate) continue;
           if (this.getInflight(candidate.poolId) >= candidate.capacity) continue;
-          const slot =
-            providerCapacity === undefined
-              ? this.acquire(candidate.poolId, candidate.capacity)
-              : this.acquireProvider(
-                  candidate.poolId,
-                  providerId,
-                  candidate.capacity,
-                  providerCapacity,
-                );
-          if (!slot.acquired) continue;
+          const slot = this.localAcquire(candidate, providerId, providerCapacity);
+          if (!slot) continue;
           advanceRotation(this.state.rotationCursors, rotation.key, scored.length, absolute, rotation.rotateCount);
-          return { poolId: candidate.poolId, release: slot.release };
+          return slot;
         }
         return undefined;
       }
@@ -716,11 +790,7 @@ export class NetworkPoolSelector {
         }
       }
       if (!bestPool) return undefined;
-      const slot =
-        providerCapacity === undefined
-          ? this.acquire(bestPool.poolId, bestPool.capacity)
-          : this.acquireProvider(bestPool.poolId, providerId, bestPool.capacity, providerCapacity);
-      return slot.acquired ? { poolId: bestPool.poolId, release: slot.release } : undefined;
+      return this.localAcquire(bestPool, providerId, providerCapacity);
     }
 
     const redis = this.redis;
@@ -795,34 +865,8 @@ export class NetworkPoolSelector {
       });
     }
     for (const candidate of withCounts) {
-      const providerKey =
-        providerCapacity === undefined
-          ? undefined
-          : proxyProviderInflightKey(candidate.poolId, providerId);
-      const result =
-        providerKey === undefined
-          ? await redisEvalNumber(
-              redis,
-              POOL_ADMIT_SCRIPT,
-              1,
-              proxyInflightKey(candidate.poolId),
-              String(candidate.capacity),
-              String(poolInflightTtlSeconds()),
-            )
-          : await redisEvalNumber(
-              redis,
-              POOL_PROVIDER_ADMIT_SCRIPT,
-              2,
-              proxyInflightKey(candidate.poolId),
-              providerKey,
-              String(candidate.capacity),
-              String(providerCapacity),
-              String(poolInflightTtlSeconds()),
-            );
-      // 1 admitted; 0 pool full; 2 provider full. Either full result fails over
-      // to the next pool, which is the whole point of the ceiling: the request
-      // moves to an egress that still has room for this provider.
-      if (result !== 1) continue;
+      const admitted = await this.redisAdmit(redis, candidate, providerId, providerCapacity);
+      if (admitted === undefined) continue;
       if (rotation && rotation.strategy !== "random") {
         advanceRotation(
           this.state.rotationCursors,
@@ -832,37 +876,7 @@ export class NetworkPoolSelector {
           rotation.rotateCount,
         );
       }
-      enforceInflightLimit(this.state.inflight);
-      this.state.inflight.set(candidate.poolId, this.getInflight(candidate.poolId) + 1);
-      if (providerKey !== undefined) {
-        enforceInflightLimit(this.state.providerInflight);
-        const localProviderKey = providerInflightStateKey(candidate.poolId, providerId);
-        this.state.providerInflight.set(
-          localProviderKey,
-          this.getProviderInflight(candidate.poolId, providerId) + 1,
-        );
-      }
-      this.notifyPoolUsage();
-      let released = false;
-      return {
-        poolId: candidate.poolId,
-        release: () => {
-          if (released) return;
-          released = true;
-          decrementLocal(this.state.inflight, candidate.poolId);
-          if (providerKey !== undefined) {
-            decrementLocal(this.state.providerInflight, providerInflightStateKey(candidate.poolId, providerId));
-          }
-          this.notifyPoolUsage();
-          const script = providerKey === undefined ? POOL_RELEASE_SCRIPT : POOL_PROVIDER_RELEASE_SCRIPT;
-          const keys = providerKey === undefined ? [proxyInflightKey(candidate.poolId)] : [proxyInflightKey(candidate.poolId), providerKey];
-          void redisEvalNumber(redis, script, keys.length, ...keys).catch(
-            (error: unknown) => {
-              log.error("[pool-selector] failed to release distributed slot", error as Error);
-            },
-          );
-        },
-      };
+      return admitted;
     }
     return undefined;
   }

package/src/network/pool-cooldown-store.ts

@@ -18,6 +18,7 @@ import type { CartethyiaDatabase } from "../persistence/db-handle";
 import { networkPools, healthEvents } from "../persistence/schema";
 import { log } from "../observability/logger";
 import { providerRateLimitIsIpScoped } from "../providers/provider-metadata";
+import { resolvePoolProviderRateLimitCooldownMs } from "../config/cooldowns";
 import { PROVIDER_BLOCK_MS, type ProxyCooldownEntry } from "./pool/selector";
 
 type PoolProviderCooldownMap = NonNullable<
@@ -120,6 +121,37 @@ export async function dropAllProviderCooldownMarkers(
   }
 }
 
+/**
+ * Removes expired provider markers from one pool row, keeping live cooldowns
+ * and permanent blocks intact. Returns how many markers were dropped.
+ *
+ * The selector and Redis prune expired entries on read, but the durable JSON
+ * mirror is only ever written, never swept — so a marker that expired while
+ * the process was down survives on the row forever. Harmless to routing (boot
+ * restores only future-dated markers), but it bloats the row and makes the
+ * backup/restore surface carry dead state.
+ */
+export async function pruneExpiredProviderCooldownMarkers(
+  db: CartethyiaDatabase,
+  poolId: string,
+): Promise<number> {
+  try {
+    const map = await readMarkerMap(db, poolId);
+    const now = Date.now();
+    const expiredKeys: string[] = [];
+    for (const [providerKey, marker] of Object.entries(map)) {
+      if (marker.until <= now) expiredKeys.push(providerKey);
+    }
+    if (expiredKeys.length === 0) return 0;
+    for (const providerKey of expiredKeys) delete map[providerKey];
+    await writeMarkerMap(db, poolId, map);
+    return expiredKeys.length;
+  } catch (error) {
+    log.warn("pool provider cooldown mirror prune failed", { networkPoolId: poolId, error: String(error) });
+    return 0;
+  }
+}
+
 /**
  * Writes a probe-measured verdict the same way a dispatch-side event would be
  * recorded: one mirror marker and one audit row.
@@ -201,19 +233,22 @@ function probeEndpointOf(endpointConfig: unknown): string {
 /**
  * Re-enforces the stored probe verdicts at boot (new: replay, read-ahead).
  *
- * A stored `blocked` verdict is a *state*, not a decoration. If the marker the
- * probe flagged failed to persist, or the verdict predated the marker column,
- * the egress would return to rotation after restart and get refused again —
- * dispatch spending a real request to rediscover a refusal the row already
- * knew. So the boot read reconciles: a fresh (endpoint still matches) `blocked`
- * verdict for an address-keyed provider becomes a block flag, even when the
- * mirror has nothing (or has an older, cooler entry — the retirement claim is
- * the sharper fact and wins).
+ * A stored verdict is a *state*, not a decoration. If the marker the probe
+ * flagged failed to persist, or the verdict predated the marker column, the
+ * egress would return to rotation after restart and get refused again — dispatch
+ * spending a real request to rediscover a refusal the row already knew. So the
+ * boot read reconciles:
  *
- * `throttled` is deliberately **not** replayed into a cooldown: the cooldown
- * the probe flagged already carries the measured window, and blind-replaying
- * one from a verdict that predates the restart could outlive the upstream's
- * actual window.
+ * - A fresh `blocked` verdict for an address-keyed provider becomes a block
+ *   flag, even when the mirror has nothing (or has an older, cooler entry — the
+ *   retirement claim is the sharper fact and wins).
+ * - A fresh `throttled` verdict for an address-keyed provider becomes the same
+ *   cooldown a dispatch-side 429 would have written. The probe measured the
+ *   rate-limit window itself (`provider_probe_at` is when it observed the 429),
+ *   so the replayed window starts at that measurement, not at boot — without
+ *   the monotonic clock, a restart mid-window would shorten the cool-off the
+ *   probe proved. The configured default window is applied when the probe did
+ *   not record a duration.
  *
  * Returns how many verdicts were replayed.
  */
@@ -221,6 +256,7 @@ export async function restorePoolProbeVerdicts(
   selector: { hydrateProviderCooldown(entry: ProxyCooldownEntry): void },
   db: CartethyiaDatabase,
 ): Promise<number> {
+  const probeCooldownMs = resolvePoolProviderRateLimitCooldownMs();
   const rows = await db
     .select({
       id: networkPools.id,
@@ -230,21 +266,40 @@ export async function restorePoolProbeVerdicts(
       providerProbeStatus: networkPools.providerProbeStatus,
       providerProbeReason: networkPools.providerProbeReason,
       providerProbeEndpoint: networkPools.providerProbeEndpoint,
+      providerProbeAt: networkPools.providerProbeAt,
     })
     .from(networkPools);
   let replayed = 0;
   for (const row of rows) {
     if (row.status !== "active") continue;
-    if (row.providerProbeStatus !== "blocked") continue;
+    if (row.providerProbeStatus !== "blocked" && row.providerProbeStatus !== "throttled") continue;
     if (row.providerProbeEndpoint !== probeEndpointOf(row.endpointConfig)) continue;
     const providerId = row.providerProbeProviderId;
     if (typeof providerId !== "string" || !providerRateLimitIsIpScoped(providerId)) continue;
+    if (row.providerProbeStatus === "blocked") {
+      selector.hydrateProviderCooldown({
+        poolId: row.id,
+        providerId: providerId.toLowerCase(),
+        until: Date.now() + PROVIDER_BLOCK_MS,
+        reason: row.providerProbeReason ?? "Upstream refused this egress address",
+        blocked: true,
+      });
+      replayed += 1;
+      continue;
+    }
+    // `throttled`: replay the cooldown the probe measured. The upstream's
+    // window began when the probe saw the 429, so the marker expires at
+    // `probeAt + window`, unless that is already past — an allowance that
+    // refilled while the process was down must not be resurrected.
+    const probeAt = row.providerProbeAt?.getTime();
+    const until =
+      probeAt !== undefined ? probeAt + probeCooldownMs : Date.now() + probeCooldownMs;
+    if (until <= Date.now()) continue;
     selector.hydrateProviderCooldown({
       poolId: row.id,
       providerId: providerId.toLowerCase(),
-      until: Date.now() + PROVIDER_BLOCK_MS,
-      reason: row.providerProbeReason ?? "Upstream refused this egress address",
-      blocked: true,
+      until,
+      reason: row.providerProbeReason ?? "Operator probe measured a provider 429",
     });
     replayed += 1;
   }

package/src/network/pool-health-machine.ts

@@ -5,6 +5,7 @@ import type { CartethyiaDatabase } from "../persistence/db-handle";
 import { healthEvents, networkPools } from "../persistence/schema";
 import { drainPoolByteDelta } from "./pool/byte-accounting";
 import { resolvePoolCooldownMs } from "../config/cooldowns";
+import { dropProviderCooldownMarker, pruneExpiredProviderCooldownMarkers } from "./pool-cooldown-store";
 export interface PoolHealthEvent {
   readonly id: string;
   readonly networkPoolId: string;
@@ -15,7 +16,7 @@ export interface PoolHealthEvent {
   readonly createdAt: string;
 }
 
-export interface PoolHealthSnapshot {
+interface PoolHealthSnapshot {
   readonly poolId: string;
   readonly tenantId: string;
   readonly status: "active" | "cooldown" | "disabled";
@@ -54,7 +55,14 @@ function poolFaultCategory(error: unknown, origin: string | undefined): string |
 export async function recordPoolDispatchOutcome(
   db: CartethyiaDatabase,
   poolId: string,
-  input: { readonly succeeded: boolean; readonly error?: unknown; readonly errorOrigin?: string; readonly snapshotInvalidator?: { invalidate(): unknown } },
+  input: {
+    readonly succeeded: boolean;
+    readonly error?: unknown;
+    readonly errorOrigin?: string;
+    /** The provider the attempt dialed; used to clear its stale egress marker on success. */
+    readonly providerId?: string;
+    readonly snapshotInvalidator?: { invalidate(): unknown };
+  },
 ): Promise<void> {
   const category = input.succeeded ? undefined : poolFaultCategory(input.error, input.errorOrigin);
   // Metered bytes are banked even when health is untouched: an upstream-origin
@@ -95,6 +103,14 @@ export async function recordPoolDispatchOutcome(
     lastErrorCategory = null;
     lastErrorAt = null;
     lastSuccessAt = now;
+    // A completed dispatch proves the egress is serving again for this
+    // provider — clear its stale rate-limit marker (the selector's local
+    // entry and the Redis copy) so a spent pool is not kept out of rotation
+    // after it demonstrably recovered. Fire-and-forget: a failed clear must
+    // never fail a completed request.
+    if (input.providerId !== undefined) {
+      void dropProviderCooldownMarker(db, poolId, input.providerId).catch(() => undefined);
+    }
   } else {
     failures += 1;
     // Every failed probe parks the pool: a pool that cannot carry traffic is
@@ -252,5 +268,9 @@ export async function sweepExpiredPoolCooldowns(db: CartethyiaDatabase): Promise
   await db.update(networkPools).set({ status: "active", consecutiveFailures: 0, cooldownUntil: null, lastRecoveredAt: now }).where(inArray(networkPools.id, expired.map((row) => row.id)));
   await db.insert(healthEvents).values(expired.map((row) => ({ id: randomUUID(), entityKind: "pool" as const, networkPoolId: row.id, fromStatus: row.status, toStatus: "active" as const, reason: "Cooldown elapsed — auto-recovered", errorCategory: null, createdAt: now })));
   for (const row of expired) publishPoolHealth({ poolId: row.id, tenantId: row.tenantId, status: "active", consecutiveFailures: 0 });
+  // The pool-level cooldown elapsed, so the same window's provider markers are
+  // expired too. Drop them from the durable JSON mirror — the selector and
+  // Redis prune on read, but the row's stale copy would otherwise live forever.
+  await Promise.all(expired.map((row) => pruneExpiredProviderCooldownMarkers(db, row.id)));
   return expired.length;
 }

package/src/network/rate-limit-verdict.ts

@@ -0,0 +1,68 @@
+/**
+ * Unified egress-failure classification for ip-scoped, rate-limited providers.
+ *
+ * One 429 is read in several places — dispatch (`attempt-loop`, streaming
+ * `finalize`), the provider probe, account health, and the pool health machine
+ * each had their own vocabulary. The two questions that decide whether a probe
+ * *pool* (not an account) is responsible for a failure are:
+ *
+ * - Did the upstream count this request against the egress address? Only a
+ *   provider whose limits follow the IP (`ipScopedRateLimit`) qualifies — for
+ *   everyone else the 429 belongs to the credential that was dialed, and
+ *   cooling the pool would sideline healthy accounts.
+ * - Is the address itself retired (free-tier 403) rather than merely
+ *   throttled (429)? A block is terminal; a cooldown refills.
+ *
+ * This module is the single source of truth for those two facts. Dispatch and
+ * the probe both read it, so the operator's "test" and gateway's own dial
+ * cannot disagree about which egress is spent.
+ */
+import { GatewayError } from "../protocol/gateway-error";
+import { providerRateLimitIsIpScoped } from "../providers/provider-metadata";
+
+/** What one upstream failure means for the egress that carried it. */
+type EgressVerdict =
+  | { kind: "none" }
+  | { kind: "cooldown"; retryAfterMs?: number }
+  | { kind: "blocked" };
+
+/** The upstream's own wording for a retired free-tier address. */
+export const FREE_TIER_REFUSAL = /freetier|free tier can only be used/i;
+
+/**
+ * Classifies one upstream failure against one provider.
+ *
+ * `providerId` is the caller's own candidate, not a field read off the error:
+ * the in-stream error frames the protocol modules build carry no `providerId`
+ * detail, so reading it from `details` would silently disable the IP-scoped
+ * rule on exactly the streaming path it exists for.
+ */
+export function classifyEgressFailure(error: unknown, providerId: string): EgressVerdict {
+  if (!(error instanceof GatewayError) || error.origin !== "upstream") {
+    return { kind: "none" };
+  }
+  if (!providerRateLimitIsIpScoped(providerId)) return { kind: "none" };
+
+  // A provider-scoped 429 exhausted the address's allowance: cool the pool
+  // until the window refills.
+  if (error.status === 429 && error.details.rateLimitScope === "provider") {
+    const retryAfterMs =
+      typeof error.details.retryAfterMs === "number" ? error.details.retryAfterMs : undefined;
+    return retryAfterMs === undefined ? { kind: "cooldown" } : { kind: "cooldown", retryAfterMs };
+  }
+
+  // A free-tier 403 retires the address: the request was never read, so the
+  // same egress cannot succeed no matter what the payload says. Deliberately
+  // narrower than "any 403" — a content-policy or licence refusal is also a
+  // 403 and must not retire a healthy proxy.
+  if (error.status === 403) {
+    const providerCode =
+      typeof error.details.providerCode === "string" ? error.details.providerCode : "";
+    const raw = String(error.details.raw ?? "");
+    if (/freetier|free.?tier/i.test(providerCode) || FREE_TIER_REFUSAL.test(`${error.message} ${raw}`)) {
+      return { kind: "blocked" };
+    }
+  }
+
+  return { kind: "none" };
+}

package/src/network/retry.ts

@@ -9,7 +9,7 @@
 import { exponentialBackoff, type BackoffOptions } from "./backoff";
 import { sleep as defaultSleep } from "../shared/sleep";
 
-export interface RetryOptions {
+interface RetryOptions {
   readonly maxRetries?: number;
   readonly baseDelayMs?: number;
   readonly maxDelayMs?: number;