cartethyia 2.7.3 → 2.7.4

package/src/network/pool/resolver.ts

@@ -364,21 +364,21 @@ function connectValidatedWebSocket(
       if (settled) return;
       settled = true;
       socket.terminate();
-      reject(new GatewayError("transport_unavailable", 502, "WebSocket handshake timed out"));
+      reject(new GatewayError("transport_unavailable", 502, "WebSocket handshake timed out (request {requestId}). Some things just take longer 🐢"));
     }, 15_000);
     const abort = (): void => {
       if (settled) return;
       settled = true;
       clearTimeout(timeout);
       socket.terminate();
-      reject(new GatewayError("transport_closed", 499, "WebSocket connection was cancelled"));
+      reject(new GatewayError("transport_closed", 499, "WebSocket connection was cancelled (request {requestId}). Send it again whenever ready 🙃"));
     };
     const onError = (): void => {
       if (settled) return;
       settled = true;
       clearTimeout(timeout);
       signal.removeEventListener("abort", abort);
-      reject(new GatewayError("transport_unavailable", 502, "WebSocket connection failed"));
+      reject(new GatewayError("transport_unavailable", 502, "WebSocket connection failed (request {requestId}). The connection ghosted us — come back in a moment 👻"));
     };
     socket.once("open", () => {
       if (settled) return;
@@ -398,7 +398,7 @@ function connectValidatedWebSocket(
 function agentForWebSocket(agent: PoolAgent): import("http").Agent | import("https").Agent {
   if (isProxyAgentPair(agent)) return agent.https;
   if (agent instanceof HttpsAgent) return agent;
-  throw new GatewayError("proxy_pool_unhealthy", 503, "configured WebSocket proxy pool does not support WSS");
+  throw new GatewayError("proxy_pool_unhealthy", 503, "configured WebSocket proxy pool does not support WSS (request {requestId}). This bridge doesn't speak WSS — reconfigure the pool 🔧");
 }
 
 /**
@@ -411,20 +411,20 @@ function agentForWebSocket(agent: PoolAgent): import("http").Agent | import("htt
  */
 function assertBidiTarget(url: URL): void {
   if (url.protocol !== "https:") {
-    throw new GatewayError("invalid_request", 400, "validated HTTP/2 egress requires https");
+    throw new GatewayError("invalid_request", 400, "validated HTTP/2 egress requires https (request {requestId}). Check the shape of what you sent — then resend 🪞");
   }
   if (url.username || url.password || url.hash) {
-    throw new GatewayError("invalid_request", 400, "invalid validated HTTP/2 target");
+    throw new GatewayError("invalid_request", 400, "invalid validated HTTP/2 target (request {requestId}). Check the shape of what you sent — then resend 🪞");
   }
   if (url.port !== "" && url.port !== "443") {
-    throw new GatewayError("invalid_request", 400, "validated HTTP/2 egress runs on port 443");
+    throw new GatewayError("invalid_request", 400, "validated HTTP/2 egress runs on port 443 (request {requestId}). Check the shape of what you sent — then resend 🪞");
   }
   const hostname = url.hostname;
   if (hostname.length === 0) {
-    throw new GatewayError("invalid_request", 400, "HTTP/2 target hostname is required");
+    throw new GatewayError("invalid_request", 400, "HTTP/2 target hostname is required (request {requestId}). Check the shape of what you sent — then resend 🪞");
   }
   if (isIP(hostname) !== 0) {
-    throw new GatewayError("invalid_request", 400, "HTTP/2 target must be a hostname, not an IP literal");
+    throw new GatewayError("invalid_request", 400, "HTTP/2 target must be a hostname, not an IP literal (request {requestId}). Check the shape of what you sent — then resend 🪞");
   }
 }
 
@@ -455,13 +455,13 @@ function dialDirect(
     const onAbort = (): void => {
       cleanup();
       socket.destroy(new Error("HTTP/2 connection aborted"));
-      reject(new GatewayError("transport_closed", 499, "HTTP/2 connection was cancelled"));
+      reject(new GatewayError("transport_closed", 499, "HTTP/2 connection was cancelled (request {requestId}). Send it again whenever ready 🙃"));
     };
     const timer = setTimeout(() => {
       cleanup();
       socket.destroy();
       reject(
-        new GatewayError("transport_unavailable", 502, "HTTP/2 connection timed out", {}, "network"),
+        new GatewayError("transport_unavailable", 502, "HTTP/2 connection timed out (request {requestId}). Some things just take longer 🐢", {}, "network"),
       );
     }, HTTP2_CONNECT_TIMEOUT_MS);
     socket.once("secureConnect", () => {
@@ -474,7 +474,7 @@ function dialDirect(
         new GatewayError(
           "transport_unavailable",
           502,
-          `HTTP/2 connection failed: ${error.message}`,
+          `HTTP/2 connection failed: ${error.message} (request {requestId}). The connection ghosted us — come back in a moment 👻`,
           {},
           "network",
         ),
@@ -506,7 +506,7 @@ function dialThroughPool(
     throw new GatewayError(
       "proxy_pool_unhealthy",
       503,
-      "configured proxy pool does not support HTTPS tunneling",
+      "configured proxy pool does not support HTTPS tunneling (request {requestId}). This bridge doesn't speak HTTPS tunnels — reconfigure the pool 🔧",
     );
   }
   return new Promise<tls.TLSSocket>((resolve, reject) => {
@@ -519,14 +519,14 @@ function dialThroughPool(
               ? new GatewayError(
                   "proxy_pool_unhealthy",
                   503,
-                  "Configured proxy pool could not be established.",
+                  "Configured proxy pool could not be established (request {requestId}). This bridge looks shaky — repair or reroute 🔧",
                   { reason: error.message.slice(0, 200) },
                   "network",
                 )
               : new GatewayError(
                   "transport_unavailable",
                   502,
-                  `HTTP/2 tunnel failed: ${error?.message ?? "no socket"}`,
+                  `HTTP/2 tunnel failed: ${error?.message ?? "no socket"} (request {requestId}). The connection ghosted us — come back in a moment 👻`,
                   {},
                   "network",
                 ),
@@ -546,13 +546,13 @@ function dialThroughPool(
         const onAbort = (): void => {
           cleanup();
           secured.destroy();
-          reject(new GatewayError("transport_closed", 499, "HTTP/2 connection was cancelled"));
+          reject(new GatewayError("transport_closed", 499, "HTTP/2 connection was cancelled (request {requestId}). Send it again whenever ready 🙃"));
         };
         const timer = setTimeout(() => {
           cleanup();
           secured.destroy();
           reject(
-            new GatewayError("transport_unavailable", 502, "HTTP/2 connection timed out", {}, "network"),
+            new GatewayError("transport_unavailable", 502, "HTTP/2 connection timed out (request {requestId}). Some things just take longer 🐢", {}, "network"),
           );
         }, HTTP2_CONNECT_TIMEOUT_MS);
         secured.once("secureConnect", () => {
@@ -565,7 +565,7 @@ function dialThroughPool(
             new GatewayError(
               "transport_unavailable",
               502,
-              `HTTP/2 TLS handshake failed: ${tlsError.message}`,
+              `HTTP/2 TLS handshake failed: ${tlsError.message} (request {requestId}). The connection ghosted us — come back in a moment 👻`,
               {},
               "network",
             ),
@@ -636,7 +636,7 @@ class Http2BidiSession implements ValidatedHttp2BidiStream {
           new GatewayError(
             "platform_unavailable",
             502,
-            "HTTP/2 response exceeded the buffer bound",
+            "HTTP/2 response exceeded the buffer bound (request {requestId}). The upstream sent more than we can hold — retry 🌙",
             {},
             "network",
           ),
@@ -646,7 +646,7 @@ class Http2BidiSession implements ValidatedHttp2BidiStream {
     stream.on("end", () => this.finishInbound());
     stream.on("error", (error) => this.fail(error));
     stream.on("aborted", () =>
-      this.fail(new GatewayError("transport_unavailable", 502, "HTTP/2 stream aborted by remote", {}, "network")),
+      this.fail(new GatewayError("transport_unavailable", 502, "HTTP/2 stream aborted by remote (request {requestId}). The connection ghosted us — come back in a moment 👻", {}, "network")),
     );
     stream.on("close", () => {
       this.finishInbound();
@@ -699,7 +699,7 @@ class Http2BidiSession implements ValidatedHttp2BidiStream {
         new GatewayError(
           "transport_unavailable",
           504,
-          `HTTP/2 stream read-idle timeout after ${this.readIdleMs}ms — connection presumed dead`,
+          `HTTP/2 stream read-idle timeout after ${this.readIdleMs}ms — connection presumed dead (request {requestId}). The connection ghosted us — come back in a moment 👻`,
           {},
           "network",
         ),
@@ -836,7 +836,7 @@ function openBidiStream(
         new GatewayError(
           "transport_unavailable",
           502,
-          `HTTP/2 session failed: ${error.message}`,
+          `HTTP/2 session failed: ${error.message} (request {requestId}). The connection ghosted us — come back in a moment 👻`,
           {},
           "network",
         ),
@@ -871,7 +871,7 @@ function openBidiStream(
         // Already destroying.
       }
       reject(
-        new GatewayError("transport_unavailable", 502, "HTTP/2 session timed out", {}, "network"),
+        new GatewayError("transport_unavailable", 502, "HTTP/2 session timed out (request {requestId}). Some things just take longer 🐢", {}, "network"),
       );
     }, HTTP2_CONNECT_TIMEOUT_MS);
     session.once("error", onError);
@@ -895,7 +895,7 @@ class WsSession implements ProviderWebSocketSession {
     this.#signal = signal;
     socket.on("message", (data: RawData, isBinary: boolean) => {
       if (isBinary) {
-        this.#fail(new GatewayError("transport_unavailable", 502, "unexpected binary WebSocket frame"));
+        this.#fail(new GatewayError("transport_unavailable", 502, "unexpected binary WebSocket frame (request {requestId}). The connection ghosted us — come back in a moment 👻"));
         return;
       }
       const message = data.toString();
@@ -907,14 +907,14 @@ class WsSession implements ProviderWebSocketSession {
         this.#messages.push(message);
       }
     });
-    socket.on("close", () => this.#fail(new GatewayError("transport_closed", 502, "WebSocket connection closed")));
-    socket.on("error", () => this.#fail(new GatewayError("transport_unavailable", 502, "WebSocket connection failed")));
+    socket.on("close", () => this.#fail(new GatewayError("transport_closed", 502, "WebSocket connection closed (request {requestId}). The connection ghosted us — come back in a moment 👻")));
+    socket.on("error", () => this.#fail(new GatewayError("transport_unavailable", 502, "WebSocket connection failed (request {requestId}). The connection ghosted us — come back in a moment 👻")));
   }
 
   send(message: string): void {
     if (this.#closed) throw this.#closed;
     if (this.#socket.readyState !== WebSocket.OPEN) {
-      throw new GatewayError("transport_closed", 499, "WebSocket connection is not open");
+      throw new GatewayError("transport_closed", 499, "WebSocket connection is not open (request {requestId}). Send it again whenever ready 🙃");
     }
     this.#socket.send(message, { binary: false });
   }
@@ -927,11 +927,11 @@ class WsSession implements ProviderWebSocketSession {
     return new Promise<string>((resolve, reject) => {
       const timeout = setTimeout(() => {
         cleanup();
-        reject(new GatewayError("transport_unavailable", 504, "WebSocket receive timed out"));
+        reject(new GatewayError("transport_unavailable", 504, "WebSocket receive timed out (request {requestId}). Some things just take longer 🐢"));
       }, 45_000);
       const onAbort = (): void => {
         cleanup();
-        reject(new GatewayError("transport_closed", 499, "WebSocket receive was cancelled"));
+        reject(new GatewayError("transport_closed", 499, "WebSocket receive was cancelled (request {requestId}). Send it again whenever ready 🙃"));
       };
       const cleanup = (): void => {
         clearTimeout(timeout);
@@ -997,7 +997,7 @@ export class ValidatedNetworkBindingFactory {
           throw new GatewayError(
             "proxy_pool_unhealthy",
             503,
-            "Configured proxy pool could not be established.",
+            "Configured proxy pool could not be established (request {requestId}). This bridge looks shaky — repair or reroute 🔧",
             { poolId: networkPoolId, reason: error.message.slice(0, 200) },
             "network",
           );
@@ -1009,27 +1009,27 @@ export class ValidatedNetworkBindingFactory {
   webSocket(networkPoolId?: string, tenantId?: string): ValidatedOutboundWebSocket {
     return async (url, headers, signal) => {
       if (url.protocol !== "wss:") {
-        throw new GatewayError("invalid_request", 400, "validated WebSocket egress requires wss");
+        throw new GatewayError("invalid_request", 400, "validated WebSocket egress requires wss (request {requestId}). Check the shape of what you sent — then resend 🪞");
       }
       if (url.username || url.password || url.hash || (url.port !== "" && url.port !== "443")) {
-        throw new GatewayError("invalid_request", 400, "invalid validated WebSocket target");
+        throw new GatewayError("invalid_request", 400, "invalid validated WebSocket target (request {requestId}). Check the shape of what you sent — then resend 🪞");
       }
       const hostname = url.hostname;
-      if (!hostname) throw new GatewayError("invalid_request", 400, "WebSocket target hostname is required");
+      if (!hostname) throw new GatewayError("invalid_request", 400, "WebSocket target hostname is required (request {requestId}). Check the shape of what you sent — then resend 🪞");
       const destination = await this.resolve(hostname, signal, 443);
       if (!isAddressAllowed(destination.resolvedAddress, this.policy)) {
-        throw new GatewayError("invalid_request", 400, "unsafe upstream address rejected");
+        throw new GatewayError("invalid_request", 400, "unsafe upstream address rejected (request {requestId}). Check the shape of what you sent — then resend 🪞");
       }
       let agent: PoolAgent | undefined;
       if (networkPoolId || tenantId) {
         if (!networkPoolId || !tenantId || !this.poolResolver) {
-          throw new GatewayError("proxy_pool_unhealthy", 503, "configured WebSocket proxy pool could not be established");
+          throw new GatewayError("proxy_pool_unhealthy", 503, "configured WebSocket proxy pool could not be established (request {requestId}). This bridge looks shaky — repair or reroute 🔧");
         }
         try {
           agent = await this.poolResolver.resolveWebSocketAgent(networkPoolId, tenantId);
         } catch (error) {
           if (error instanceof PoolBindingError) {
-            throw new GatewayError("proxy_pool_unhealthy", 503, "configured WebSocket proxy pool could not be established");
+            throw new GatewayError("proxy_pool_unhealthy", 503, "configured WebSocket proxy pool could not be established (request {requestId}). This bridge looks shaky — repair or reroute 🔧");
           }
           throw error;
         }
@@ -1058,7 +1058,7 @@ export class ValidatedNetworkBindingFactory {
       assertBidiTarget(url);
       const { hostname } = url;
       if (options?.allowHostname !== undefined && !options.allowHostname(hostname)) {
-        throw new GatewayError("invalid_request", 400, "HTTP/2 target host is not allowlisted");
+        throw new GatewayError("invalid_request", 400, "HTTP/2 target host is not allowlisted (request {requestId}). All roads blocked — time to find a new route 🗺️");
       }
       const signal = options?.signal ?? AbortSignal.timeout(HTTP2_CONNECT_TIMEOUT_MS);
       // Validation runs on every new connection, not once per capability: a
@@ -1066,7 +1066,7 @@ export class ValidatedNetworkBindingFactory {
       // resolved safely the first time is not safe the twentieth.
       const destination = await this.resolve(hostname, signal, 443);
       if (!isAddressAllowed(destination.resolvedAddress, this.policy)) {
-        throw new GatewayError("invalid_request", 400, "unsafe upstream address rejected");
+        throw new GatewayError("invalid_request", 400, "unsafe upstream address rejected (request {requestId}). All roads blocked — time to find a new route 🗺️");
       }
       // Mirrors `fetch`: no pool selected (or none configured) means validated
       // *direct* egress, which is already resolved and SSRF-checked above.
@@ -1086,7 +1086,7 @@ export class ValidatedNetworkBindingFactory {
             throw new GatewayError(
               "proxy_pool_unhealthy",
               503,
-              "Configured proxy pool could not be established.",
+              "Configured proxy pool could not be established (request {requestId}). This bridge looks shaky — repair or reroute 🔧",
               { poolId: networkPoolId, reason: error.message.slice(0, 200) },
               "network",
             );

package/src/network/pool-cooldown-store.ts

@@ -18,6 +18,7 @@ import type { CartethyiaDatabase } from "../persistence/db-handle";
 import { networkPools, healthEvents } from "../persistence/schema";
 import { log } from "../observability/logger";
 import { providerRateLimitIsIpScoped } from "../providers/provider-metadata";
+import { resolvePoolProviderRateLimitCooldownMs } from "../config/cooldowns";
 import { PROVIDER_BLOCK_MS, type ProxyCooldownEntry } from "./pool/selector";
 
 type PoolProviderCooldownMap = NonNullable<
@@ -120,6 +121,37 @@ export async function dropAllProviderCooldownMarkers(
   }
 }
 
+/**
+ * Removes expired provider markers from one pool row, keeping live cooldowns
+ * and permanent blocks intact. Returns how many markers were dropped.
+ *
+ * The selector and Redis prune expired entries on read, but the durable JSON
+ * mirror is only ever written, never swept — so a marker that expired while
+ * the process was down survives on the row forever. Harmless to routing (boot
+ * restores only future-dated markers), but it bloats the row and makes the
+ * backup/restore surface carry dead state.
+ */
+export async function pruneExpiredProviderCooldownMarkers(
+  db: CartethyiaDatabase,
+  poolId: string,
+): Promise<number> {
+  try {
+    const map = await readMarkerMap(db, poolId);
+    const now = Date.now();
+    const expiredKeys: string[] = [];
+    for (const [providerKey, marker] of Object.entries(map)) {
+      if (marker.until <= now) expiredKeys.push(providerKey);
+    }
+    if (expiredKeys.length === 0) return 0;
+    for (const providerKey of expiredKeys) delete map[providerKey];
+    await writeMarkerMap(db, poolId, map);
+    return expiredKeys.length;
+  } catch (error) {
+    log.warn("pool provider cooldown mirror prune failed", { networkPoolId: poolId, error: String(error) });
+    return 0;
+  }
+}
+
 /**
  * Writes a probe-measured verdict the same way a dispatch-side event would be
  * recorded: one mirror marker and one audit row.
@@ -201,19 +233,22 @@ function probeEndpointOf(endpointConfig: unknown): string {
 /**
  * Re-enforces the stored probe verdicts at boot (new: replay, read-ahead).
  *
- * A stored `blocked` verdict is a *state*, not a decoration. If the marker the
- * probe flagged failed to persist, or the verdict predated the marker column,
- * the egress would return to rotation after restart and get refused again —
- * dispatch spending a real request to rediscover a refusal the row already
- * knew. So the boot read reconciles: a fresh (endpoint still matches) `blocked`
- * verdict for an address-keyed provider becomes a block flag, even when the
- * mirror has nothing (or has an older, cooler entry — the retirement claim is
- * the sharper fact and wins).
+ * A stored verdict is a *state*, not a decoration. If the marker the probe
+ * flagged failed to persist, or the verdict predated the marker column, the
+ * egress would return to rotation after restart and get refused again — dispatch
+ * spending a real request to rediscover a refusal the row already knew. So the
+ * boot read reconciles:
  *
- * `throttled` is deliberately **not** replayed into a cooldown: the cooldown
- * the probe flagged already carries the measured window, and blind-replaying
- * one from a verdict that predates the restart could outlive the upstream's
- * actual window.
+ * - A fresh `blocked` verdict for an address-keyed provider becomes a block
+ *   flag, even when the mirror has nothing (or has an older, cooler entry — the
+ *   retirement claim is the sharper fact and wins).
+ * - A fresh `throttled` verdict for an address-keyed provider becomes the same
+ *   cooldown a dispatch-side 429 would have written. The probe measured the
+ *   rate-limit window itself (`provider_probe_at` is when it observed the 429),
+ *   so the replayed window starts at that measurement, not at boot — without
+ *   the monotonic clock, a restart mid-window would shorten the cool-off the
+ *   probe proved. The configured default window is applied when the probe did
+ *   not record a duration.
  *
  * Returns how many verdicts were replayed.
  */
@@ -221,6 +256,7 @@ export async function restorePoolProbeVerdicts(
   selector: { hydrateProviderCooldown(entry: ProxyCooldownEntry): void },
   db: CartethyiaDatabase,
 ): Promise<number> {
+  const probeCooldownMs = resolvePoolProviderRateLimitCooldownMs();
   const rows = await db
     .select({
       id: networkPools.id,
@@ -230,21 +266,40 @@ export async function restorePoolProbeVerdicts(
       providerProbeStatus: networkPools.providerProbeStatus,
       providerProbeReason: networkPools.providerProbeReason,
       providerProbeEndpoint: networkPools.providerProbeEndpoint,
+      providerProbeAt: networkPools.providerProbeAt,
     })
     .from(networkPools);
   let replayed = 0;
   for (const row of rows) {
     if (row.status !== "active") continue;
-    if (row.providerProbeStatus !== "blocked") continue;
+    if (row.providerProbeStatus !== "blocked" && row.providerProbeStatus !== "throttled") continue;
     if (row.providerProbeEndpoint !== probeEndpointOf(row.endpointConfig)) continue;
     const providerId = row.providerProbeProviderId;
     if (typeof providerId !== "string" || !providerRateLimitIsIpScoped(providerId)) continue;
+    if (row.providerProbeStatus === "blocked") {
+      selector.hydrateProviderCooldown({
+        poolId: row.id,
+        providerId: providerId.toLowerCase(),
+        until: Date.now() + PROVIDER_BLOCK_MS,
+        reason: row.providerProbeReason ?? "Upstream refused this egress address",
+        blocked: true,
+      });
+      replayed += 1;
+      continue;
+    }
+    // `throttled`: replay the cooldown the probe measured. The upstream's
+    // window began when the probe saw the 429, so the marker expires at
+    // `probeAt + window`, unless that is already past — an allowance that
+    // refilled while the process was down must not be resurrected.
+    const probeAt = row.providerProbeAt?.getTime();
+    const until =
+      probeAt !== undefined ? probeAt + probeCooldownMs : Date.now() + probeCooldownMs;
+    if (until <= Date.now()) continue;
     selector.hydrateProviderCooldown({
       poolId: row.id,
       providerId: providerId.toLowerCase(),
-      until: Date.now() + PROVIDER_BLOCK_MS,
-      reason: row.providerProbeReason ?? "Upstream refused this egress address",
-      blocked: true,
+      until,
+      reason: row.providerProbeReason ?? "Operator probe measured a provider 429",
     });
     replayed += 1;
   }

package/src/network/pool-health-machine.ts

@@ -5,6 +5,7 @@ import type { CartethyiaDatabase } from "../persistence/db-handle";
 import { healthEvents, networkPools } from "../persistence/schema";
 import { drainPoolByteDelta } from "./pool/byte-accounting";
 import { resolvePoolCooldownMs } from "../config/cooldowns";
+import { dropProviderCooldownMarker, pruneExpiredProviderCooldownMarkers } from "./pool-cooldown-store";
 export interface PoolHealthEvent {
   readonly id: string;
   readonly networkPoolId: string;
@@ -54,7 +55,14 @@ function poolFaultCategory(error: unknown, origin: string | undefined): string |
 export async function recordPoolDispatchOutcome(
   db: CartethyiaDatabase,
   poolId: string,
-  input: { readonly succeeded: boolean; readonly error?: unknown; readonly errorOrigin?: string; readonly snapshotInvalidator?: { invalidate(): unknown } },
+  input: {
+    readonly succeeded: boolean;
+    readonly error?: unknown;
+    readonly errorOrigin?: string;
+    /** The provider the attempt dialed; used to clear its stale egress marker on success. */
+    readonly providerId?: string;
+    readonly snapshotInvalidator?: { invalidate(): unknown };
+  },
 ): Promise<void> {
   const category = input.succeeded ? undefined : poolFaultCategory(input.error, input.errorOrigin);
   // Metered bytes are banked even when health is untouched: an upstream-origin
@@ -95,6 +103,14 @@ export async function recordPoolDispatchOutcome(
     lastErrorCategory = null;
     lastErrorAt = null;
     lastSuccessAt = now;
+    // A completed dispatch proves the egress is serving again for this
+    // provider — clear its stale rate-limit marker (the selector's local
+    // entry and the Redis copy) so a spent pool is not kept out of rotation
+    // after it demonstrably recovered. Fire-and-forget: a failed clear must
+    // never fail a completed request.
+    if (input.providerId !== undefined) {
+      void dropProviderCooldownMarker(db, poolId, input.providerId).catch(() => undefined);
+    }
   } else {
     failures += 1;
     // Every failed probe parks the pool: a pool that cannot carry traffic is
@@ -252,5 +268,9 @@ export async function sweepExpiredPoolCooldowns(db: CartethyiaDatabase): Promise
   await db.update(networkPools).set({ status: "active", consecutiveFailures: 0, cooldownUntil: null, lastRecoveredAt: now }).where(inArray(networkPools.id, expired.map((row) => row.id)));
   await db.insert(healthEvents).values(expired.map((row) => ({ id: randomUUID(), entityKind: "pool" as const, networkPoolId: row.id, fromStatus: row.status, toStatus: "active" as const, reason: "Cooldown elapsed — auto-recovered", errorCategory: null, createdAt: now })));
   for (const row of expired) publishPoolHealth({ poolId: row.id, tenantId: row.tenantId, status: "active", consecutiveFailures: 0 });
+  // The pool-level cooldown elapsed, so the same window's provider markers are
+  // expired too. Drop them from the durable JSON mirror — the selector and
+  // Redis prune on read, but the row's stale copy would otherwise live forever.
+  await Promise.all(expired.map((row) => pruneExpiredProviderCooldownMarkers(db, row.id)));
   return expired.length;
 }

package/src/network/rate-limit-verdict.ts

@@ -0,0 +1,68 @@
+/**
+ * Unified egress-failure classification for ip-scoped, rate-limited providers.
+ *
+ * One 429 is read in several places — dispatch (`attempt-loop`, streaming
+ * `finalize`), the provider probe, account health, and the pool health machine
+ * each had their own vocabulary. The two questions that decide whether a probe
+ * *pool* (not an account) is responsible for a failure are:
+ *
+ * - Did the upstream count this request against the egress address? Only a
+ *   provider whose limits follow the IP (`ipScopedRateLimit`) qualifies — for
+ *   everyone else the 429 belongs to the credential that was dialed, and
+ *   cooling the pool would sideline healthy accounts.
+ * - Is the address itself retired (free-tier 403) rather than merely
+ *   throttled (429)? A block is terminal; a cooldown refills.
+ *
+ * This module is the single source of truth for those two facts. Dispatch and
+ * the probe both read it, so the operator's "test" and gateway's own dial
+ * cannot disagree about which egress is spent.
+ */
+import { GatewayError } from "../protocol/gateway-error";
+import { providerRateLimitIsIpScoped } from "../providers/provider-metadata";
+
+/** What one upstream failure means for the egress that carried it. */
+export type EgressVerdict =
+  | { kind: "none" }
+  | { kind: "cooldown"; retryAfterMs?: number }
+  | { kind: "blocked" };
+
+/** The upstream's own wording for a retired free-tier address. */
+export const FREE_TIER_REFUSAL = /freetier|free tier can only be used/i;
+
+/**
+ * Classifies one upstream failure against one provider.
+ *
+ * `providerId` is the caller's own candidate, not a field read off the error:
+ * the in-stream error frames the protocol modules build carry no `providerId`
+ * detail, so reading it from `details` would silently disable the IP-scoped
+ * rule on exactly the streaming path it exists for.
+ */
+export function classifyEgressFailure(error: unknown, providerId: string): EgressVerdict {
+  if (!(error instanceof GatewayError) || error.origin !== "upstream") {
+    return { kind: "none" };
+  }
+  if (!providerRateLimitIsIpScoped(providerId)) return { kind: "none" };
+
+  // A provider-scoped 429 exhausted the address's allowance: cool the pool
+  // until the window refills.
+  if (error.status === 429 && error.details.rateLimitScope === "provider") {
+    const retryAfterMs =
+      typeof error.details.retryAfterMs === "number" ? error.details.retryAfterMs : undefined;
+    return retryAfterMs === undefined ? { kind: "cooldown" } : { kind: "cooldown", retryAfterMs };
+  }
+
+  // A free-tier 403 retires the address: the request was never read, so the
+  // same egress cannot succeed no matter what the payload says. Deliberately
+  // narrower than "any 403" — a content-policy or licence refusal is also a
+  // 403 and must not retire a healthy proxy.
+  if (error.status === 403) {
+    const providerCode =
+      typeof error.details.providerCode === "string" ? error.details.providerCode : "";
+    const raw = String(error.details.raw ?? "");
+    if (/freetier|free.?tier/i.test(providerCode) || FREE_TIER_REFUSAL.test(`${error.message} ${raw}`)) {
+      return { kind: "blocked" };
+    }
+  }
+
+  return { kind: "none" };
+}

package/src/network/ssrf.ts

@@ -170,7 +170,7 @@ export function validateResolvedAddresses(
   policy: SsrfPolicy,
 ): string {
   if (addresses.length === 0 || addresses.some((address) => !isAddressAllowed(address, policy))) {
-    throw new GatewayError("invalid_request", 400, "The upstream address this route resolves to is not allowed. Point the provider at a permitted host.");
+    throw new GatewayError("invalid_request", 400, "The upstream address this route resolves to is not allowed (request {requestId}). Point the provider at a permitted host. All roads blocked — time to find a new route 🗺️");
   }
   return addresses[0]!;
 }
@@ -227,7 +227,7 @@ export async function resolveAllAddresses(
     throw new GatewayError(
       "transport_unavailable",
       502,
-      "upstream DNS resolution failed",
+      "upstream DNS resolution failed (request {requestId}). The connection ghosted us — come back in a moment 👻",
       { cause: String(error) },
       "network",
     );
@@ -238,8 +238,8 @@ function abortErrorForSignal(signal: AbortSignal): GatewayError {
   const reason = signal.reason;
   if (reason instanceof GatewayError) return reason;
   if (reason instanceof DOMException && reason.name === "TimeoutError")
-    return new GatewayError("deadline_exceeded", 504, "request deadline exceeded", {}, "cartethyia");
-  return new GatewayError("transport_closed", 499, "request was cancelled");
+    return new GatewayError("deadline_exceeded", 504, "request deadline exceeded (request {requestId}). Some things just take longer 🐢", {}, "cartethyia");
+  return new GatewayError("transport_closed", 499, "The request was cancelled (request {requestId}). Send it again whenever ready 🙃");
 }
 
 export async function resolveAndValidateOnce(