cartethyia 2.7.9 → 2.8.1

package/src/network/egress/ws.ts

@@ -0,0 +1,175 @@
+/**
+ * Validated WebSocket egress — WSS session + the dial that opens one.
+ *
+ * The dial resolves the target through the SSRF policy (done by the caller,
+ * the binding factory) and then either connects through the selected pool
+ * agent's tunnel or dials direct with pinned, SSRF-validated address.
+ */
+import * as tls from "node:tls";
+import WebSocket, { type ClientOptions, type RawData } from "ws";
+import { GatewayError } from "../../protocol/gateway-error";
+import { isProxyAgentPair, type PoolAgent } from "../pool/agent";
+import { Agent as HttpsAgent } from "node:https";
+import type { ProviderWebSocketSession } from "../../providers/provider-contracts";
+import type { ValidatedDestination } from "../ssrf";
+
+class WsSession implements ProviderWebSocketSession {
+  readonly #socket: WebSocket;
+  readonly #signal: AbortSignal;
+  readonly #messages: string[] = [];
+  readonly #waiters: Array<{
+    resolve(value: string): void;
+    reject(reason: Error): void;
+    cleanup(): void;
+  }> = [];
+  #closed: Error | undefined;
+
+  constructor(socket: WebSocket, signal: AbortSignal) {
+    this.#socket = socket;
+    this.#signal = signal;
+    socket.on("message", (data: RawData, isBinary: boolean) => {
+      if (isBinary) {
+        this.#fail(new GatewayError("transport_unavailable", 502, "unexpected binary WebSocket frame (request {requestId}). The connection ghosted us — come back in a moment 👻"));
+        return;
+      }
+      const message = data.toString();
+      const waiter = this.#waiters.shift();
+      if (waiter) {
+        waiter.cleanup();
+        waiter.resolve(message);
+      } else {
+        this.#messages.push(message);
+      }
+    });
+    socket.on("close", () => this.#fail(new GatewayError("transport_closed", 502, "WebSocket connection closed (request {requestId}). The connection ghosted us — come back in a moment 👻")));
+    socket.on("error", () => this.#fail(new GatewayError("transport_unavailable", 502, "WebSocket connection failed (request {requestId}). The connection ghosted us — come back in a moment 👻")));
+  }
+
+  send(message: string): void {
+    if (this.#closed) throw this.#closed;
+    if (this.#socket.readyState !== WebSocket.OPEN) {
+      throw new GatewayError("transport_closed", 499, "WebSocket connection is not open (request {requestId}). Send it again whenever ready 🙃");
+    }
+    this.#socket.send(message, { binary: false });
+  }
+
+  receive(signal: AbortSignal): Promise<string> {
+    const queued = this.#messages.shift();
+    if (queued !== undefined) return Promise.resolve(queued);
+    if (this.#closed) return Promise.reject(this.#closed);
+    const combined = AbortSignal.any([this.#signal, signal]);
+    return new Promise<string>((resolve, reject) => {
+      const timeout = setTimeout(() => {
+        cleanup();
+        reject(new GatewayError("transport_unavailable", 504, "WebSocket receive timed out (request {requestId}). Some things just take longer 🐢"));
+      }, 45_000);
+      const onAbort = (): void => {
+        cleanup();
+        reject(new GatewayError("transport_closed", 499, "WebSocket receive was cancelled (request {requestId}). Send it again whenever ready 🙃"));
+      };
+      const cleanup = (): void => {
+        clearTimeout(timeout);
+        combined.removeEventListener("abort", onAbort);
+      };
+      combined.addEventListener("abort", onAbort, { once: true });
+      this.#waiters.push({ resolve, reject, cleanup });
+      if (combined.aborted) onAbort();
+    });
+  }
+
+  close(code?: number, reason?: string): void {
+    if (this.#socket.readyState === WebSocket.CLOSED) return;
+    this.#socket.close(code, reason);
+  }
+
+  #fail(error: Error): void {
+    if (this.#closed) return;
+    this.#closed = error;
+    for (const waiter of this.#waiters.splice(0)) {
+      waiter.cleanup();
+      waiter.reject(error);
+    }
+  }
+}
+
+function connectValidatedWebSocket(
+  url: URL,
+  headers: Readonly<Record<string, string>>,
+  signal: AbortSignal,
+  destination: ValidatedDestination,
+  agent?: PoolAgent,
+): Promise<ProviderWebSocketSession> {
+  const targetUrl = new URL(url.toString());
+  targetUrl.protocol = "wss:";
+  targetUrl.port = "";
+  targetUrl.hash = "";
+  const connectionOptions: ClientOptions = {
+    handshakeTimeout: 15_000,
+    perMessageDeflate: false,
+    followRedirects: false,
+    headers: { ...headers },
+    ...(agent === undefined ? {} : { agent: agentForWebSocket(agent) }),
+    ...(agent === undefined
+      ? {
+          createConnection: () => {
+            const socket = tls.connect({
+              host: destination.resolvedAddress,
+              port: 443,
+              servername: destination.hostname,
+              rejectUnauthorized: true,
+              ALPNProtocols: ["http/1.1"],
+            });
+            const onAbort = (): void => {
+              socket.destroy(new Error("connection aborted"));
+            };
+            signal.addEventListener("abort", onAbort, { once: true });
+            socket.once("secureConnect", () => signal.removeEventListener("abort", onAbort));
+            return socket;
+          },
+        }
+      : {}),
+  };
+  return new Promise<ProviderWebSocketSession>((resolve, reject) => {
+    const socket = new WebSocket(targetUrl, connectionOptions);
+    let settled = false;
+    const timeout = setTimeout(() => {
+      if (settled) return;
+      settled = true;
+      socket.terminate();
+      reject(new GatewayError("transport_unavailable", 502, "WebSocket handshake timed out (request {requestId}). Some things just take longer 🐢"));
+    }, 15_000);
+    const abort = (): void => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timeout);
+      socket.terminate();
+      reject(new GatewayError("transport_closed", 499, "WebSocket connection was cancelled (request {requestId}). Send it again whenever ready 🙃"));
+    };
+    const onError = (): void => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timeout);
+      signal.removeEventListener("abort", abort);
+      reject(new GatewayError("transport_unavailable", 502, "WebSocket connection failed (request {requestId}). The connection ghosted us — come back in a moment 👻"));
+    };
+    socket.once("open", () => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timeout);
+      signal.removeEventListener("abort", abort);
+      resolve(new WsSession(socket, signal));
+    });
+    socket.once("error", onError);
+    socket.once("unexpected-response", onError);
+    signal.addEventListener("abort", abort, { once: true });
+    if (signal.aborted) abort();
+  });
+}
+
+function agentForWebSocket(agent: PoolAgent): import("http").Agent | import("https").Agent {
+  if (isProxyAgentPair(agent)) return agent.https;
+  if (agent instanceof HttpsAgent) return agent;
+  throw new GatewayError("proxy_pool_unhealthy", 503, "configured WebSocket proxy pool does not support WSS (request {requestId}). This bridge doesn't speak WSS — reconfigure the pool 🔧");
+}
+
+export { connectValidatedWebSocket, WsSession };

package/src/network/pool/{byte-accounting.ts → accounting.ts}

@@ -13,6 +13,8 @@
  * byte, and nothing is allocated.
  */
 
+import { evictOldestEntry } from "../../transport/routing/bounded-lru";
+
 /** Running totals for one pool. */
 interface PoolByteTotals {
   /** Bytes sent to the proxy (uploads, TLS handshake, CONNECT). */
@@ -33,8 +35,7 @@ function entry(poolId: string): { sent: number; received: number } {
   let current = totals.get(poolId);
   if (!current) {
     if (totals.size >= MAX_BYTE_ENTRY_COUNT) {
-      const oldest = totals.keys().next().value;
-      if (oldest !== undefined) totals.delete(oldest);
+      evictOldestEntry(totals, MAX_BYTE_ENTRY_COUNT - 1);
     }
     current = { sent: 0, received: 0 };
     totals.set(poolId, current);

package/src/network/{types.ts → pool/agent-config.ts}

@@ -11,7 +11,7 @@
  * and the agent resolver, so a malformed row fails loudly instead of producing
  * a silently-wrong agent.
  */
-import { isRecord } from "../protocol/codecs";
+import { isRecord } from "../../protocol/codecs";
 
 /** Thrown when a stored/requested pool config cannot be decoded. */
 export class ProxyConfigError extends Error {

package/src/network/pool/agent.ts

@@ -4,7 +4,7 @@
  * caches instances of these; the selector owns admission.
  */
 import { isIP } from "node:net";
-import { accountSocketBytes, accountSocketWrites, recordPoolBytes } from "./byte-accounting";
+import { accountSocketBytes, accountSocketWrites, recordPoolBytes } from "./accounting";
 import {
   Agent as HttpAgent,
   request as httpRequest,
@@ -19,7 +19,7 @@ import { SocksProxyAgent } from "socks-proxy-agent";
 import { resolveAllAddresses, validateResolvedAddresses } from "../ssrf";
 import type { SsrfPolicy } from "../../config/network";
 import { resolveProxyMaxFreeSockets, resolveProxyMaxSockets } from "../../config/network";
-import type { AgentConfig } from "../types";
+import type { AgentConfig } from "./agent-config";
 
 /**
  * Idle keep-alive socket timeout for every pool agent. Matches the 60s DB pool

package/src/network/{pool-cooldown-store.ts → pool/cooldown-store.ts}

@@ -14,12 +14,13 @@
  */
 import { randomUUID } from "node:crypto";
 import { eq } from "drizzle-orm";
-import type { CartethyiaDatabase } from "../persistence/db-handle";
-import { networkPools, healthEvents } from "../persistence/schema";
-import { log } from "../observability/logger";
-import { providerRateLimitIsIpScoped } from "../providers/provider-metadata";
-import { resolvePoolProviderRateLimitCooldownMs } from "../config/cooldowns";
-import { PROVIDER_BLOCK_MS, type ProxyCooldownEntry } from "./pool/selector";
+import type { CartethyiaDatabase } from "../../persistence/db-handle";
+import { networkPools, healthEvents } from "../../persistence/schema";
+import { log } from "../../observability/logger";
+import { metrics } from "../../observability/metrics";
+import { providerRateLimitIsIpScoped } from "../../providers/provider-metadata";
+import { resolvePoolProviderRateLimitCooldownMs } from "../../config/cooldowns";
+import { PROVIDER_BLOCK_MS, type ProxyCooldownEntry } from "./selector";
 
 type PoolProviderCooldownMap = NonNullable<
   (typeof networkPools.$inferSelect)["providerCooldowns"]
@@ -152,6 +153,88 @@ export async function pruneExpiredProviderCooldownMarkers(
   }
 }
 
+/** Max simultaneously tracked (pool, provider) coalescing keys. */
+const MAX_TRACKED_PAIRS = 5_000;
+
+/** Last coalesced row timestamp per (pool, provider), for the cooldown window only. */
+const lastRecordedByPoolProvider = new Map<string, number>();
+
+/**
+ * Appends one provider-scoped egress verdict row to `health_events` — the
+ * durable audit log the console's pool page reads.
+ *
+ * Shared by the three writers of this row shape: the dispatch-side 429
+ * recorder (`recordPoolCooldownEvent`), the egress-block recorder
+ * (`recordPoolBlockEvent`), and the probe verdict recorder
+ * (`recordProbeVerdict`). The row is always `entityKind: "pool"`,
+ * `fromStatus: null`, `toStatus: "cooldown"`, and the category is chosen by
+ * `blocked`; the only divergences are the optional code suffix (block path)
+ * and the optional coalescing window (cooldown path).
+ *
+ * Coalescing: `coalesceWindowMs` suppresses a duplicate row for the same
+ * (pool, provider) pair within the window — used by the cooldown path, where
+ * dispatch flags the same pair on every throttled request and the audit log
+ * needs one row per episode, not one per request. The default `undefined`
+ * means no coalescing, so block and probe rows are never deduped; it also
+ * means the pair never enters the tracking map, so the eviction sweep below
+ * only ever sees coalesced pairs. A failed insert does not stamp the pair, so
+ * the next attempt within the window is still tried.
+ *
+ * Swallow contract: this never throws to dispatch (observability writes are
+ * non-fatal); failures bump `pool_cooldown_record_failed` and are logged.
+ */
+export async function appendPoolProviderEgressAuditRow(
+  db: CartethyiaDatabase,
+  opts: {
+    poolId: string;
+    providerId: string;
+    reason: string;
+    blocked: boolean;
+    codeSuffix?: string;
+    coalesceWindowMs?: number;
+  },
+): Promise<void> {
+  const { poolId, providerId, reason, blocked, codeSuffix, coalesceWindowMs } = opts;
+  let key: string | undefined;
+  let now = 0;
+  if (coalesceWindowMs !== undefined) {
+    key = `${poolId}::${providerId.toLowerCase()}`;
+    now = Date.now();
+    const last = lastRecordedByPoolProvider.get(key);
+    if (last !== undefined && now - last < coalesceWindowMs) return;
+  }
+  try {
+    await db.insert(healthEvents).values({
+      id: randomUUID(),
+      entityKind: "pool",
+      networkPoolId: poolId,
+      fromStatus: null,
+      toStatus: "cooldown",
+      reason: `provider ${providerId}: ${reason}${codeSuffix ?? ""}`,
+      errorCategory: blocked ? "egress_blocked" : "rate_limit_transient",
+      createdAt: new Date(),
+    });
+  } catch (error) {
+    metrics.pool_cooldown_record_failed.inc(1);
+    log.warn("pool provider egress audit row failed", {
+      networkPoolId: poolId,
+      providerId,
+      error: String(error),
+    });
+    return;
+  }
+  if (key !== undefined && coalesceWindowMs !== undefined) {
+    lastRecordedByPoolProvider.set(key, now);
+    if (lastRecordedByPoolProvider.size > MAX_TRACKED_PAIRS) {
+      for (const [trackedKey, recordedAt] of lastRecordedByPoolProvider) {
+        if (now - recordedAt >= coalesceWindowMs) {
+          lastRecordedByPoolProvider.delete(trackedKey);
+        }
+      }
+    }
+  }
+}
+
 /**
  * Writes a probe-measured verdict the same way a dispatch-side event would be
  * recorded: one mirror marker and one audit row.
@@ -174,20 +257,12 @@ export async function recordProbeVerdict(
   marker: { until: number; reason: string; blocked?: boolean },
 ): Promise<void> {
   await upsertProviderCooldownMarker(db, poolId, providerId, marker);
-  try {
-    await db.insert(healthEvents).values({
-      id: randomUUID(),
-      entityKind: "pool",
-      networkPoolId: poolId,
-      fromStatus: null,
-      toStatus: "cooldown",
-      reason: `provider ${providerId}: ${marker.reason}`,
-      errorCategory: marker.blocked === true ? "egress_blocked" : "rate_limit_transient",
-      createdAt: new Date(),
-    });
-  } catch (error) {
-    log.warn("pool probe event failed", { networkPoolId: poolId, providerId, error: String(error) });
-  }
+  await appendPoolProviderEgressAuditRow(db, {
+    poolId,
+    providerId,
+    reason: marker.reason,
+    blocked: marker.blocked === true,
+  });
 }
 
 /**

package/src/network/{pool-health-machine.ts → pool/health.ts}

@@ -1,11 +1,210 @@
+/**
+ * Pool health — flag helpers + state machine.
+ *
+ * Two halves of one domain, kept in one home:
+ *
+ * - **Flag helpers** (`flagPoolBlocked`, `flagPoolCooldown`,
+ *   `flagPoolOnEgressFailure`): durable per-(pool, provider) 429/block
+ *   history. The volatile flags live in `NetworkPoolSelector` (fast, consulted
+ *   on every dispatch); these mirror each flag into two durable homes — the
+ *   append-only `health_events` table (`entity_kind: "pool"`) for audit, and
+ *   the `provider_cooldowns` column as the state boot restores. They never
+ *   touch `network_pools.status`: a single provider's 429 must not flip the
+ *   whole pool to `cooldown`.
+ *
+ * - **State machine** (`recordPoolDispatchOutcome`, `disablePoolForProxyHttpStatus`,
+ *   `recoverNetworkPool`, `sweepExpiredPoolCooldowns`): the pool row's health
+ *   transitions, subscriber feed, and recovery sweep.
+ */
 import { randomUUID } from "node:crypto";
-import { healthEventBuffer } from "../persistence/health-event-flush";
 import { and, desc, eq, inArray, isNotNull, lte, sql } from "drizzle-orm";
-import type { CartethyiaDatabase } from "../persistence/db-handle";
-import { healthEvents, networkPools } from "../persistence/schema";
-import { drainPoolByteDelta } from "./pool/byte-accounting";
-import { resolvePoolCooldownMs } from "../config/cooldowns";
-import { dropProviderCooldownMarker, pruneExpiredProviderCooldownMarkers } from "./pool-cooldown-store";
+import type { CartethyiaDatabase } from "../../persistence/db-handle";
+import { healthEventBuffer } from "../../persistence/health-event-flush";
+import { healthEvents, networkPools } from "../../persistence/schema";
+import { GatewayError } from "../../protocol/gateway-error";
+import { metrics } from "../../observability/metrics";
+import { log } from "../../observability/logger";
+import { resolvePoolProviderRateLimitCooldownMs, resolvePoolCooldownMs } from "../../config/cooldowns";
+import type { NetworkPoolSelector } from "./selector";
+import { appendPoolProviderEgressAuditRow, dropProviderCooldownMarker, pruneExpiredProviderCooldownMarkers, upsertProviderCooldownMarker } from "./cooldown-store";
+import { drainPoolByteDelta } from "./accounting";
+import type { EgressVerdict } from "../verdict";
+
+/**
+ * Coalescing window for repeated per-(pool, provider) 429 events. Dispatch
+ * flags the same pair on every throttled request, but the durable audit log
+ * needs one row per cooldown episode — not one per throttled request.
+ */
+const POOL_COOLDOWN_EVENT_WINDOW_MS = 15 * 60 * 1000;
+
+/** Durable half of `flagPoolCooldown`; not a second public entry point. */
+async function recordPoolCooldownEvent(
+  db: CartethyiaDatabase,
+  poolId: string,
+  providerId: string,
+  reason: string,
+): Promise<void> {
+  await appendPoolProviderEgressAuditRow(db, {
+    poolId,
+    providerId,
+    reason,
+    blocked: false,
+    coalesceWindowMs: POOL_COOLDOWN_EVENT_WINDOW_MS,
+  });
+}
+
+/**
+ * Retires a pool for one provider because the upstream flagged its egress.
+ *
+ * Separate from `flagPoolCooldown` on purpose: the two facts are not
+ * interchangeable. A 429 is an allowance that refills, so the pool returns to
+ * rotation when the window ends. A free-tier refusal means the address itself
+ * is retired upstream, and putting it back would spend a real request to learn
+ * the same answer. The audit row therefore carries `errorCategory:
+ * "egress_blocked"` rather than `rate_limit_transient`, which is what the
+ * console reads to draw it as a block instead of a cooldown.
+ */
+export function flagPoolBlocked(
+  poolSelector: NetworkPoolSelector,
+  db: CartethyiaDatabase,
+  poolId: string,
+  providerId: string,
+  error: GatewayError,
+): void {
+  const reason = `Provider ${providerId} refused this egress`;
+  void poolSelector
+    .flagProviderBlocked(poolId, providerId, reason)
+    .then((entry) =>
+      // Persisted even when the flag was refused (a credential-keyed
+      // provider): the selector returned an inert entry, so the mirror
+      // records nothing a reader would act on.
+      upsertProviderCooldownMarker(db, poolId, providerId, entry),
+    )
+    .catch((cause: unknown) => {
+      metrics.pool_cooldown_record_failed.inc(1);
+      log.warn("pool block flag failed", {
+        networkPoolId: poolId,
+        providerId,
+        error: String(cause),
+      });
+    });
+  void recordPoolBlockEvent(db, poolId, providerId, reason, error);
+}
+
+/**
+ * Durable half of `flagPoolBlocked`.
+ *
+ * Not coalesced the way a cooldown is: a block happens once per (pool,
+ * provider) for the life of the deployment, so there is no burst to collapse.
+ * A repeated event means the operator re-enabled the pool and it was refused
+ * again, which is worth a second row.
+ */
+async function recordPoolBlockEvent(
+  db: CartethyiaDatabase,
+  poolId: string,
+  providerId: string,
+  reason: string,
+  error: GatewayError,
+): Promise<void> {
+  await appendPoolProviderEgressAuditRow(db, {
+    poolId,
+    providerId,
+    reason,
+    blocked: true,
+    codeSuffix: providerCodeSuffix(error),
+  });
+}
+
+/** The upstream's own code, when it named one, so the audit row is actionable. */
+function providerCodeSuffix(error: GatewayError): string {
+  const code = error.details.providerCode;
+  return typeof code === "string" && code.length > 0 ? ` (${code})` : "";
+}
+
+/** Cooldown applied when an upstream 429 carries no `retryAfterMs`. */
+const DEFAULT_PROVIDER_COOLDOWN_MS = 15 * 60 * 1000;
+
+/**
+ * The fallback window for a provider-scoped egress cooldown. Upstream `Retry-After`
+ * always wins; the configured default exists because a request-counted free
+ * tier — the one provider that flags at all — stays refused for hours after
+ * the 15-minute guess this function used before.
+ */
+function providerCooldownFallbackMs(): number {
+  return Math.max(resolvePoolProviderRateLimitCooldownMs(), DEFAULT_PROVIDER_COOLDOWN_MS);
+}
+
+/**
+ * Records a provider-scoped 429 cooldown: the volatile per-(pool, provider)
+ * flag consulted on every dispatch, its durable audit row, and the durable
+ * state boot restores from.
+ *
+ * Both paths that reach this decision — the non-streaming attempt loop and
+ * the streaming failure path — call this one function, because each write
+ * reports its own failure through the same metric and warn log. Two copies
+ * would have to be edited together, and a divergence would surface only as
+ * telemetry that silently differs by response mode.
+ *
+ * Callers gate on `shouldCooldownPool`, which narrows the error to the
+ * upstream/provider-scoped 429 this function is written for.
+ */
+export function flagPoolCooldown(
+  poolSelector: NetworkPoolSelector,
+  db: CartethyiaDatabase,
+  poolId: string,
+  providerId: string,
+  error: GatewayError,
+): void {
+  const reason = `Provider ${providerId} rate limit`;
+  const retryAfterMs =
+    typeof error.details.retryAfterMs === "number"
+      ? error.details.retryAfterMs
+      : providerCooldownFallbackMs();
+  void poolSelector
+    .flagProviderCooldown(poolId, providerId, retryAfterMs, reason)
+    .then((entry) => upsertProviderCooldownMarker(db, poolId, providerId, entry))
+    .catch((cause: unknown) => {
+      metrics.pool_cooldown_record_failed.inc(1);
+      log.warn("pool cooldown flag failed", {
+        networkPoolId: poolId,
+        providerId,
+        error: String(cause),
+      });
+    });
+  void recordPoolCooldownEvent(db, poolId, providerId, reason);
+}
+
+/**
+ * One egress-failure verdict → pool-flag dispatch, shared by the attempt loop
+ * and the streaming finalizer.
+ *
+ * Both paths classify the same failure with `classifyEgressFailure`, then must
+ * retire the same pool the same way; a policy change — which verdict maps to a
+ * cooldown vs a block, or a new verdict — belongs here so the two response
+ * modes cannot drift in telemetry.
+ *
+ * The call sites compute the verdict (the attempt loop also needs it for its
+ * retry decision) and pass it in; the gating is identical to what both
+ * previously applied by hand: only a real pool id + selector pair, only a
+ * `GatewayError`, and never a `"none"` verdict. The flags themselves are
+ * fire-and-forget advisory writes that never fail the request.
+ */
+export function flagPoolOnEgressFailure(
+  poolSelector: NetworkPoolSelector | undefined,
+  db: CartethyiaDatabase,
+  poolId: string | undefined,
+  providerId: string,
+  error: unknown,
+  verdict: EgressVerdict,
+): void {
+  if (poolId === undefined || poolSelector === undefined || !(error instanceof GatewayError)) return;
+  if (verdict.kind === "cooldown") {
+    flagPoolCooldown(poolSelector, db, poolId, providerId, error);
+  } else if (verdict.kind === "blocked") {
+    flagPoolBlocked(poolSelector, db, poolId, providerId, error);
+  }
+}
+
 export interface PoolHealthEvent {
   readonly id: string;
   readonly networkPoolId: string;