cartethyia 2.8.0 → 2.8.1

package/src/network/egress/ws.ts

@@ -0,0 +1,175 @@
+/**
+ * Validated WebSocket egress — WSS session + the dial that opens one.
+ *
+ * The dial resolves the target through the SSRF policy (done by the caller,
+ * the binding factory) and then either connects through the selected pool
+ * agent's tunnel or dials direct with pinned, SSRF-validated address.
+ */
+import * as tls from "node:tls";
+import WebSocket, { type ClientOptions, type RawData } from "ws";
+import { GatewayError } from "../../protocol/gateway-error";
+import { isProxyAgentPair, type PoolAgent } from "../pool/agent";
+import { Agent as HttpsAgent } from "node:https";
+import type { ProviderWebSocketSession } from "../../providers/provider-contracts";
+import type { ValidatedDestination } from "../ssrf";
+
+class WsSession implements ProviderWebSocketSession {
+  readonly #socket: WebSocket;
+  readonly #signal: AbortSignal;
+  readonly #messages: string[] = [];
+  readonly #waiters: Array<{
+    resolve(value: string): void;
+    reject(reason: Error): void;
+    cleanup(): void;
+  }> = [];
+  #closed: Error | undefined;
+
+  constructor(socket: WebSocket, signal: AbortSignal) {
+    this.#socket = socket;
+    this.#signal = signal;
+    socket.on("message", (data: RawData, isBinary: boolean) => {
+      if (isBinary) {
+        this.#fail(new GatewayError("transport_unavailable", 502, "unexpected binary WebSocket frame (request {requestId}). The connection ghosted us — come back in a moment 👻"));
+        return;
+      }
+      const message = data.toString();
+      const waiter = this.#waiters.shift();
+      if (waiter) {
+        waiter.cleanup();
+        waiter.resolve(message);
+      } else {
+        this.#messages.push(message);
+      }
+    });
+    socket.on("close", () => this.#fail(new GatewayError("transport_closed", 502, "WebSocket connection closed (request {requestId}). The connection ghosted us — come back in a moment 👻")));
+    socket.on("error", () => this.#fail(new GatewayError("transport_unavailable", 502, "WebSocket connection failed (request {requestId}). The connection ghosted us — come back in a moment 👻")));
+  }
+
+  send(message: string): void {
+    if (this.#closed) throw this.#closed;
+    if (this.#socket.readyState !== WebSocket.OPEN) {
+      throw new GatewayError("transport_closed", 499, "WebSocket connection is not open (request {requestId}). Send it again whenever ready 🙃");
+    }
+    this.#socket.send(message, { binary: false });
+  }
+
+  receive(signal: AbortSignal): Promise<string> {
+    const queued = this.#messages.shift();
+    if (queued !== undefined) return Promise.resolve(queued);
+    if (this.#closed) return Promise.reject(this.#closed);
+    const combined = AbortSignal.any([this.#signal, signal]);
+    return new Promise<string>((resolve, reject) => {
+      const timeout = setTimeout(() => {
+        cleanup();
+        reject(new GatewayError("transport_unavailable", 504, "WebSocket receive timed out (request {requestId}). Some things just take longer 🐢"));
+      }, 45_000);
+      const onAbort = (): void => {
+        cleanup();
+        reject(new GatewayError("transport_closed", 499, "WebSocket receive was cancelled (request {requestId}). Send it again whenever ready 🙃"));
+      };
+      const cleanup = (): void => {
+        clearTimeout(timeout);
+        combined.removeEventListener("abort", onAbort);
+      };
+      combined.addEventListener("abort", onAbort, { once: true });
+      this.#waiters.push({ resolve, reject, cleanup });
+      if (combined.aborted) onAbort();
+    });
+  }
+
+  close(code?: number, reason?: string): void {
+    if (this.#socket.readyState === WebSocket.CLOSED) return;
+    this.#socket.close(code, reason);
+  }
+
+  #fail(error: Error): void {
+    if (this.#closed) return;
+    this.#closed = error;
+    for (const waiter of this.#waiters.splice(0)) {
+      waiter.cleanup();
+      waiter.reject(error);
+    }
+  }
+}
+
+function connectValidatedWebSocket(
+  url: URL,
+  headers: Readonly<Record<string, string>>,
+  signal: AbortSignal,
+  destination: ValidatedDestination,
+  agent?: PoolAgent,
+): Promise<ProviderWebSocketSession> {
+  const targetUrl = new URL(url.toString());
+  targetUrl.protocol = "wss:";
+  targetUrl.port = "";
+  targetUrl.hash = "";
+  const connectionOptions: ClientOptions = {
+    handshakeTimeout: 15_000,
+    perMessageDeflate: false,
+    followRedirects: false,
+    headers: { ...headers },
+    ...(agent === undefined ? {} : { agent: agentForWebSocket(agent) }),
+    ...(agent === undefined
+      ? {
+          createConnection: () => {
+            const socket = tls.connect({
+              host: destination.resolvedAddress,
+              port: 443,
+              servername: destination.hostname,
+              rejectUnauthorized: true,
+              ALPNProtocols: ["http/1.1"],
+            });
+            const onAbort = (): void => {
+              socket.destroy(new Error("connection aborted"));
+            };
+            signal.addEventListener("abort", onAbort, { once: true });
+            socket.once("secureConnect", () => signal.removeEventListener("abort", onAbort));
+            return socket;
+          },
+        }
+      : {}),
+  };
+  return new Promise<ProviderWebSocketSession>((resolve, reject) => {
+    const socket = new WebSocket(targetUrl, connectionOptions);
+    let settled = false;
+    const timeout = setTimeout(() => {
+      if (settled) return;
+      settled = true;
+      socket.terminate();
+      reject(new GatewayError("transport_unavailable", 502, "WebSocket handshake timed out (request {requestId}). Some things just take longer 🐢"));
+    }, 15_000);
+    const abort = (): void => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timeout);
+      socket.terminate();
+      reject(new GatewayError("transport_closed", 499, "WebSocket connection was cancelled (request {requestId}). Send it again whenever ready 🙃"));
+    };
+    const onError = (): void => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timeout);
+      signal.removeEventListener("abort", abort);
+      reject(new GatewayError("transport_unavailable", 502, "WebSocket connection failed (request {requestId}). The connection ghosted us — come back in a moment 👻"));
+    };
+    socket.once("open", () => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timeout);
+      signal.removeEventListener("abort", abort);
+      resolve(new WsSession(socket, signal));
+    });
+    socket.once("error", onError);
+    socket.once("unexpected-response", onError);
+    signal.addEventListener("abort", abort, { once: true });
+    if (signal.aborted) abort();
+  });
+}
+
+function agentForWebSocket(agent: PoolAgent): import("http").Agent | import("https").Agent {
+  if (isProxyAgentPair(agent)) return agent.https;
+  if (agent instanceof HttpsAgent) return agent;
+  throw new GatewayError("proxy_pool_unhealthy", 503, "configured WebSocket proxy pool does not support WSS (request {requestId}). This bridge doesn't speak WSS — reconfigure the pool 🔧");
+}
+
+export { connectValidatedWebSocket, WsSession };

package/src/network/{types.ts → pool/agent-config.ts}

@@ -11,7 +11,7 @@
  * and the agent resolver, so a malformed row fails loudly instead of producing
  * a silently-wrong agent.
  */
-import { isRecord } from "../protocol/codecs";
+import { isRecord } from "../../protocol/codecs";
 
 /** Thrown when a stored/requested pool config cannot be decoded. */
 export class ProxyConfigError extends Error {

package/src/network/pool/agent.ts

@@ -4,7 +4,7 @@
  * caches instances of these; the selector owns admission.
  */
 import { isIP } from "node:net";
-import { accountSocketBytes, accountSocketWrites, recordPoolBytes } from "./byte-accounting";
+import { accountSocketBytes, accountSocketWrites, recordPoolBytes } from "./accounting";
 import {
   Agent as HttpAgent,
   request as httpRequest,
@@ -19,7 +19,7 @@ import { SocksProxyAgent } from "socks-proxy-agent";
 import { resolveAllAddresses, validateResolvedAddresses } from "../ssrf";
 import type { SsrfPolicy } from "../../config/network";
 import { resolveProxyMaxFreeSockets, resolveProxyMaxSockets } from "../../config/network";
-import type { AgentConfig } from "../types";
+import type { AgentConfig } from "./agent-config";
 
 /**
  * Idle keep-alive socket timeout for every pool agent. Matches the 60s DB pool

package/src/network/{pool-cooldown-store.ts → pool/cooldown-store.ts}

@@ -14,13 +14,13 @@
  */
 import { randomUUID } from "node:crypto";
 import { eq } from "drizzle-orm";
-import type { CartethyiaDatabase } from "../persistence/db-handle";
-import { networkPools, healthEvents } from "../persistence/schema";
-import { log } from "../observability/logger";
-import { metrics } from "../observability/metrics";
-import { providerRateLimitIsIpScoped } from "../providers/provider-metadata";
-import { resolvePoolProviderRateLimitCooldownMs } from "../config/cooldowns";
-import { PROVIDER_BLOCK_MS, type ProxyCooldownEntry } from "./pool/selector";
+import type { CartethyiaDatabase } from "../../persistence/db-handle";
+import { networkPools, healthEvents } from "../../persistence/schema";
+import { log } from "../../observability/logger";
+import { metrics } from "../../observability/metrics";
+import { providerRateLimitIsIpScoped } from "../../providers/provider-metadata";
+import { resolvePoolProviderRateLimitCooldownMs } from "../../config/cooldowns";
+import { PROVIDER_BLOCK_MS, type ProxyCooldownEntry } from "./selector";
 
 type PoolProviderCooldownMap = NonNullable<
   (typeof networkPools.$inferSelect)["providerCooldowns"]

package/src/network/{pool-health-machine.ts → pool/health.ts}

@@ -1,11 +1,210 @@
+/**
+ * Pool health — flag helpers + state machine.
+ *
+ * Two halves of one domain, kept in one home:
+ *
+ * - **Flag helpers** (`flagPoolBlocked`, `flagPoolCooldown`,
+ *   `flagPoolOnEgressFailure`): durable per-(pool, provider) 429/block
+ *   history. The volatile flags live in `NetworkPoolSelector` (fast, consulted
+ *   on every dispatch); these mirror each flag into two durable homes — the
+ *   append-only `health_events` table (`entity_kind: "pool"`) for audit, and
+ *   the `provider_cooldowns` column as the state boot restores. They never
+ *   touch `network_pools.status`: a single provider's 429 must not flip the
+ *   whole pool to `cooldown`.
+ *
+ * - **State machine** (`recordPoolDispatchOutcome`, `disablePoolForProxyHttpStatus`,
+ *   `recoverNetworkPool`, `sweepExpiredPoolCooldowns`): the pool row's health
+ *   transitions, subscriber feed, and recovery sweep.
+ */
 import { randomUUID } from "node:crypto";
-import { healthEventBuffer } from "../persistence/health-event-flush";
 import { and, desc, eq, inArray, isNotNull, lte, sql } from "drizzle-orm";
-import type { CartethyiaDatabase } from "../persistence/db-handle";
-import { healthEvents, networkPools } from "../persistence/schema";
-import { drainPoolByteDelta } from "./pool/byte-accounting";
-import { resolvePoolCooldownMs } from "../config/cooldowns";
-import { dropProviderCooldownMarker, pruneExpiredProviderCooldownMarkers } from "./pool-cooldown-store";
+import type { CartethyiaDatabase } from "../../persistence/db-handle";
+import { healthEventBuffer } from "../../persistence/health-event-flush";
+import { healthEvents, networkPools } from "../../persistence/schema";
+import { GatewayError } from "../../protocol/gateway-error";
+import { metrics } from "../../observability/metrics";
+import { log } from "../../observability/logger";
+import { resolvePoolProviderRateLimitCooldownMs, resolvePoolCooldownMs } from "../../config/cooldowns";
+import type { NetworkPoolSelector } from "./selector";
+import { appendPoolProviderEgressAuditRow, dropProviderCooldownMarker, pruneExpiredProviderCooldownMarkers, upsertProviderCooldownMarker } from "./cooldown-store";
+import { drainPoolByteDelta } from "./accounting";
+import type { EgressVerdict } from "../verdict";
+
+/**
+ * Coalescing window for repeated per-(pool, provider) 429 events. Dispatch
+ * flags the same pair on every throttled request, but the durable audit log
+ * needs one row per cooldown episode — not one per throttled request.
+ */
+const POOL_COOLDOWN_EVENT_WINDOW_MS = 15 * 60 * 1000;
+
+/** Durable half of `flagPoolCooldown`; not a second public entry point. */
+async function recordPoolCooldownEvent(
+  db: CartethyiaDatabase,
+  poolId: string,
+  providerId: string,
+  reason: string,
+): Promise<void> {
+  await appendPoolProviderEgressAuditRow(db, {
+    poolId,
+    providerId,
+    reason,
+    blocked: false,
+    coalesceWindowMs: POOL_COOLDOWN_EVENT_WINDOW_MS,
+  });
+}
+
+/**
+ * Retires a pool for one provider because the upstream flagged its egress.
+ *
+ * Separate from `flagPoolCooldown` on purpose: the two facts are not
+ * interchangeable. A 429 is an allowance that refills, so the pool returns to
+ * rotation when the window ends. A free-tier refusal means the address itself
+ * is retired upstream, and putting it back would spend a real request to learn
+ * the same answer. The audit row therefore carries `errorCategory:
+ * "egress_blocked"` rather than `rate_limit_transient`, which is what the
+ * console reads to draw it as a block instead of a cooldown.
+ */
+export function flagPoolBlocked(
+  poolSelector: NetworkPoolSelector,
+  db: CartethyiaDatabase,
+  poolId: string,
+  providerId: string,
+  error: GatewayError,
+): void {
+  const reason = `Provider ${providerId} refused this egress`;
+  void poolSelector
+    .flagProviderBlocked(poolId, providerId, reason)
+    .then((entry) =>
+      // Persisted even when the flag was refused (a credential-keyed
+      // provider): the selector returned an inert entry, so the mirror
+      // records nothing a reader would act on.
+      upsertProviderCooldownMarker(db, poolId, providerId, entry),
+    )
+    .catch((cause: unknown) => {
+      metrics.pool_cooldown_record_failed.inc(1);
+      log.warn("pool block flag failed", {
+        networkPoolId: poolId,
+        providerId,
+        error: String(cause),
+      });
+    });
+  void recordPoolBlockEvent(db, poolId, providerId, reason, error);
+}
+
+/**
+ * Durable half of `flagPoolBlocked`.
+ *
+ * Not coalesced the way a cooldown is: a block happens once per (pool,
+ * provider) for the life of the deployment, so there is no burst to collapse.
+ * A repeated event means the operator re-enabled the pool and it was refused
+ * again, which is worth a second row.
+ */
+async function recordPoolBlockEvent(
+  db: CartethyiaDatabase,
+  poolId: string,
+  providerId: string,
+  reason: string,
+  error: GatewayError,
+): Promise<void> {
+  await appendPoolProviderEgressAuditRow(db, {
+    poolId,
+    providerId,
+    reason,
+    blocked: true,
+    codeSuffix: providerCodeSuffix(error),
+  });
+}
+
+/** The upstream's own code, when it named one, so the audit row is actionable. */
+function providerCodeSuffix(error: GatewayError): string {
+  const code = error.details.providerCode;
+  return typeof code === "string" && code.length > 0 ? ` (${code})` : "";
+}
+
+/** Cooldown applied when an upstream 429 carries no `retryAfterMs`. */
+const DEFAULT_PROVIDER_COOLDOWN_MS = 15 * 60 * 1000;
+
+/**
+ * The fallback window for a provider-scoped egress cooldown. Upstream `Retry-After`
+ * always wins; the configured default exists because a request-counted free
+ * tier — the one provider that flags at all — stays refused for hours after
+ * the 15-minute guess this function used before.
+ */
+function providerCooldownFallbackMs(): number {
+  return Math.max(resolvePoolProviderRateLimitCooldownMs(), DEFAULT_PROVIDER_COOLDOWN_MS);
+}
+
+/**
+ * Records a provider-scoped 429 cooldown: the volatile per-(pool, provider)
+ * flag consulted on every dispatch, its durable audit row, and the durable
+ * state boot restores from.
+ *
+ * Both paths that reach this decision — the non-streaming attempt loop and
+ * the streaming failure path — call this one function, because each write
+ * reports its own failure through the same metric and warn log. Two copies
+ * would have to be edited together, and a divergence would surface only as
+ * telemetry that silently differs by response mode.
+ *
+ * Callers gate on `shouldCooldownPool`, which narrows the error to the
+ * upstream/provider-scoped 429 this function is written for.
+ */
+export function flagPoolCooldown(
+  poolSelector: NetworkPoolSelector,
+  db: CartethyiaDatabase,
+  poolId: string,
+  providerId: string,
+  error: GatewayError,
+): void {
+  const reason = `Provider ${providerId} rate limit`;
+  const retryAfterMs =
+    typeof error.details.retryAfterMs === "number"
+      ? error.details.retryAfterMs
+      : providerCooldownFallbackMs();
+  void poolSelector
+    .flagProviderCooldown(poolId, providerId, retryAfterMs, reason)
+    .then((entry) => upsertProviderCooldownMarker(db, poolId, providerId, entry))
+    .catch((cause: unknown) => {
+      metrics.pool_cooldown_record_failed.inc(1);
+      log.warn("pool cooldown flag failed", {
+        networkPoolId: poolId,
+        providerId,
+        error: String(cause),
+      });
+    });
+  void recordPoolCooldownEvent(db, poolId, providerId, reason);
+}
+
+/**
+ * One egress-failure verdict → pool-flag dispatch, shared by the attempt loop
+ * and the streaming finalizer.
+ *
+ * Both paths classify the same failure with `classifyEgressFailure`, then must
+ * retire the same pool the same way; a policy change — which verdict maps to a
+ * cooldown vs a block, or a new verdict — belongs here so the two response
+ * modes cannot drift in telemetry.
+ *
+ * The call sites compute the verdict (the attempt loop also needs it for its
+ * retry decision) and pass it in; the gating is identical to what both
+ * previously applied by hand: only a real pool id + selector pair, only a
+ * `GatewayError`, and never a `"none"` verdict. The flags themselves are
+ * fire-and-forget advisory writes that never fail the request.
+ */
+export function flagPoolOnEgressFailure(
+  poolSelector: NetworkPoolSelector | undefined,
+  db: CartethyiaDatabase,
+  poolId: string | undefined,
+  providerId: string,
+  error: unknown,
+  verdict: EgressVerdict,
+): void {
+  if (poolId === undefined || poolSelector === undefined || !(error instanceof GatewayError)) return;
+  if (verdict.kind === "cooldown") {
+    flagPoolCooldown(poolSelector, db, poolId, providerId, error);
+  } else if (verdict.kind === "blocked") {
+    flagPoolBlocked(poolSelector, db, poolId, providerId, error);
+  }
+}
+
 export interface PoolHealthEvent {
   readonly id: string;
   readonly networkPoolId: string;