cartethyia 2.7.8 → 2.8.0

package/src/network/pool-cooldown-store.ts

@@ -17,6 +17,7 @@ import { eq } from "drizzle-orm";
 import type { CartethyiaDatabase } from "../persistence/db-handle";
 import { networkPools, healthEvents } from "../persistence/schema";
 import { log } from "../observability/logger";
+import { metrics } from "../observability/metrics";
 import { providerRateLimitIsIpScoped } from "../providers/provider-metadata";
 import { resolvePoolProviderRateLimitCooldownMs } from "../config/cooldowns";
 import { PROVIDER_BLOCK_MS, type ProxyCooldownEntry } from "./pool/selector";
@@ -152,6 +153,88 @@ export async function pruneExpiredProviderCooldownMarkers(
   }
 }
 
+/** Max simultaneously tracked (pool, provider) coalescing keys. */
+const MAX_TRACKED_PAIRS = 5_000;
+
+/** Last coalesced row timestamp per (pool, provider), for the cooldown window only. */
+const lastRecordedByPoolProvider = new Map<string, number>();
+
+/**
+ * Appends one provider-scoped egress verdict row to `health_events` — the
+ * durable audit log the console's pool page reads.
+ *
+ * Shared by the three writers of this row shape: the dispatch-side 429
+ * recorder (`recordPoolCooldownEvent`), the egress-block recorder
+ * (`recordPoolBlockEvent`), and the probe verdict recorder
+ * (`recordProbeVerdict`). The row is always `entityKind: "pool"`,
+ * `fromStatus: null`, `toStatus: "cooldown"`, and the category is chosen by
+ * `blocked`; the only divergences are the optional code suffix (block path)
+ * and the optional coalescing window (cooldown path).
+ *
+ * Coalescing: `coalesceWindowMs` suppresses a duplicate row for the same
+ * (pool, provider) pair within the window — used by the cooldown path, where
+ * dispatch flags the same pair on every throttled request and the audit log
+ * needs one row per episode, not one per request. The default `undefined`
+ * means no coalescing, so block and probe rows are never deduped; it also
+ * means the pair never enters the tracking map, so the eviction sweep below
+ * only ever sees coalesced pairs. A failed insert does not stamp the pair, so
+ * the next attempt within the window is still tried.
+ *
+ * Swallow contract: this never throws to dispatch (observability writes are
+ * non-fatal); failures bump `pool_cooldown_record_failed` and are logged.
+ */
+export async function appendPoolProviderEgressAuditRow(
+  db: CartethyiaDatabase,
+  opts: {
+    poolId: string;
+    providerId: string;
+    reason: string;
+    blocked: boolean;
+    codeSuffix?: string;
+    coalesceWindowMs?: number;
+  },
+): Promise<void> {
+  const { poolId, providerId, reason, blocked, codeSuffix, coalesceWindowMs } = opts;
+  let key: string | undefined;
+  let now = 0;
+  if (coalesceWindowMs !== undefined) {
+    key = `${poolId}::${providerId.toLowerCase()}`;
+    now = Date.now();
+    const last = lastRecordedByPoolProvider.get(key);
+    if (last !== undefined && now - last < coalesceWindowMs) return;
+  }
+  try {
+    await db.insert(healthEvents).values({
+      id: randomUUID(),
+      entityKind: "pool",
+      networkPoolId: poolId,
+      fromStatus: null,
+      toStatus: "cooldown",
+      reason: `provider ${providerId}: ${reason}${codeSuffix ?? ""}`,
+      errorCategory: blocked ? "egress_blocked" : "rate_limit_transient",
+      createdAt: new Date(),
+    });
+  } catch (error) {
+    metrics.pool_cooldown_record_failed.inc(1);
+    log.warn("pool provider egress audit row failed", {
+      networkPoolId: poolId,
+      providerId,
+      error: String(error),
+    });
+    return;
+  }
+  if (key !== undefined && coalesceWindowMs !== undefined) {
+    lastRecordedByPoolProvider.set(key, now);
+    if (lastRecordedByPoolProvider.size > MAX_TRACKED_PAIRS) {
+      for (const [trackedKey, recordedAt] of lastRecordedByPoolProvider) {
+        if (now - recordedAt >= coalesceWindowMs) {
+          lastRecordedByPoolProvider.delete(trackedKey);
+        }
+      }
+    }
+  }
+}
+
 /**
  * Writes a probe-measured verdict the same way a dispatch-side event would be
  * recorded: one mirror marker and one audit row.
@@ -174,20 +257,12 @@ export async function recordProbeVerdict(
   marker: { until: number; reason: string; blocked?: boolean },
 ): Promise<void> {
   await upsertProviderCooldownMarker(db, poolId, providerId, marker);
-  try {
-    await db.insert(healthEvents).values({
-      id: randomUUID(),
-      entityKind: "pool",
-      networkPoolId: poolId,
-      fromStatus: null,
-      toStatus: "cooldown",
-      reason: `provider ${providerId}: ${marker.reason}`,
-      errorCategory: marker.blocked === true ? "egress_blocked" : "rate_limit_transient",
-      createdAt: new Date(),
-    });
-  } catch (error) {
-    log.warn("pool probe event failed", { networkPoolId: poolId, providerId, error: String(error) });
-  }
+  await appendPoolProviderEgressAuditRow(db, {
+    poolId,
+    providerId,
+    reason: marker.reason,
+    blocked: marker.blocked === true,
+  });
 }
 
 /**

package/src/network/pool-health.ts

@@ -1,12 +1,11 @@
-import { randomUUID } from "node:crypto";
 import type { CartethyiaDatabase } from "../persistence/db-handle";
-import { healthEvents } from "../persistence/schema";
 import type { NetworkPoolSelector } from "./pool/selector";
-import type { GatewayError } from "../protocol/gateway-error";
+import { GatewayError } from "../protocol/gateway-error";
 import { metrics } from "../observability/metrics";
 import { log } from "../observability/logger";
-import { upsertProviderCooldownMarker } from "./pool-cooldown-store";
+import { appendPoolProviderEgressAuditRow, upsertProviderCooldownMarker } from "./pool-cooldown-store";
 import { resolvePoolProviderRateLimitCooldownMs } from "../config/cooldowns";
+import type { EgressVerdict } from "./rate-limit-verdict";
 
 /**
  * Durable pool rate-limit history.
@@ -29,9 +28,6 @@ import { resolvePoolProviderRateLimitCooldownMs } from "../config/cooldowns";
  * needs one row per cooldown episode — not one per throttled request.
  */
 const POOL_COOLDOWN_EVENT_WINDOW_MS = 15 * 60 * 1000;
-const MAX_TRACKED_PAIRS = 5_000;
-
-const lastRecordedByPoolProvider = new Map<string, number>();
 
 /** Durable half of `flagPoolCooldown`; not a second public entry point. */
 async function recordPoolCooldownEvent(
@@ -40,35 +36,13 @@ async function recordPoolCooldownEvent(
   providerId: string,
   reason: string,
 ): Promise<void> {
-  const key = `${poolId}::${providerId.toLowerCase()}`;
-  const now = Date.now();
-  const last = lastRecordedByPoolProvider.get(key);
-  if (last !== undefined && now - last < POOL_COOLDOWN_EVENT_WINDOW_MS) return;
-  try {
-    await db.insert(healthEvents).values({
-      id: randomUUID(),
-      entityKind: "pool",
-      networkPoolId: poolId,
-      fromStatus: null,
-      toStatus: "cooldown",
-      reason: `provider ${providerId}: ${reason}`,
-      errorCategory: "rate_limit_transient",
-      createdAt: new Date(),
-    });
-  } catch {
-    // Non-fatal: observability writes must never fail dispatch. The
-    // coalescing stamp is only written below, so a failed insert does not
-    // suppress the next attempt for the whole window.
-    return;
-  }
-  lastRecordedByPoolProvider.set(key, now);
-  if (lastRecordedByPoolProvider.size > MAX_TRACKED_PAIRS) {
-    for (const [trackedKey, recordedAt] of lastRecordedByPoolProvider) {
-      if (now - recordedAt >= POOL_COOLDOWN_EVENT_WINDOW_MS) {
-        lastRecordedByPoolProvider.delete(trackedKey);
-      }
-    }
-  }
+  await appendPoolProviderEgressAuditRow(db, {
+    poolId,
+    providerId,
+    reason,
+    blocked: false,
+    coalesceWindowMs: POOL_COOLDOWN_EVENT_WINDOW_MS,
+  });
 }
 
 /**
@@ -106,14 +80,7 @@ export function flagPoolBlocked(
         error: String(cause),
       });
     });
-  void recordPoolBlockEvent(db, poolId, providerId, reason, error).catch((cause: unknown) => {
-    metrics.pool_cooldown_record_failed.inc(1);
-    log.warn("pool block event failed", {
-      networkPoolId: poolId,
-      providerId,
-      error: String(cause),
-    });
-  });
+  void recordPoolBlockEvent(db, poolId, providerId, reason, error);
 }
 
 /**
@@ -131,15 +98,12 @@ async function recordPoolBlockEvent(
   reason: string,
   error: GatewayError,
 ): Promise<void> {
-  await db.insert(healthEvents).values({
-    id: randomUUID(),
-    entityKind: "pool",
-    networkPoolId: poolId,
-    fromStatus: null,
-    toStatus: "cooldown",
-    reason: `provider ${providerId}: ${reason}${providerCodeSuffix(error)}`,
-    errorCategory: "egress_blocked",
-    createdAt: new Date(),
+  await appendPoolProviderEgressAuditRow(db, {
+    poolId,
+    providerId,
+    reason,
+    blocked: true,
+    codeSuffix: providerCodeSuffix(error),
   });
 }
 
@@ -199,12 +163,36 @@ export function flagPoolCooldown(
         error: String(cause),
       });
     });
-  void recordPoolCooldownEvent(db, poolId, providerId, reason).catch((cause: unknown) => {
-    metrics.pool_cooldown_record_failed.inc(1);
-    log.warn("pool cooldown event failed", {
-      networkPoolId: poolId,
-      providerId,
-      error: String(cause),
-    });
-  });
+  void recordPoolCooldownEvent(db, poolId, providerId, reason);
+}
+
+/**
+ * One egress-failure verdict → pool-flag dispatch, shared by the attempt loop
+ * and the streaming finalizer.
+ *
+ * Both paths classify the same failure with `classifyEgressFailure`, then must
+ * retire the same pool the same way; a policy change — which verdict maps to a
+ * cooldown vs a block, or a new verdict — belongs here so the two response
+ * modes cannot drift in telemetry.
+ *
+ * The call sites compute the verdict (the attempt loop also needs it for its
+ * retry decision) and pass it in; the gating is identical to what both
+ * previously applied by hand: only a real pool id + selector pair, only a
+ * `GatewayError`, and never a `"none"` verdict. The flags themselves are
+ * fire-and-forget advisory writes that never fail the request.
+ */
+export function flagPoolOnEgressFailure(
+  poolSelector: NetworkPoolSelector | undefined,
+  db: CartethyiaDatabase,
+  poolId: string | undefined,
+  providerId: string,
+  error: unknown,
+  verdict: EgressVerdict,
+): void {
+  if (poolId === undefined || poolSelector === undefined || !(error instanceof GatewayError)) return;
+  if (verdict.kind === "cooldown") {
+    flagPoolCooldown(poolSelector, db, poolId, providerId, error);
+  } else if (verdict.kind === "blocked") {
+    flagPoolBlocked(poolSelector, db, poolId, providerId, error);
+  }
 }

package/src/network/rate-limit-verdict.ts

@@ -21,7 +21,7 @@ import { GatewayError } from "../protocol/gateway-error";
 import { providerRateLimitIsIpScoped } from "../providers/provider-metadata";
 
 /** What one upstream failure means for the egress that carried it. */
-type EgressVerdict =
+export type EgressVerdict =
   | { kind: "none" }
   | { kind: "cooldown"; retryAfterMs?: number }
   | { kind: "blocked" };

package/src/network/ssrf.ts

@@ -3,6 +3,7 @@ import { lookup } from "node:dns/promises";
 import { isIP } from "node:net";
 import { GatewayError } from "../protocol/gateway-error";
 import type { SsrfPolicy } from "../config/network";
+import { BoundedLruMap } from "../transport/routing/bounded-lru";
 import {
   cidrContainsAddress,
   ipv4ToNumberStrict,
@@ -12,17 +13,16 @@ import {
 
 const DNS_CACHE_TTL_MS = 60_000;
 const DNS_CACHE_MAX = 64;
-const dnsCache = new Map<string, { addrs: string[]; at: number }>();
+// Capacity-bounded: `set` re-inserts the host (moving it to the tail) and
+// evicts the least-recently-used entry when over the cap — the same
+// insertion-order eviction the hand-rolled loop performed, since every write
+// here re-inserts.
+const dnsCache = new BoundedLruMap<string, { addrs: string[]; at: number }>(DNS_CACHE_MAX);
 
 /** Bounds cache memory: evicts oldest entries once size exceeds the cap, not just expired ones. */
 function cacheDnsResult(hostname: string, addrs: string[], now: number): void {
   dnsCache.delete(hostname);
   dnsCache.set(hostname, { addrs, at: now });
-  while (dnsCache.size > DNS_CACHE_MAX) {
-    const oldest = dnsCache.keys().next().value;
-    if (oldest === undefined) break;
-    dnsCache.delete(oldest);
-  }
 }