cartethyia 2.7.0 → 2.7.2

package/src/console/routing/pools/store.ts

@@ -1,5 +1,5 @@
 // Drizzle-backed console persistence for network pools.
-import { and, asc, eq, sql } from "drizzle-orm";
+import { and, asc, eq, inArray, sql } from "drizzle-orm";
 import type { CartethyiaDatabase } from "../../../persistence/db-handle";
 import { networkPools, poolRoutingSettings } from "../../../persistence/schema";
 import {
@@ -34,6 +34,7 @@ import {
   recoverNetworkPool,
 } from "../../../network/pool-health-machine";
 import { drainPoolByteDelta, recordPoolBytes } from "../../../network/pool/byte-accounting";
+import * as poolCooldownStore from "../../../network/pool-cooldown-store";
 // Cloudflare's trace endpoint answers with plain `key=value` lines including
 // `ip=`, the address the request egressed from. Dialing it THROUGH the pool
 // therefore reports the pool's public address — which is what an operator
@@ -255,7 +256,7 @@ export class DrizzleNetworkPoolStore implements NetworkPoolStore {
         reason: error instanceof Error ? error.message : "Failed to build pool agent",
       };
     }
-    return await this.dialProviderProbe(agent, poolId, providerId, poolId);
+    return await this.dialProviderProbe(agent, poolId, providerId, poolId, rec.endpoint);
   }
 
   /**
@@ -295,13 +296,19 @@ export class DrizzleNetworkPoolStore implements NetworkPoolStore {
    *
    * `label` is what the result reports (a pool id, or a pasted endpoint that
    * has no row yet). `poolId` is the row to bank the verdict on — omitted for
-   * an ad-hoc probe, because there is nothing to attach it to.
+   * an ad-hoc probe, because there is nothing to attach it to. `endpoint` is
+   * the address the verdict describes, recorded beside it so
+   * `storedProviderProbe` can verify it later — it must be the pool's *current
+   * endpoint*, never the label: a verdict banked against the pool id fails the
+   * endpoint comparison on read and is hidden as "never probed", which is
+   * exactly the disappearing-verdict report.
    */
   private async dialProviderProbe(
     agent: PoolAgent,
     label: string,
     providerId: string,
     poolId?: string,
+    endpoint?: string,
   ): Promise<PoolProviderProbeResult> {
     const spec = await buildProviderProbeRequest(providerId);
     if (!spec) {
@@ -343,7 +350,7 @@ export class DrizzleNetworkPoolStore implements NetworkPoolStore {
       latencyMs,
       ...(verdict.reason === undefined ? {} : { reason: verdict.reason }),
     };
-    if (poolId !== undefined) await this.persistProviderProbe(poolId, label, result);
+    if (poolId !== undefined) await this.persistProviderProbe(poolId, endpoint ?? label, result);
     return result;
   }
 
@@ -403,12 +410,24 @@ export class DrizzleNetworkPoolStore implements NetworkPoolStore {
   }
 
   async create(record: NetworkPoolRecord): Promise<void> {
-    const kind = record.kind === "https" ? "http" : record.kind;
-    await this.db.insert(networkPools).values({
+    await this.createMany([record]).then(() => undefined);
+  }
+
+  /**
+   * Inserts every record in one statement.
+   *
+   * A batch import used to be one round trip per proxy, so a long paste took
+   * minutes and the dialog sat there while the operator waited. Drizzle emits a
+   * single multi-row INSERT, which makes the cost independent of the paste
+   * length.
+   */
+  async createMany(records: readonly NetworkPoolRecord[]): Promise<number> {
+    if (records.length === 0) return 0;
+    const values = records.map((record) => ({
       id: record.id,
       tenantId: record.tenantId,
       createdAt: new Date(record.createdAt),
-      kind,
+      kind: record.kind === "https" ? "http" : record.kind,
       endpointConfig: {
         endpoint: record.endpoint,
         ...(record.label ? { label: record.label } : {}),
@@ -423,7 +442,50 @@ export class DrizzleNetworkPoolStore implements NetworkPoolStore {
       consecutiveFailures: 0,
       ...(record.quotaBytes !== undefined ? { quotaBytes: record.quotaBytes } : {}),
       ...(record.family !== undefined ? { family: record.family } : {}),
-    });
+    }));
+    await this.db.insert(networkPools).values(values);
+    return values.length;
+  }
+
+  /**
+   * Sets one status on every named pool in a single statement.
+   *
+   * Bulk enable/disable was a loop of per-row updates — one round trip each,
+   * and one snapshot invalidation each. A tenant-wide selection is exactly the
+   * case where that is felt.
+   */
+  async updateManyStatus(
+    tenantId: string,
+    poolIds: readonly string[],
+    status: NetworkPoolRecord["status"],
+  ): Promise<number> {
+    if (poolIds.length === 0) return 0;
+    const rows = await this.db
+      .update(networkPools)
+      .set({ status })
+      .where(
+        and(
+          eq(networkPools.tenantId, tenantId),
+          inArray(networkPools.id, [...poolIds]),
+        ),
+      )
+      .returning({ id: networkPools.id });
+    return rows.length;
+  }
+
+  /** Deletes every named pool in one statement. */
+  async deleteMany(tenantId: string, poolIds: readonly string[]): Promise<number> {
+    if (poolIds.length === 0) return 0;
+    const rows = await this.db
+      .delete(networkPools)
+      .where(
+        and(
+          eq(networkPools.tenantId, tenantId),
+          inArray(networkPools.id, [...poolIds]),
+        ),
+      )
+      .returning({ id: networkPools.id });
+    return rows.length;
   }
 
   async update(
@@ -513,6 +575,34 @@ export class DrizzleNetworkPoolStore implements NetworkPoolStore {
     return listNetworkPoolHealthEvents(this.db, tenantId, poolId);
   }
 
+  async dropProviderCooldownMarker(
+    tenantId: string,
+    poolId: string,
+    providerId: string,
+    onlyBlocked: boolean,
+  ): Promise<void> {
+    if ((await this.get(tenantId, poolId)) === undefined) return;
+    await poolCooldownStore.dropProviderCooldownMarker(this.db, poolId, providerId, onlyBlocked);
+  }
+
+  async dropAllProviderCooldownMarkers(
+    tenantId: string,
+    poolId: string,
+  ): Promise<void> {
+    if ((await this.get(tenantId, poolId)) === undefined) return;
+    await poolCooldownStore.dropAllProviderCooldownMarkers(this.db, poolId);
+  }
+
+  async recordProviderProbeCooldown(
+    tenantId: string,
+    poolId: string,
+    providerId: string,
+    marker: { until: number; reason: string; blocked?: boolean },
+  ): Promise<void> {
+    if ((await this.get(tenantId, poolId)) === undefined) return;
+    await poolCooldownStore.recordProbeVerdict(this.db, poolId, providerId, marker);
+  }
+
   async recover(tenantId: string, poolId: string): Promise<boolean> {
     return recoverNetworkPool(this.db, tenantId, poolId);
   }

package/src/main.ts

@@ -1,167 +1,183 @@
-// Must run before any module that registers a route: it wires Elysia's TypeBox
-// bridge statically, which the source-run path needs under NODE_ENV=production.
-import "./boot/elysia-typebox-bridge";
-import { createGatewayApp, createGatewayShell } from "./boot/app";
-import { bootstrap } from "./boot/lifecycle";
-import type { CartethyiaBoot } from "./boot/lifecycle";
-import {
-  resolveDrainToken,
-  resolveIdleTimeout,
-  resolveMaxBodyBytes,
-  resolvePort,
-  resolveProcessTopology,
-} from "./config/runtime";
-import { Manifest } from "elysia";
-import { log } from "./observability/logger";
-
-const port = resolvePort();
-const entryMemory = process.memoryUsage();
-log.info(
-  `[startup] phase=entrypoint-ready elapsed_ms=${Math.round(process.uptime() * 1_000)} ` +
-    `rss_mb=${(entryMemory.rss / 1024 / 1024).toFixed(1)} ` +
-    `heap_used_mb=${(entryMemory.heapUsed / 1024 / 1024).toFixed(1)}`,
-);
-
-declare global {
-  // eslint-disable-next-line no-var -- globalThis augmentation requires `var`
-  var __cartethyiaBoot: CartethyiaBoot | undefined;
-  // eslint-disable-next-line no-var
-  var __cartethyiaSignalsRegistered: boolean | undefined;
-}
-
-const boot = Manifest.isCapturing()
-  ? undefined
-  : (globalThis.__cartethyiaBoot ??= await bootstrap());
-
-const app = boot
-  ? createGatewayApp({
-      mode: "production",
-      db: boot.deps.db,
-      proxyPreparer: boot.deps.proxyPreparer,
-      resolveProviderAdapter: boot.deps.resolveProviderAdapter,
-      byokUpstreamHosts: boot.deps.byokUpstreamHosts,
-      networkBindingFactory: boot.deps.networkBindingFactory,
-      ipAbuseProtection: boot.deps.ipAbuseProtection,
-      trustedProxyBoundary: boot.deps.trustedProxyBoundary,
-      poolSelector: boot.deps.poolSelector,
-      snapshotService: boot.deps.snapshotService,
-      readiness: boot.deps.readiness,
-      telemetryBuffer: boot.deps.telemetryBuffer,
-      resolveOAuthRefresher: boot.deps.resolveOAuthRefresher,
-      oauthRefreshService: boot.deps.oauthRefreshService,
-      maxBodyBytes: resolveMaxBodyBytes(),
-      scheduledTasks: boot.deps.scheduledTasks,
-      // The sampler is constructed in the dependency graph and already
-      // sampling on its own interval; without passing it here the app cannot
-      // bind its in-flight source, so `cartethyia_process_in_flight` stays
-      // pinned at the constructor's `() => 0` placeholder forever while the
-      // registry it was meant to read reports real flights.
-      runtimeMetricsSampler: boot.deps.runtimeMetricsSampler,
-      shutdownCoordinator: boot.shutdownCoordinator,
-      // Signal-free graceful stop for platforms where a catchable signal cannot
-      // be delivered (Windows). Off unless `CARTETHYIA_DRAIN_TOKEN` is set.
-      ...(resolveDrainToken() !== undefined
-        ? { drainToken: resolveDrainToken() as string, triggerDrain: () => shutdown("SIGTERM") }
-        : {}),
-      // The console mounts in every boot: without REDIS_URL it runs on the
-      // in-process memory backend like the rest of the gateway.
-      consoleApi: {
-        db: boot.deps.db,
-        accessResolver: () => undefined,
-        routeSnapshotService: boot.deps.snapshotService,
-        poolSelector: boot.deps.poolSelector,
-        telemetryBuffer: boot.deps.telemetryBuffer,
-        providerRegistry: boot.deps.providerRegistry,
-        bundledModelCatalog: boot.deps.bundledModelCatalog,
-        networkBindingFactory: boot.deps.networkBindingFactory,
-        redis: boot.deps.redis,
-        oauthRefreshService: boot.deps.oauthRefreshService,
-        admissionService: boot.deps.admissionService,
-        readRoutingAccountInflight: boot.deps.readRoutingAccountInflight,
-        // The same three writers the shutdown coordinator drains, so a manual
-        // flush from the console does exactly what a graceful stop would do
-        // for the queues, minus the stop.
-        flushBuffers: {
-          flush: async () => {
-            await boot.deps.telemetryBuffer.flush();
-            await boot.deps.usageFlush.flush();
-            await boot.deps.healthEventFlush.flush();
-          },
-        },
-      },
-    })
-  : createGatewayShell();
-export { app };
-
-function shutdown(signal: "SIGINT" | "SIGTERM", reason: "SIGINT" | "SIGTERM" | "update" = signal): void {
-  if (!boot) return;
-  log.info(`[shutdown] ${signal} received, draining...`);
-  // Derived from the coordinator's own budget, not a hand-kept literal: a
-  // force-exit shorter than the drain would hard-kill a process that was about
-  // to finish gracefully, truncating the in-flight responses the drain exists
-  // to protect.
-  const forceExit = setTimeout(() => {
-    log.error("[shutdown] forced exit after drain budget elapsed");
-    process.exit(1);
-  }, boot.shutdownCoordinator.totalShutdownBudgetMs());
-  forceExit.unref();
-  boot.shutdownCoordinator
-    .begin(reason)
-    .then(() => {
-      log.info("[shutdown] complete");
-      process.exit(0);
-    })
-    .catch((error: unknown) => {
-      log.error("[shutdown] failed", error as Error);
-      process.exit(1);
-    });
-}
-
-if (boot) {
-  boot.server = app.listen(
-    {
-      port,
-      // Cap idle sockets so a Slowloris/keep-alive-abuse client cannot
-      // pin one of the server's `maxRequestBodySize`-budgeted sockets
-      // indefinitely. 60 s aligns with the DB pool idle timeout so all
-      // long-idle resources reap together.
-      idleTimeout: resolveIdleTimeout(),
-      // SO_REUSEPORT load-balancing, so several processes can share the port
-      // without a reverse proxy. Tied to the declared topology rather than
-      // left always-on: with one process declared, enabling it lets an
-      // unrelated process on the same port silently take half the traffic —
-      // which is worse than a bind error, because nothing reports it.
-      reusePort: resolveProcessTopology().multiProcess,
-      // 8 MiB cap on request bodies keeps a rogue client from allocating
-      // gigabytes of memory in the ingress path before validation rejects.
-      maxRequestBodySize: resolveMaxBodyBytes(),
-    },
-    () => {
-      log.info(`Cartethyia listening on :${port} (Bun ${Bun.version})`);
-      const memory = process.memoryUsage();
-      log.info(
-        `[startup] phase=listener-ready rss_mb=${(memory.rss / 1024 / 1024).toFixed(1)} ` +
-          `heap_used_mb=${(memory.heapUsed / 1024 / 1024).toFixed(1)}`,
-      );
-    },
-  );
-  // Started here rather than inside the dependency builder: the first tick of
-  // the lease sweep and the health sweep should not run before the listener
-  // exists to serve traffic.
-  boot.deps.scheduledTasks.start();
-  if (!globalThis.__cartethyiaSignalsRegistered) {
-    globalThis.__cartethyiaSignalsRegistered = true;
-    process.on("SIGINT", () => shutdown("SIGINT"));
-    process.on("SIGTERM", () => shutdown("SIGTERM"));
-    // An in-place update signals SIGUSR2 before swapping the image so the old
-    // process drains with the `update` reason and its callers are told the
-    // replacement is seconds away instead of seeing a generic shutdown. Not
-    // available on Windows, where the listener is simply never registered —
-    // production runs in Linux containers, which is where an update happens.
-    try {
-      process.on("SIGUSR2", () => shutdown("SIGTERM", "update"));
-    } catch {
-      // Platform without SIGUSR2: update drains fall back to the SIGTERM notice.
-    }
-  }
-}
+// Must run before any module that registers a route: it wires Elysia's TypeBox
+// bridge statically, which the source-run path needs under NODE_ENV=production.
+import "./boot/elysia-typebox-bridge";
+import { createGatewayApp, createGatewayShell } from "./boot/app";
+import { bootstrap } from "./boot/lifecycle";
+import type { CartethyiaBoot } from "./boot/lifecycle";
+import {
+  resolveDrainToken,
+  resolveIdleTimeout,
+  resolveMaxBodyBytes,
+  resolvePort,
+  resolveProcessTopology,
+} from "./config/runtime";
+import { Manifest } from "elysia";
+import { log } from "./observability/logger";
+
+const port = resolvePort();
+const entryMemory = process.memoryUsage();
+log.info(
+  `[startup] phase=entrypoint-ready elapsed_ms=${Math.round(process.uptime() * 1_000)} ` +
+    `rss_mb=${(entryMemory.rss / 1024 / 1024).toFixed(1)} ` +
+    `heap_used_mb=${(entryMemory.heapUsed / 1024 / 1024).toFixed(1)}`,
+);
+
+declare global {
+  // eslint-disable-next-line no-var -- globalThis augmentation requires `var`
+  var __cartethyiaBoot: CartethyiaBoot | undefined;
+  // eslint-disable-next-line no-var
+  var __cartethyiaSignalsRegistered: boolean | undefined;
+}
+
+const boot = Manifest.isCapturing()
+  ? undefined
+  : (globalThis.__cartethyiaBoot ??= await bootstrap());
+
+const app = boot
+  ? createGatewayApp({
+      mode: "production",
+      db: boot.deps.db,
+      proxyPreparer: boot.deps.proxyPreparer,
+      resolveProviderAdapter: boot.deps.resolveProviderAdapter,
+      byokUpstreamHosts: boot.deps.byokUpstreamHosts,
+      networkBindingFactory: boot.deps.networkBindingFactory,
+      ipAbuseProtection: boot.deps.ipAbuseProtection,
+      trustedProxyBoundary: boot.deps.trustedProxyBoundary,
+      poolSelector: boot.deps.poolSelector,
+      snapshotService: boot.deps.snapshotService,
+      readiness: boot.deps.readiness,
+      telemetryBuffer: boot.deps.telemetryBuffer,
+      resolveOAuthRefresher: boot.deps.resolveOAuthRefresher,
+      oauthRefreshService: boot.deps.oauthRefreshService,
+      maxBodyBytes: resolveMaxBodyBytes(),
+      scheduledTasks: boot.deps.scheduledTasks,
+      // The sampler is constructed in the dependency graph and already
+      // sampling on its own interval; without passing it here the app cannot
+      // bind its in-flight source, so `cartethyia_process_in_flight` stays
+      // pinned at the constructor's `() => 0` placeholder forever while the
+      // registry it was meant to read reports real flights.
+      runtimeMetricsSampler: boot.deps.runtimeMetricsSampler,
+      shutdownCoordinator: boot.shutdownCoordinator,
+      // Signal-free graceful stop for platforms where a catchable signal cannot
+      // be delivered (Windows). Off unless `CARTETHYIA_DRAIN_TOKEN` is set.
+      ...(resolveDrainToken() !== undefined
+        ? { drainToken: resolveDrainToken() as string, triggerDrain: () => shutdown("SIGTERM") }
+        : {}),
+      // The console mounts in every boot: without REDIS_URL it runs on the
+      // in-process memory backend like the rest of the gateway.
+      consoleApi: {
+        db: boot.deps.db,
+        accessResolver: () => undefined,
+        routeSnapshotService: boot.deps.snapshotService,
+        poolSelector: boot.deps.poolSelector,
+        telemetryBuffer: boot.deps.telemetryBuffer,
+        providerRegistry: boot.deps.providerRegistry,
+        bundledModelCatalog: boot.deps.bundledModelCatalog,
+        networkBindingFactory: boot.deps.networkBindingFactory,
+        redis: boot.deps.redis,
+        oauthRefreshService: boot.deps.oauthRefreshService,
+        admissionService: boot.deps.admissionService,
+        readRoutingAccountInflight: boot.deps.readRoutingAccountInflight,
+        // The same three writers the shutdown coordinator drains, so a manual
+        // flush from the console does exactly what a graceful stop would do
+        // for the queues, minus the stop.
+        flushBuffers: {
+          flush: async () => {
+            await boot.deps.telemetryBuffer.flush();
+            await boot.deps.usageFlush.flush();
+            await boot.deps.healthEventFlush.flush();
+          },
+        },
+      },
+    })
+  : createGatewayShell();
+export { app };
+
+function shutdown(signal: "SIGINT" | "SIGTERM", reason: "SIGINT" | "SIGTERM" | "update" = signal): void {
+  if (!boot) return;
+  log.info(`[shutdown] ${signal} received, draining...`);
+  // Derived from the coordinator's own budget, not a hand-kept literal: a
+  // force-exit shorter than the drain would hard-kill a process that was about
+  // to finish gracefully, truncating the in-flight responses the drain exists
+  // to protect.
+  const forceExit = setTimeout(() => {
+    log.error("[shutdown] forced exit after drain budget elapsed");
+    process.exit(1);
+  }, boot.shutdownCoordinator.totalShutdownBudgetMs());
+  forceExit.unref();
+  boot.shutdownCoordinator
+    .begin(reason)
+    .then(() => {
+      log.info("[shutdown] complete");
+      process.exit(0);
+    })
+    .catch((error: unknown) => {
+      log.error("[shutdown] failed", error as Error);
+      process.exit(1);
+    });
+}
+
+if (boot) {
+  // `bun --hot` re-runs this module against the cached boot object; stop the
+  // previous listener before rebinding so a reload does not stack two servers
+  // on the same port.
+  if (boot.server && typeof boot.server.stop === "function") {
+    try {
+      boot.server.stop();
+    } catch {
+      // Best-effort on reload; the bind below would fail loudly if it stuck.
+    }
+  }
+  boot.server = app.listen(
+    {
+      port,
+      // Cap idle sockets so a Slowloris/keep-alive-abuse client cannot
+      // pin one of the server's `maxRequestBodySize`-budgeted sockets
+      // indefinitely. 60 s aligns with the DB pool idle timeout so all
+      // long-idle resources reap together.
+      idleTimeout: resolveIdleTimeout(),
+      // SO_REUSEPORT load-balancing, so several processes can share the port
+      // without a reverse proxy. Tied to the declared topology rather than
+      // left always-on: with one process declared, enabling it lets an
+      // unrelated process on the same port silently take half the traffic —
+      // which is worse than a bind error, because nothing reports it.
+      reusePort: resolveProcessTopology().multiProcess,
+      // 8 MiB cap on request bodies keeps a rogue client from allocating
+      // gigabytes of memory in the ingress path before validation rejects.
+      maxRequestBodySize: resolveMaxBodyBytes(),
+    },
+    () => {
+      log.info(`Cartethyia listening on :${port} (Bun ${Bun.version})`);
+      const memory = process.memoryUsage();
+      log.info(
+        `[startup] phase=listener-ready rss_mb=${(memory.rss / 1024 / 1024).toFixed(1)} ` +
+          `heap_used_mb=${(memory.heapUsed / 1024 / 1024).toFixed(1)}`,
+      );
+    },
+  );
+  // Started here rather than inside the dependency builder: the first tick of
+  // the lease sweep and the health sweep should not run before the listener
+  // exists to serve traffic.
+  boot.deps.scheduledTasks.start();
+  // Warm the route snapshot before the first request lands: the first /v1/*
+  // request otherwise pays the full catalog build synchronously (ten+ SELECTs).
+  // A failure here is a soft miss — the next request rebuilds as before.
+  void boot.deps.snapshotService.getSnapshot().catch((error: unknown) => {
+    log.warn("[startup] initial route snapshot warm failed", error as Error);
+  });
+  if (!globalThis.__cartethyiaSignalsRegistered) {
+    globalThis.__cartethyiaSignalsRegistered = true;
+    process.on("SIGINT", () => shutdown("SIGINT"));
+    process.on("SIGTERM", () => shutdown("SIGTERM"));
+    // An in-place update signals SIGUSR2 before swapping the image so the old
+    // process drains with the `update` reason and its callers are told the
+    // replacement is seconds away instead of seeing a generic shutdown. Not
+    // available on Windows, where the listener is simply never registered —
+    // production runs in Linux containers, which is where an update happens.
+    try {
+      process.on("SIGUSR2", () => shutdown("SIGTERM", "update"));
+    } catch {
+      // Platform without SIGUSR2: update drains fall back to the SIGTERM notice.
+    }
+  }
+}