cartethyia 2.6.7 → 2.7.1

package/src/console/routing/pools/routes.ts

@@ -87,6 +87,36 @@ export function createNetworkPoolRoutes(config: NetworkPoolConfig): Elysia {
     .delete("/:poolId", async ({ request, params }) => {
       return await factory.deletePool(config.accessResolver(request), params.poolId);
 })
+    .post(
+      "/bulk/create",
+      { body: t.Object({ targets: t.Array(createPoolBody, { minItems: 1, maxItems: MAX_BATCH_PROBE_TARGETS }) }) },
+      async ({ request, body, set }) => {
+        set.status = 200;
+        return await factory.createPoolBatch(
+          config.accessResolver(request),
+          body.targets as CreateNetworkPoolRequest[],
+        );
+      },
+    )
+    .post(
+      "/bulk/delete",
+      { body: t.Object({ ids: t.Array(t.String(), { minItems: 1 }) }) },
+      async ({ request, body }) => {
+        return await factory.deletePoolBatch(config.accessResolver(request), body.ids);
+      },
+    )
+    .post(
+      "/bulk/status",
+      {
+        body: t.Object({
+          ids: t.Array(t.String(), { minItems: 1 }),
+          status: literalUnion(POOL_STATUSES),
+        }),
+      },
+      async ({ request, body }) => {
+        return await factory.setPoolStatusBatch(config.accessResolver(request), body.ids, body.status);
+      },
+    )
     .get("/:poolId/health-events", async ({ request, params }) => {
       return await factory.listPoolHealthEvents(config.accessResolver(request), params.poolId);
 })

package/src/console/routing/pools/store.ts

@@ -1,5 +1,5 @@
 // Drizzle-backed console persistence for network pools.
-import { and, asc, eq, sql } from "drizzle-orm";
+import { and, asc, eq, inArray, sql } from "drizzle-orm";
 import type { CartethyiaDatabase } from "../../../persistence/db-handle";
 import { networkPools, poolRoutingSettings } from "../../../persistence/schema";
 import {
@@ -34,6 +34,7 @@ import {
   recoverNetworkPool,
 } from "../../../network/pool-health-machine";
 import { drainPoolByteDelta, recordPoolBytes } from "../../../network/pool/byte-accounting";
+import * as poolCooldownStore from "../../../network/pool-cooldown-store";
 // Cloudflare's trace endpoint answers with plain `key=value` lines including
 // `ip=`, the address the request egressed from. Dialing it THROUGH the pool
 // therefore reports the pool's public address — which is what an operator
@@ -255,7 +256,7 @@ export class DrizzleNetworkPoolStore implements NetworkPoolStore {
         reason: error instanceof Error ? error.message : "Failed to build pool agent",
       };
     }
-    return await this.dialProviderProbe(agent, poolId, providerId, poolId);
+    return await this.dialProviderProbe(agent, poolId, providerId, poolId, rec.endpoint);
   }
 
   /**
@@ -295,13 +296,19 @@ export class DrizzleNetworkPoolStore implements NetworkPoolStore {
    *
    * `label` is what the result reports (a pool id, or a pasted endpoint that
    * has no row yet). `poolId` is the row to bank the verdict on — omitted for
-   * an ad-hoc probe, because there is nothing to attach it to.
+   * an ad-hoc probe, because there is nothing to attach it to. `endpoint` is
+   * the address the verdict describes, recorded beside it so
+   * `storedProviderProbe` can verify it later — it must be the pool's *current
+   * endpoint*, never the label: a verdict banked against the pool id fails the
+   * endpoint comparison on read and is hidden as "never probed", which is
+   * exactly the disappearing-verdict report.
    */
   private async dialProviderProbe(
     agent: PoolAgent,
     label: string,
     providerId: string,
     poolId?: string,
+    endpoint?: string,
   ): Promise<PoolProviderProbeResult> {
     const spec = await buildProviderProbeRequest(providerId);
     if (!spec) {
@@ -343,7 +350,7 @@ export class DrizzleNetworkPoolStore implements NetworkPoolStore {
       latencyMs,
       ...(verdict.reason === undefined ? {} : { reason: verdict.reason }),
     };
-    if (poolId !== undefined) await this.persistProviderProbe(poolId, label, result);
+    if (poolId !== undefined) await this.persistProviderProbe(poolId, endpoint ?? label, result);
     return result;
   }
 
@@ -403,12 +410,24 @@ export class DrizzleNetworkPoolStore implements NetworkPoolStore {
   }
 
   async create(record: NetworkPoolRecord): Promise<void> {
-    const kind = record.kind === "https" ? "http" : record.kind;
-    await this.db.insert(networkPools).values({
+    await this.createMany([record]).then(() => undefined);
+  }
+
+  /**
+   * Inserts every record in one statement.
+   *
+   * A batch import used to be one round trip per proxy, so a long paste took
+   * minutes and the dialog sat there while the operator waited. Drizzle emits a
+   * single multi-row INSERT, which makes the cost independent of the paste
+   * length.
+   */
+  async createMany(records: readonly NetworkPoolRecord[]): Promise<number> {
+    if (records.length === 0) return 0;
+    const values = records.map((record) => ({
       id: record.id,
       tenantId: record.tenantId,
       createdAt: new Date(record.createdAt),
-      kind,
+      kind: record.kind === "https" ? "http" : record.kind,
       endpointConfig: {
         endpoint: record.endpoint,
         ...(record.label ? { label: record.label } : {}),
@@ -423,7 +442,50 @@ export class DrizzleNetworkPoolStore implements NetworkPoolStore {
       consecutiveFailures: 0,
       ...(record.quotaBytes !== undefined ? { quotaBytes: record.quotaBytes } : {}),
       ...(record.family !== undefined ? { family: record.family } : {}),
-    });
+    }));
+    await this.db.insert(networkPools).values(values);
+    return values.length;
+  }
+
+  /**
+   * Sets one status on every named pool in a single statement.
+   *
+   * Bulk enable/disable was a loop of per-row updates — one round trip each,
+   * and one snapshot invalidation each. A tenant-wide selection is exactly the
+   * case where that is felt.
+   */
+  async updateManyStatus(
+    tenantId: string,
+    poolIds: readonly string[],
+    status: NetworkPoolRecord["status"],
+  ): Promise<number> {
+    if (poolIds.length === 0) return 0;
+    const rows = await this.db
+      .update(networkPools)
+      .set({ status })
+      .where(
+        and(
+          eq(networkPools.tenantId, tenantId),
+          inArray(networkPools.id, [...poolIds]),
+        ),
+      )
+      .returning({ id: networkPools.id });
+    return rows.length;
+  }
+
+  /** Deletes every named pool in one statement. */
+  async deleteMany(tenantId: string, poolIds: readonly string[]): Promise<number> {
+    if (poolIds.length === 0) return 0;
+    const rows = await this.db
+      .delete(networkPools)
+      .where(
+        and(
+          eq(networkPools.tenantId, tenantId),
+          inArray(networkPools.id, [...poolIds]),
+        ),
+      )
+      .returning({ id: networkPools.id });
+    return rows.length;
   }
 
   async update(
@@ -513,6 +575,34 @@ export class DrizzleNetworkPoolStore implements NetworkPoolStore {
     return listNetworkPoolHealthEvents(this.db, tenantId, poolId);
   }
 
+  async dropProviderCooldownMarker(
+    tenantId: string,
+    poolId: string,
+    providerId: string,
+    onlyBlocked: boolean,
+  ): Promise<void> {
+    if ((await this.get(tenantId, poolId)) === undefined) return;
+    await poolCooldownStore.dropProviderCooldownMarker(this.db, poolId, providerId, onlyBlocked);
+  }
+
+  async dropAllProviderCooldownMarkers(
+    tenantId: string,
+    poolId: string,
+  ): Promise<void> {
+    if ((await this.get(tenantId, poolId)) === undefined) return;
+    await poolCooldownStore.dropAllProviderCooldownMarkers(this.db, poolId);
+  }
+
+  async recordProviderProbeCooldown(
+    tenantId: string,
+    poolId: string,
+    providerId: string,
+    marker: { until: number; reason: string; blocked?: boolean },
+  ): Promise<void> {
+    if ((await this.get(tenantId, poolId)) === undefined) return;
+    await poolCooldownStore.recordProbeVerdict(this.db, poolId, providerId, marker);
+  }
+
   async recover(tenantId: string, poolId: string): Promise<boolean> {
     return recoverNetworkPool(this.db, tenantId, poolId);
   }

package/src/main.ts

@@ -49,7 +49,6 @@ const app = boot
       telemetryBuffer: boot.deps.telemetryBuffer,
       resolveOAuthRefresher: boot.deps.resolveOAuthRefresher,
       oauthRefreshService: boot.deps.oauthRefreshService,
-      modelStrikes: boot.deps.modelStrikes,
       maxBodyBytes: resolveMaxBodyBytes(),
       scheduledTasks: boot.deps.scheduledTasks,
       // The sampler is constructed in the dependency graph and already
@@ -78,7 +77,6 @@ const app = boot
         redis: boot.deps.redis,
         oauthRefreshService: boot.deps.oauthRefreshService,
         admissionService: boot.deps.admissionService,
-        modelStrikes: boot.deps.modelStrikes,
         readRoutingAccountInflight: boot.deps.readRoutingAccountInflight,
         // The same three writers the shutdown coordinator drains, so a manual
         // flush from the console does exactly what a graceful stop would do
@@ -120,6 +118,16 @@ function shutdown(signal: "SIGINT" | "SIGTERM", reason: "SIGINT" | "SIGTERM" | "
 }
 
 if (boot) {
+  // `bun --hot` re-runs this module against the cached boot object; stop the
+  // previous listener before rebinding so a reload does not stack two servers
+  // on the same port.
+  if (boot.server && typeof boot.server.stop === "function") {
+    try {
+      boot.server.stop();
+    } catch {
+      // Best-effort on reload; the bind below would fail loudly if it stuck.
+    }
+  }
   boot.server = app.listen(
     {
       port,
@@ -151,6 +159,12 @@ if (boot) {
   // the lease sweep and the health sweep should not run before the listener
   // exists to serve traffic.
   boot.deps.scheduledTasks.start();
+  // Warm the route snapshot before the first request lands: the first /v1/*
+  // request otherwise pays the full catalog build synchronously (ten+ SELECTs).
+  // A failure here is a soft miss — the next request rebuilds as before.
+  void boot.deps.snapshotService.getSnapshot().catch((error: unknown) => {
+    log.warn("[startup] initial route snapshot warm failed", error as Error);
+  });
   if (!globalThis.__cartethyiaSignalsRegistered) {
     globalThis.__cartethyiaSignalsRegistered = true;
     process.on("SIGINT", () => shutdown("SIGINT"));