cartethyia 2.7.0 → 2.7.1

package/src/console/quota/account-quota-global.ts

@@ -3,7 +3,7 @@ import type { Elysia } from "elysia";
 import { and, eq, isNull } from "drizzle-orm";
 import { adminAuditLog, providerAccounts } from "../../persistence/schema";
 import { requireGlobalAdmin } from "../shared/errors";
-import { GLOBAL_QUOTA_LENS, invalidateQuotaCache } from "./cache";
+import { GLOBAL_QUOTA_LENS, invalidateQuotaCache } from "../../providers/quota/cache";
 import { invalidateCredentialCache } from "../../providers/operations/provider-credential-service";
 import {
   QUOTA_REFRESH_TIMEOUT_MS,
@@ -12,7 +12,7 @@ import {
   signalFetch,
   timeoutSignal,
   type QuotaRefreshDeps,
-} from "./refresh";
+} from "../../providers/quota/refresh";
 import { sanitizeGlobalAccount } from "./quota-view";
 import {
   ACCOUNT_STATUSES,

package/src/console/quota/account-quota-shared.ts

@@ -1,7 +1,7 @@
 import type { ConsoleAccessResolver } from "../auth/access";
 import type { AuditRecorder } from "../auth/audit-recorder";
 import { markClineApiKeyCredential } from "../../providers/integrations/cline/cline-quota";
-import { type QuotaRefreshDeps } from "./refresh";
+import { type QuotaRefreshDeps } from "../../providers/quota/refresh";
 import type { AccountTestEgressDeps } from "./account-test-egress";
 
 /**

package/src/console/quota/account-quota-tenant.ts

@@ -14,7 +14,7 @@ import {
   getCachedQuotaEntries,
   getCachedQuotaEntry,
   invalidateQuotaCache,
-} from "./cache";
+} from "../../providers/quota/cache";
 import {
   QUOTA_REFRESH_TIMEOUT_MS,
   refreshAccountQuota,
@@ -22,7 +22,7 @@ import {
   timeoutSignal,
   type QuotaRefreshDeps,
   type QuotaRefreshFailureCode,
-} from "./refresh";
+} from "../../providers/quota/refresh";
 import { selectAccountTestEgress } from "./account-test-egress";
 import type { ValidatedFetch } from "../../network/outbound-fetch";
 import type { ProviderId } from "../../providers/provider-contracts";
@@ -39,6 +39,10 @@ import {
   type BuddyActivityReportResult,
   type DailyCheckinResult,
 } from "../../providers/integrations/buddy/buddy-checkin";
+import {
+  fetchQoderDailyCheckin,
+  isQoderCheckinProvider,
+} from "../../providers/integrations/qoder/qoder-checkin";
 import { buddyAccountUid } from "../../providers/integrations/buddy/buddy-oauth-shared";
 import {
   supportsAccountReset,
@@ -99,12 +103,20 @@ function toBuddyUsageView(
   };
 }
 
-/** One account's check-in through the buddy billing facade. */
+/** One account's check-in through the buddy billing facade (Qoder uses its own surface). */
 async function runAccountCheckin(
   account: { id: string; providerId: string },
   credential: string,
   forceClaim: boolean,
 ): Promise<DailyCheckinResult> {
+  if (isQoderCheckinProvider(account.providerId)) {
+    return fetchQoderDailyCheckin({
+      providerId: account.providerId,
+      accountId: account.id,
+      credential,
+      fetcher: globalThis.fetch,
+    });
+  }
   return fetchDailyCheckin({
     providerId: account.providerId,
     accountId: account.id,
@@ -153,10 +165,14 @@ async function runDailyGrowthForAccount(
   }
 
   // Report leg: needs a uid, which only a JWT carries, so an opaque API key
-  // fails closed instead of reporting under another account's identity.
+  // fails closed instead of reporting under another account's identity. Qoder
+  // has no activity-report leg — its daily grant is purely the check-in.
+  const qoderAccount = isQoderCheckinProvider(String(providerId));
   const userId = buddyAccountUid(credential);
   let report: { ok: boolean; message: string };
-  if (userId === undefined || userId.length === 0) {
+  if (qoderAccount) {
+    report = { ok: true, message: "Qoder has no activity-report leg" };
+  } else if (userId === undefined || userId.length === 0) {
     report = { ok: false, message: "Report needs an OAuth account (no uid in this credential)" };
   } else {
     try {
@@ -180,13 +196,20 @@ async function runDailyGrowthForAccount(
 
   let checkin: { ok: boolean; message: string };
   try {
-    const result = await fetchDailyCheckin({
-      providerId,
-      accountId: account.id,
-      credential,
-      forceClaim: true,
-      fetcher,
-    });
+    const result = qoderAccount
+      ? await fetchQoderDailyCheckin({
+          providerId: String(providerId),
+          accountId: account.id,
+          credential,
+          fetcher,
+        })
+      : await fetchDailyCheckin({
+          providerId,
+          accountId: account.id,
+          credential,
+          forceClaim: true,
+          fetcher,
+        });
     checkin =
       result.state === "error"
         ? { ok: false, message: result.error ?? "Daily check-in failed" }

package/src/console/quota/quota-view.ts

@@ -1,14 +1,14 @@
 import { providerAccounts } from "../../persistence/schema";
 import type { ProviderQuotaResult } from "../../providers/quota/quota-contracts";
 import { log } from "../../observability/logger";
-import { GLOBAL_QUOTA_LENS } from "./cache";
+import { GLOBAL_QUOTA_LENS } from "../../providers/quota/cache";
 import {
   QUOTA_REFRESH_TIMEOUT_MS,
   refreshAccountQuota,
   signalFetch,
   type QuotaRefreshDeps,
   type QuotaRefreshTarget,
-} from "./refresh";
+} from "../../providers/quota/refresh";
 
 /** Shape returned for quota data on dashboard account responses. */
 export interface QuotaRefreshView {

package/src/console/routing/pools/contracts.ts

@@ -221,7 +221,7 @@ export type PoolStatus = (typeof POOL_STATUSES)[number];
  * direct callers against the same list (mirrors `POOL_STATUSES`). Values must
  * stay aligned with the `pool_routing_strategy` enum in `schema.ts`.
  */
-export const POOL_ROUTING_STRATEGIES = ["least_loaded", "round_robin"] as const;
+export const POOL_ROUTING_STRATEGIES = ["least_loaded", "round_robin", "random"] as const;
 export type PoolRoutingStrategyValue = (typeof POOL_ROUTING_STRATEGIES)[number];
 
 export interface PoolStrategySetting {
@@ -351,12 +351,22 @@ export interface NetworkPoolStore {
   list(tenantId: string): Promise<readonly NetworkPoolRecord[]>;
   get(tenantId: string, poolId: string): Promise<NetworkPoolRecord | undefined>;
   create(record: NetworkPoolRecord): Promise<void>;
+  /** Inserts many rows in one statement; returns how many landed. */
+  createMany?(records: readonly NetworkPoolRecord[]): Promise<number>;
   update(
     tenantId: string,
     poolId: string,
     patch: Partial<NetworkPoolRecord>,
   ): Promise<NetworkPoolRecord | undefined>;
+  /** Marks many rows `deleted`; returns how many matched. */
+  updateManyStatus?(
+    tenantId: string,
+    poolIds: readonly string[],
+    status: NetworkPoolRecord["status"],
+  ): Promise<number>;
   delete(tenantId: string, poolId: string): Promise<boolean>;
+  /** Deletes many rows in one statement; returns how many matched. */
+  deleteMany?(tenantId: string, poolIds: readonly string[]): Promise<number>;
   healthCheck(tenantId: string, poolId: string): Promise<HealthCheckResult>;
   /** Measures download throughput through the pool; never throws for a slow
    * or refused transfer — a failed measurement is itself the result. */
@@ -379,6 +389,36 @@ export interface NetworkPoolStore {
   ): Promise<PoolProviderProbeResult>;
   listHealthEvents(tenantId: string, poolId: string): Promise<readonly PoolHealthEvent[]>;
   recover(tenantId: string, poolId: string): Promise<boolean>;
+  /**
+   * Drops the durable provider-cooldown markers on one pool row — the mirror
+   * the selector restores from at boot. The in-memory/Redis state is cleared
+   * separately by the selector; this keeps the two from drifting after a
+   * restart. `onlyBlocked` limits the drop to retired-egress markers, mirroring
+   * `clearProviderBlockedOnly`: a 429 probe may prove an address was never
+   * retired but must not release its live cooldown. Optional (and a no-op in
+   * tests without a backing row store).
+   */
+  dropProviderCooldownMarker?(
+    tenantId: string,
+    poolId: string,
+    providerId: string,
+    onlyBlocked: boolean,
+  ): Promise<void>;
+  /** Drops every provider's marker on one row (disable/replace paths). */
+  dropAllProviderCooldownMarkers?(tenantId: string, poolId: string): Promise<void>;
+  /**
+   * Writes a probe-measured 429 into the durable mirror and the pool's health
+   * history. The probe is the only writer that can *flag* a cooldown from the
+   * control plane — its 429 is a fresh measurement of the same condition a
+   * dispatch-side 429 records, so it lands in both stores and appears as
+   * activity. Optional; absent in tests without a backing row store.
+   */
+  recordProviderProbeCooldown?(
+    tenantId: string,
+    poolId: string,
+    providerId: string,
+    marker: { until: number; reason: string; blocked?: boolean },
+  ): Promise<void>;
   /** Per-tenant pool-group selection strategy; an absent row reads as
    * `DEFAULT_POOL_STRATEGY`. */
   getStrategy(tenantId: string): Promise<PoolStrategySetting>;

package/src/console/routing/pools/operations.ts

@@ -2,6 +2,8 @@
 // behind the routes. Depends only on the contracts and the shared helpers;
 // `routes.ts` is the only HTTP surface.
 import { ConsoleDomainError, requireTenantScope } from "../../shared/errors";
+import { providerRateLimitIsIpScoped } from "../../../providers/provider-metadata";
+import { resolvePoolProviderRateLimitCooldownMs } from "../../../config/cooldowns";
 import type { PoolHealthEvent } from "../../../network/pool-health-machine";
 import type { AccessDecision } from "../../../security/access-control";
 import {
@@ -28,6 +30,7 @@ import {
   type NetworkPoolConfig,
   type NetworkPoolRecord,
   type NetworkPoolResponse,
+  type PoolStatus,
   type PoolBatchProbeResult,
   type PoolSpeedTestResult,
   type PoolStrategySetting,
@@ -194,6 +197,130 @@ export function createNetworkPoolOperations(config: NetworkPoolConfig) {
       }
       return { created, skipped };
     },
+    async createPoolBatch(
+      access: AccessDecision | undefined,
+      requests: readonly CreateNetworkPoolRequest[],
+    ): Promise<{ created: NetworkPoolResponse[]; failed: number }> {
+      const a = requireTenantScope(access, "dashboard:write");
+      let failed = 0;
+      const records: NetworkPoolRecord[] = [];
+      for (const request of requests) {
+        try {
+          const normalizedConfig = normalizePoolConfig(request.config);
+          const endpoint = request.endpoint;
+          validateEndpoint(request.kind, endpoint, config.ssrfPolicy);
+          validateTransportConfig(request.kind, normalizedConfig);
+          validatePoolLimits(request.maxInflight, request.weight);
+          records.push({
+            id: crypto.randomUUID(),
+            kind: request.kind,
+            endpoint,
+            maxInflight: request.maxInflight ?? 10,
+            weight: request.weight ?? 100,
+            status: "active",
+            inflight: 0,
+            consecutiveFailures: 0,
+            tenantId: a.tenantId,
+            createdAt: new Date().toISOString(),
+            ...(request.label === undefined ? {} : { label: request.label }),
+            ...(Object.keys(normalizedConfig).length > 0 ? { config: normalizedConfig } : {}),
+            ...(request.credential === undefined ? {} : { credential: request.credential }),
+            ...(request.credential !== undefined ? { hasCredential: true } : {}),
+            ...(typeof request.quotaBytes === "number" ? { quotaBytes: request.quotaBytes } : {}),
+            ...(request.family === undefined ? {} : { family: request.family }),
+          });
+        } catch {
+          failed += 1;
+        }
+      }
+      // All rows land in one statement: a long paste was one round trip per
+      // proxy, which made a 40-line import take minutes.
+      let created: NetworkPoolResponse[] = [];
+      if (records.length > 0) {
+        if (config.store.createMany) {
+          await config.store.createMany(records);
+        } else {
+          for (const record of records) await config.store.create(record);
+        }
+        created = records.map((record) => sanitizePoolResponse(record));
+        await config.auditSink?.record({
+          access: a,
+          action: "network_pool.created",
+          target: created[0]!.id,
+          detail: { count: created.length, failed },
+        });
+        await config.snapshotInvalidator?.invalidate();
+      }
+      return { created, failed };
+    },
+    async deletePoolBatch(
+      access: AccessDecision | undefined,
+      poolIds: readonly string[],
+    ): Promise<{ deleted: number; failed: number }> {
+      const a = requireTenantScope(access, "dashboard:write");
+      let ok = 0;
+      if (config.store.deleteMany) {
+        ok = await config.store.deleteMany(a.tenantId, poolIds);
+      } else {
+        for (const poolId of poolIds) {
+          try {
+            if (await config.store.delete(a.tenantId, poolId)) ok += 1;
+          } catch {
+            // Missing rows are not failures for a bulk delete.
+          }
+        }
+      }
+      // Cooldown markers and cached agents are best-effort here: the rows are
+      // already gone, so a failed cleanup must not report a failed delete.
+      await Promise.all(
+        poolIds.map(async (poolId) => {
+          await config.poolAgentReleaser?.releasePool(poolId, a.tenantId).catch(() => undefined);
+          await config.poolSelector?.clearPoolCooldowns(poolId).catch(() => undefined);
+        }),
+      );
+      await config.auditSink?.record({
+        access: a,
+        action: "network_pool.deleted",
+        target: poolIds[0] ?? "",
+        detail: { count: ok, failed: poolIds.length - ok },
+      });
+      if (ok > 0) await config.snapshotInvalidator?.invalidate();
+      return { deleted: ok, failed: poolIds.length - ok };
+    },
+    async setPoolStatusBatch(
+      access: AccessDecision | undefined,
+      poolIds: readonly string[],
+      status: PoolStatus,
+    ): Promise<{ updated: number }> {
+      const a = requireTenantScope(access, "dashboard:write");
+      let updated = 0;
+      if (config.store.updateManyStatus) {
+        updated = await config.store.updateManyStatus(a.tenantId, poolIds, status);
+      } else {
+        for (const poolId of poolIds) {
+          try {
+            if (await config.store.update(a.tenantId, poolId, { status })) updated += 1;
+          } catch {
+            // Missing rows are not failures for a bulk status change.
+          }
+        }
+      }
+      // Any mutation can invalidate the cached dial agent; the batch releases
+      // each selected pool's agent so the next dispatch rebuilds fresh.
+      await Promise.all(
+        poolIds.map(async (poolId) => {
+          await config.poolAgentReleaser?.releasePool(poolId, a.tenantId).catch(() => undefined);
+        }),
+      );
+      await config.auditSink?.record({
+        access: a,
+        action: "network_pool.updated",
+        target: poolIds[0] ?? "",
+        detail: { status, count: updated },
+      });
+      if (updated > 0) await config.snapshotInvalidator?.invalidate();
+      return { updated };
+    },
     async createPool(
         access: AccessDecision | undefined,
         request: CreateNetworkPoolRequest,
@@ -255,6 +382,15 @@ export function createNetworkPoolOperations(config: NetworkPoolConfig) {
           validateEndpoint(effectiveKind, effectiveEndpoint, config.ssrfPolicy);
           validateTransportConfig(effectiveKind, normalizedConfig);
           request = { ...request, endpoint: effectiveEndpoint, config: normalizedConfig };
+          // A changed endpoint is a different egress address, so its cooldowns
+          // — like its probe verdict — no longer describe this pool. Drop the
+          // durable markers with the volatile ones, or a replaced proxy would
+          // come back online and immediately be skipped because the previous
+          // address's flags were restored at boot.
+          if (request.endpoint !== undefined && request.endpoint !== existing.endpoint) {
+            await config.poolSelector?.clearPoolCooldowns(poolId).catch(() => undefined);
+            await config.store.dropAllProviderCooldownMarkers?.(a.tenantId, poolId);
+          }
         }
         const updated = await config.store.update(
           a.tenantId,
@@ -380,18 +516,61 @@ export function createNetworkPoolOperations(config: NetworkPoolConfig) {
       // still picked a red one": the red was a colour, not a decision.
       //
       // `blocked` retires the egress for this provider, so the next dispatch
-      // skips it. Anything else clears a previous block, because the address is
+      // skips it. `allowed` clears a previous block, because the address is
       // serving again — an operator who replaced a proxy must not have to find
       // a separate button to un-retire it.
+      //
+      // `blocked` retires the egress — but only for a provider whose limits
+      // follow the address. `flagProviderBlocked` refuses everyone else, so the
+      // verdict reported back is what actually happened, not what was asked.
       if (result.status === "blocked") {
-        await config.poolSelector.flagProviderBlocked(
+        const entry = await config.poolSelector.flagProviderBlocked(
           poolId,
           providerId,
           "Upstream refused this egress address",
         );
-        return { ...result, blockedPool: true };
+        if (entry.blocked === true) {
+          // The retirement claim lands in the mirror and the health history
+          // with its provider named, like every other verdict — an enforced
+          // state the operator cannot see is the "colour, not a decision"
+          // failure mode again.
+          await config.store.recordProviderProbeCooldown?.(a.tenantId, poolId, providerId, {
+            until: entry.until,
+            reason: entry.reason,
+            blocked: true,
+          });
+        }
+        return { ...result, blockedPool: entry.blocked === true };
+      }
+      // A measured success releases the address — in rotation and in the
+      // durable mirror the next boot restores from.
+      if (result.status === "allowed") {
+        await config.poolSelector.clearProviderCooldown(poolId, providerId);
+        await config.store.dropProviderCooldownMarker?.(a.tenantId, poolId, providerId, false);
+        return { ...result, blockedPool: false };
+      }
+      // `throttled` (429) proves the address is *serving* — its rate limiter
+      // answered — so a stale "retired" claim is corrected. And the throttle
+      // itself is a measurement: the egress just demonstrated it is out of
+      // allowance, which is exactly when dispatch should skip it **and** the
+      // operator sees why. Flag the same (configured) window a dispatch-side
+      // 429 would, in rotation memory *and* in the durable mirror, plus the
+      // health event the activity panel lists. Only a 429 gap was leaving a
+      // rate-limited pool invisible and unflagged.
+      if (result.status === "throttled") {
+        await config.poolSelector.clearProviderBlockedOnly(poolId, providerId);
+        await config.store.dropProviderCooldownMarker?.(a.tenantId, poolId, providerId, true);
+        if (providerRateLimitIsIpScoped(providerId)) {
+          const durationMs = resolvePoolProviderRateLimitCooldownMs();
+          const reason = "Operator probe measured a provider 429";
+          await config.poolSelector.flagProviderCooldown(poolId, providerId, durationMs, reason);
+          await config.store.recordProviderProbeCooldown?.(a.tenantId, poolId, providerId, {
+            until: Date.now() + durationMs,
+            reason,
+          });
+        }
       }
-      await config.poolSelector.clearProviderCooldown(poolId, providerId);
+      // `unreachable` learned nothing about the address, so nothing changes.
       return { ...result, blockedPool: false };
     },
     /**

package/src/console/routing/pools/relay-deploy.ts

@@ -63,7 +63,14 @@ const handler = {
     }
     try {
       const response = await fetch(targetUrl, init);
-      return new Response(response.body, { status: response.status, headers: response.headers });
+      // Forward the body verbatim. Content-encoding/content-length must be
+      // stripped: the relay's own runtime already decoded the stream it hands
+      // us, so re-sending the original gzip headers makes the caller decompress
+      // twice (ZlibError). The body bytes are already plaintext here.
+      const headers = new Headers(response.headers);
+      headers.delete("content-encoding");
+      headers.delete("content-length");
+      return new Response(response.body, { status: response.status, headers });
     } catch (error) {
       return new Response(JSON.stringify({ error: String((error && error.message) || error) }), {
         status: 502,
@@ -217,10 +224,14 @@ async function deployVercel(
   // PATCH is used rather than POST /v1/projects because the project already
   // exists once the first deployment lands; the first deploy below creates it,
   // and this call is what pins its region before that deploy runs its build.
+  //
+  // `ssoProtection: null` disables Vercel Authentication (SSO) on the project.
+  // Without it every deployment is protected and answers `302 → vercel.com/
+  // sso-api` to anonymous callers — the relay URL exists but is not public.
   await fetchImpl(`https://api.vercel.com/v9/projects/${encodeURIComponent(projectName)}`, {
     method: "PATCH",
     headers: auth,
-    body: JSON.stringify({ serverlessFunctionRegion: region }),
+    body: JSON.stringify({ serverlessFunctionRegion: region, ssoProtection: null }),
   }).catch(() => undefined);
   const deployRes = await fetchImpl("https://api.vercel.com/v13/deployments", {
     method: "POST",
@@ -243,11 +254,13 @@ async function deployVercel(
     throw new GatewayError("platform_unavailable", deployRes.status, `Vercel deploy failed: ${await readError(deployRes)}`);
   // Pin the region again after the deploy created the project: the PATCH above
   // is a no-op on first deploy (the project does not exist yet), and a redeploy
-  // is what Vercel requires for a region change to take effect anyway.
+  // is what Vercel requires for a region change to take effect anyway. The SSO
+  // protection reset is repeated for the same reason — the first PATCH had no
+  // project to act on.
   await fetchImpl(`https://api.vercel.com/v9/projects/${encodeURIComponent(projectName)}`, {
     method: "PATCH",
     headers: auth,
-    body: JSON.stringify({ serverlessFunctionRegion: region }),
+    body: JSON.stringify({ serverlessFunctionRegion: region, ssoProtection: null }),
   }).catch(() => undefined);
   await fetchImpl("https://api.vercel.com/v13/deployments", {
     method: "POST",

package/src/console/routing/pools/routes.ts

@@ -87,6 +87,36 @@ export function createNetworkPoolRoutes(config: NetworkPoolConfig): Elysia {
     .delete("/:poolId", async ({ request, params }) => {
       return await factory.deletePool(config.accessResolver(request), params.poolId);
 })
+    .post(
+      "/bulk/create",
+      { body: t.Object({ targets: t.Array(createPoolBody, { minItems: 1, maxItems: MAX_BATCH_PROBE_TARGETS }) }) },
+      async ({ request, body, set }) => {
+        set.status = 200;
+        return await factory.createPoolBatch(
+          config.accessResolver(request),
+          body.targets as CreateNetworkPoolRequest[],
+        );
+      },
+    )
+    .post(
+      "/bulk/delete",
+      { body: t.Object({ ids: t.Array(t.String(), { minItems: 1 }) }) },
+      async ({ request, body }) => {
+        return await factory.deletePoolBatch(config.accessResolver(request), body.ids);
+      },
+    )
+    .post(
+      "/bulk/status",
+      {
+        body: t.Object({
+          ids: t.Array(t.String(), { minItems: 1 }),
+          status: literalUnion(POOL_STATUSES),
+        }),
+      },
+      async ({ request, body }) => {
+        return await factory.setPoolStatusBatch(config.accessResolver(request), body.ids, body.status);
+      },
+    )
     .get("/:poolId/health-events", async ({ request, params }) => {
       return await factory.listPoolHealthEvents(config.accessResolver(request), params.poolId);
 })