cartethyia 2.7.9 → 2.8.0

package/src/providers/operations/account-health-service.ts

@@ -196,6 +196,7 @@ export async function recordAccountFailure(
     //   1m limit must keep its short cooldown, not park the account for 24h).
     const autoDisableTarget =
       AUTO_DISABLE_TARGET_CATEGORIES.has(effective.category) &&
+      effective.modelScoped !== true &&
       effective.status === "cooldown" &&
       effective.mutatesAccount &&
       options?.origin === "upstream";
@@ -230,7 +231,10 @@ export async function recordAccountFailure(
     // D02: when the auto-disable intercept fired, the target category must not
     // sink into a model-scoped cooldown — the outcome is an account-wide
     // disable.
-    const shouldModelCooldown = !autoDisableActive && modelId !== undefined && isThrottle;
+    const shouldModelCooldown =
+      !autoDisableActive &&
+      modelId !== undefined &&
+      (isThrottle || effective.modelScoped === true);
     // The account-wide streak, escalated for a provider-stated window. Read once
     // so the model-scoped and account-wide paths below cannot disagree about it:
     // this is the count *before* this failure, which the UPDATE below increments.
@@ -694,6 +698,83 @@ async function writeHealthEvent(
   await client.insert(healthEvents).values({ id: randomUUID(), ...row });
 }
 
+/**
+ * The one internal seam that applies a health verdict to an account row.
+ *
+ * Every producer that persists account-state semantics — dispatch failure
+ * classification, OAuth refresh definitive failure, quota refresh parking —
+ * converges here instead of writing `status`/`cooldownUntil`/error fields
+ * independently. Producers may still classify differently; they may not
+ * persist account-state semantics separately.
+ *
+ * This writes the account-wide health state plus a health audit row, inside the
+ * caller's transaction (`client`). It returns `true` when the account row was
+ * actually updated (i.e. the account exists and is not operator-disabled with
+ * `preserveOperatorDisable`), so the caller knows whether routing must be
+ * invalidated (routesVisible) or the lease released.
+ */
+export async function applyAccountVerdict(
+  client: CartethyiaDatabase,
+  verdict: {
+    readonly accountId: string;
+    readonly status: "cooldown" | "disabled";
+    readonly reason: string;
+    readonly category: string | null;
+    readonly cooldownUntil?: Date | null;
+    /** When true, an operator-disable (status `disabled`, no deadline) is left untouched. */
+    readonly preserveOperatorDisable?: boolean;
+    /** Health event transition status (defaults to the verdict status). */
+    readonly fromStatus?: "active" | "cooldown" | "disabled" | null;
+    readonly modelId?: string | null;
+    readonly createdAt?: Date;
+  },
+): Promise<boolean> {
+  return withTransaction(
+    client,
+    async (tx) => {
+      const rows = await tx
+        .select({
+          status: providerAccounts.status,
+          cooldownUntil: providerAccounts.cooldownUntil,
+        })
+        .from(providerAccounts)
+        .where(eq(providerAccounts.id, verdict.accountId))
+        .limit(1);
+      const row = rows[0];
+      if (!row) return false;
+      if (verdict.preserveOperatorDisable === true) {
+        const isOperatorDisable = row.status === "disabled" && row.cooldownUntil == null;
+        if (isOperatorDisable) return false;
+      }
+      const now = verdict.createdAt ?? new Date();
+      await tx
+        .update(providerAccounts)
+        .set({
+          status: verdict.status,
+          lastError: (verdict.reason ?? "Account failed").slice(0, 500),
+          lastErrorCategory: verdict.category ?? "check_failed",
+          lastErrorAt: now,
+          ...(verdict.cooldownUntil === undefined
+            ? {}
+            : { cooldownUntil: verdict.cooldownUntil }),
+        })
+        .where(eq(providerAccounts.id, verdict.accountId));
+      await writeHealthEvent(tx, {
+        entityKind: "account",
+        accountId: verdict.accountId,
+        fromStatus: verdict.fromStatus ?? (row.status as "active" | "cooldown" | "disabled" | null),
+        toStatus: verdict.status,
+        reason: verdict.reason,
+        errorCategory: verdict.category,
+        modelId: verdict.modelId ?? null,
+        createdAt: now,
+      });
+      return true;
+    },
+    false,
+  );
+}
+
 export async function reportAttemptOutcome(
   db: CartethyiaDatabase,
   report: AttemptHealthReport,

package/src/providers/operations/account-refresh-status.ts

@@ -9,10 +9,10 @@
  * report `"active"` just because a refresh succeeded.
  */
 import { randomUUID } from "node:crypto";
-import { eq } from "drizzle-orm";
 import type { CartethyiaDatabase } from "../../persistence/db-handle";
-import { healthEvents, providerAccounts, providerOauthStates } from "../../persistence/schema";
+import { healthEvents } from "../../persistence/schema";
 import { pushStructuredConsoleLog } from "../../observability/log-ring";
+import { loadAccountWithOauthState } from "./provider-credential-service";
 import type { OAuthRefreshService, OAuthTokenRefresher } from "../authentication/oauth-refresh-service";
 
 export type RefreshAccountStatusCode =
@@ -62,47 +62,32 @@ async function loadAccount(
   db: CartethyiaDatabase,
   accountId: string,
 ): Promise<LoadedAccount | undefined> {
-  const selectBuilder = db
-    .select({
-      id: providerAccounts.id,
-      providerId: providerAccounts.providerId,
-      credentialKind: providerAccounts.credentialKind,
-      status: providerAccounts.status,
-      staticToken: providerAccounts.staticToken,
-      refreshCiphertext: providerOauthStates.refreshCiphertext,
-      expiresAt: providerOauthStates.expiresAt,
-    })
-    .from(providerAccounts);
-  const query =
-    "leftJoin" in selectBuilder
-      ? selectBuilder.leftJoin(
-          providerOauthStates,
-          eq(providerOauthStates.providerAccountId, providerAccounts.id),
-        )
-      : selectBuilder;
-  const rows = await query.where(eq(providerAccounts.id, accountId)).limit(1);
-  return rows[0];
+  const loaded = await loadAccountWithOauthState(db, accountId, {
+    includeStatus: true,
+    includeExpiry: true,
+  });
+  if (!loaded || loaded.status === undefined || loaded.expiresAt === undefined) return undefined;
+  return {
+    id: loaded.id,
+    providerId: loaded.providerId,
+    credentialKind: loaded.credentialKind,
+    status: loaded.status,
+    staticToken: loaded.staticToken,
+    refreshCiphertext: loaded.refreshCiphertext,
+    expiresAt: loaded.expiresAt,
+  };
 }
 
 async function readStatusAfter(
   db: CartethyiaDatabase,
   accountId: string,
 ): Promise<{ status: string; expiresAt: Date | null } | undefined> {
-  const selectBuilder = db
-    .select({
-      status: providerAccounts.status,
-      expiresAt: providerOauthStates.expiresAt,
-    })
-    .from(providerAccounts);
-  const query =
-    "leftJoin" in selectBuilder
-      ? selectBuilder.leftJoin(
-          providerOauthStates,
-          eq(providerOauthStates.providerAccountId, providerAccounts.id),
-        )
-      : selectBuilder;
-  const rows = await query.where(eq(providerAccounts.id, accountId)).limit(1);
-  return rows[0];
+  const loaded = await loadAccountWithOauthState(db, accountId, {
+    includeStatus: true,
+    includeExpiry: true,
+  });
+  if (!loaded || loaded.status === undefined || loaded.expiresAt === undefined) return undefined;
+  return { status: loaded.status, expiresAt: loaded.expiresAt };
 }
 
 /** Non-mutating health event: records the attempt, never changes status. */

package/src/providers/operations/account-tier.ts

@@ -8,24 +8,22 @@
  *
  * The mapping is deliberately conservative, following the same posture as
  * `plan-gate.ts`: a tier is only claimed when the string proves it. "free"
- * requires a free marker in the string, "pro" requires a team/enterprise/org
- * signal, and "paid" is only assigned for providers whose plan strings are
- * evidenced (Trae, the buddy family, Cursor). Everything else — an unknown
- * provider, an unrecognized plan, a `null` plan — resolves to `null`, which
- * keeps the legacy behavior intact (unknown tier ⇒ no rule applies ⇒ the old
- * constants drive the cooldown).
+ * requires the shared free marker set (exported there), "pro" requires a
+ * team/enterprise/org signal, and "paid" is only assigned for providers whose
+ * plan strings are evidenced (Trae, the buddy family, Cursor). Everything else
+ * — an unknown provider, an unrecognized plan, a `null` plan — resolves to
+ * `null`, which keeps the legacy behavior intact (unknown tier ⇒ no rule
+ * applies ⇒ the old constants drive the cooldown).
  */
 
+import { FREE_PLAN_MARKERS } from "../../transport/routing/plan-gate";
+import { BUDDY_PROVIDER_IDS } from "../provider-metadata";
+import { QODER_CHECKIN_PROVIDER_IDS } from "../integrations/qoder/qoder-checkin";
+import { TRAE_PROVIDER_IDS } from "../integrations/trae/trae-realm";
+
 /** The three-tier taxonomy for cooldown rules. `null` = unknown. */
 export type AccountTier = "free" | "paid" | "pro" | null;
 
-const TRAE_PROVIDER_IDS: ReadonlySet<string> = new Set(["trae", "traecn"]);
-const BUDDY_PROVIDER_IDS: ReadonlySet<string> = new Set([
-  "cb",
-  "cbcn",
-]);
-const QODER_PROVIDER_IDS: ReadonlySet<string> = new Set(["qoder", "qodercn"]);
-
 /** Team/enterprise/org signals that prove a "pro" tier. */
 const PRO_PLAN_MARKERS: readonly string[] = ["team", "enterprise", "org", "pro"];
 
@@ -41,10 +39,11 @@ export function mapPlanToTier(providerId: string, plan: string | null): AccountT
   const lower = (plan ?? "").trim().toLowerCase();
   if (lower.length === 0) return null;
   // Free is only claimed when the string proves it — the same marker set the
-  // routing plan-gate uses, so the two cannot disagree about an account.
-  if (lower.includes("free")) return "free";
+  // routing plan-gate uses (imported, not copied), so the two cannot disagree
+  // about an account.
+  if (FREE_PLAN_MARKERS.some((marker) => lower.includes(marker))) return "free";
 
-  if (QODER_PROVIDER_IDS.has(providerId.toLowerCase())) {
+  if ((QODER_CHECKIN_PROVIDER_IDS as readonly string[]).includes(providerId.toLowerCase())) {
     // Qoder's `userType` is `personal_standard` by default; team/enterprise/
     // org suffixes prove a higher tier. `personal_standard` itself is not
     // verified against the taxonomy, so it stays unknown rather than guessed.

package/src/providers/operations/client-version-resolver.ts

@@ -11,7 +11,7 @@
  * Discovery is owned by the background client-version monitor. Request paths
  * only call the synchronous `get`/`snapshot` accessors and never fetch.
  */
-import { getCachedVersion, resetVersionCacheForTesting } from "./provider-version-cache";
+import { getCachedVersion, resetVersionCacheForTesting } from "./provider-caches";
 import type { OutboundFetch } from "../provider-contracts";
 import { composeAbortSignal } from "../authentication/oauth-flow-store";
 

package/src/providers/operations/client-versions.ts

@@ -16,15 +16,6 @@ import {
   buildGrokShellUserAgent,
 } from "./cli-platform";
 
-async function qoderVersion(response: Response): Promise<string | null> {
-  try {
-    const parsed = (await response.json()) as { version?: unknown };
-    return isSemverish(parsed.version) ? parsed.version.trim() : null;
-  } catch {
-    return null;
-  }
-}
-
 /**
  * Newest Trae IDE version from the official changelog.
  *
@@ -67,11 +58,6 @@ async function traeVersion(response: Response): Promise<string | null> {
   }
 }
 
-async function clineClientVersion(response: Response): Promise<string | null> {
-  const data = (await response.json()) as { version?: unknown };
-  return isSemverish(data.version) ? data.version.trim() : null;
-}
-
 /**
  * Newest Cursor build from the official download API, as the IDE's own header
  * spells it.
@@ -95,29 +81,6 @@ async function cursorVersion(response: Response): Promise<string | null> {
   }
 }
 
-async function clineSdkVersion(response: Response): Promise<string | null> {
-  const data = (await response.json()) as {
-    version?: unknown;
-    "dist-tags"?: { latest?: unknown };
-  };
-  const candidate = data.version ?? data["dist-tags"]?.latest;
-  return isSemverish(candidate) ? candidate.trim() : null;
-}
-
-async function kimiVersion(response: Response): Promise<string | null> {
-  const data = (await response.json()) as { info?: { version?: unknown } };
-  return isSemverish(data.info?.version) ? data.info.version.trim() : null;
-}
-
-async function kiroCaskVersion(response: Response): Promise<string | null> {
-  try {
-    const data = (await response.json()) as { version?: unknown };
-    return isSemverish(data.version) ? data.version.trim() : null;
-  } catch {
-    return null;
-  }
-}
-
 async function codeBuddyCnVersion(response: Response): Promise<string | null> {
   try {
     const data = (await response.json()) as { version?: unknown };
@@ -236,12 +199,7 @@ export const VERSION_SOURCES = {
     // `@qoder-ai/qodercli/latest`) may report a newer CLI, but the CLI and
     // IDE are different products — the chat surface identity is the IDE's.
     fallback: "1.1.80",
-    sources: [
-      {
-        url: "https://registry.npmjs.org/@qoder-ai/qodercli/latest",
-        extract: qoderVersion,
-      },
-    ],
+    sources: [{ url: "https://registry.npmjs.org/@qoder-ai/qodercli/latest" }],
   },
   opencode: {
     key: "opencode",
@@ -274,21 +232,18 @@ export const VERSION_SOURCES = {
     fallback: "4.1.26",
     minVersion: "4.1.23",
     sources: [
-      {
-        url: "https://raw.githubusercontent.com/cline/cline/main/apps/vscode/package.json",
-        extract: clineClientVersion,
-      },
+      { url: "https://raw.githubusercontent.com/cline/cline/main/apps/vscode/package.json" },
     ],
   },
   clineSdk: {
     key: "cline-sdk",
     fallback: "0.2.0",
-    sources: [{ url: "https://registry.npmjs.org/@cline/sdk/latest", extract: clineSdkVersion }],
+    sources: [{ url: "https://registry.npmjs.org/@cline/sdk/latest" }],
   },
   clineCli: {
     key: "cline-cli",
     fallback: "3.0.74",
-    sources: [{ url: "https://registry.npmjs.org/cline/latest", extract: clineSdkVersion }],
+    sources: [{ url: "https://registry.npmjs.org/cline/latest" }],
   },
   codex: {
     key: "codex",
@@ -298,7 +253,7 @@ export const VERSION_SOURCES = {
   kimiCli: {
     key: "kimi-cli",
     fallback: "1.52.0",
-    sources: [{ url: "https://pypi.org/pypi/kimi-cli/json", extract: kimiVersion }],
+    sources: [{ url: "https://pypi.org/pypi/kimi-cli/json" }],
   },
   kiro: {
     key: "kiro",
@@ -309,7 +264,6 @@ export const VERSION_SOURCES = {
         // tracks the official installer URL and exposes its version as JSON;
         // keep the product page parser as a fallback.
         url: "https://formulae.brew.sh/api/cask/kiro.json",
-        extract: kiroCaskVersion,
       },
       { url: "https://kiro.dev/downloads/", extract: kiroVersion },
     ],

package/src/providers/operations/cooldown-rules.ts

@@ -28,24 +28,19 @@ type ProviderRules = Record<string, CategoryRules>;
 const DAY = 24 * 60 * 60 * 1000;
 const MINUTE = 60 * 1000;
 
-const COOLDOWN_RULES: ProviderRules = {
-  trae: {
-    model_unreachable: {
-      free: DAY,
-      paid: 15 * MINUTE,
-      pro: 15 * MINUTE,
-    },
+const TRAE_RULES: CategoryRules = {
+  model_unreachable: {
+    free: DAY,
+    paid: 15 * MINUTE,
+    pro: 15 * MINUTE,
   },
+};
+
+const COOLDOWN_RULES: ProviderRules = {
+  trae: TRAE_RULES,
   // Trae CN is the same upstream product under a second realm id — the
-  // classifier already treats the two identically (trae-region-unavailable,
-  // trae-quota-exhausted), so the rules mirror.
-  traecn: {
-    model_unreachable: {
-      free: DAY,
-      paid: 15 * MINUTE,
-      pro: 15 * MINUTE,
-    },
-  },
+  // classifier already treats the two identically, so the rules alias.
+  traecn: TRAE_RULES,
 };
 
 /**

package/src/providers/operations/{model-catalog-cache.ts → provider-caches.ts}

@@ -1,16 +1,17 @@
 /**
- * Process-lifetime model-catalog cache.
- *
- * `bundledModelCatalog` is called during startup seeding and again whenever a
- * caller rebuilds the routing/console catalog. Caching the loader *promise*
- * (not just the resolved value) makes concurrent and repeated calls for one
- * provider share a single load, and lets the dynamic catalog imports run once
- * per process. A rejected load is evicted so a later call can retry.
+ * In-process caches for model catalogs, model discovery, and client/provider versions.
  */
+
 import type { ModelDefinition } from "../provider-model-definition";
 import { trackModelCatalogLoad } from "../../observability/performance-metrics";
 import { resolveRedisClient } from "../../persistence/redis";
 import { cachedRead } from "../../console/shared/read-through-cache";
+import { TtlCacheFamily } from "../../persistence/ttl-cache";
+import { retryWithBackoff } from "../../network/retry";
+
+/* -------------------------------------------------------------------------- */
+/* 1. Model Catalog Cache                                                     */
+/* -------------------------------------------------------------------------- */
 
 /**
  * Provider entries retained before the least-recently-used catalog is evicted.
@@ -37,8 +38,7 @@ const catalogCache = new Map<string, Promise<readonly ModelDefinition[]>>();
 /**
  * Evicts the least-recently-used provider before inserting a new one once the
  * cache is at its bound. `Map` preserves insertion order and every cache hit
- * re-inserts its entry at the tail, so the head is the LRU key. Without this
- * bound the process-lifetime cache pinned one catalog per provider forever.
+ * re-inserts its entry at the tail, so the head is the LRU key.
  */
 function evictLeastRecentlyUsedCatalog(maxEntries: number): void {
   if (catalogCache.size < maxEntries) return;
@@ -87,13 +87,65 @@ export function resetModelCatalogCacheForTesting(): void {
   catalogCache.clear();
 }
 
-/**
- * Live entry count, for the `/metrics` gauge.
- *
- * Exported so a rebuild that saturates the cap is visible from outside the
- * process: before this, every in-process cache except the quota tier was
- * invisible, and growth could only be found with a heap dump.
- */
+/** Live entry count, for the `/metrics` gauge. */
 export function modelCatalogCacheSize(): number {
   return catalogCache.size;
 }
+
+/* -------------------------------------------------------------------------- */
+/* 2. Model Discovery Cache                                                   */
+/* -------------------------------------------------------------------------- */
+
+const MODEL_DISCOVERY_CACHE_TTL_MS = 10 * 60 * 1000;
+const discoveryCaches = new TtlCacheFamily<readonly ModelDefinition[]>({
+  defaultTtlMs: MODEL_DISCOVERY_CACHE_TTL_MS,
+});
+
+export interface CachedModelDiscoveryOptions {
+  readonly ttlMs?: number;
+}
+
+/** Returns cached models, retrying null/empty loads on the next call. */
+export async function getCachedModelDiscovery(
+  key: string,
+  loader: () => Promise<readonly ModelDefinition[] | null>,
+  options: CachedModelDiscoveryOptions = {},
+): Promise<readonly ModelDefinition[] | null> {
+  return discoveryCaches.for(options.ttlMs).load(key, loader, {
+    namespace: "model-discovery",
+    shouldCache: (models) => models.length > 0,
+  });
+}
+
+/** Drops all cached discoveries. Test-only. */
+export function resetModelDiscoveryCacheForTesting(): void {
+  discoveryCaches.clear();
+}
+
+/* -------------------------------------------------------------------------- */
+/* 3. Provider Version Cache                                                  */
+/* -------------------------------------------------------------------------- */
+
+const VERSION_CACHE_TTL_MS = 5 * 60 * 1000;
+const versionCaches = new TtlCacheFamily<string>({ defaultTtlMs: VERSION_CACHE_TTL_MS });
+
+export interface CachedVersionOptions {
+  readonly ttlMs?: number;
+  readonly maxRetries?: number;
+}
+
+export async function getCachedVersion(
+  key: string,
+  loader: () => Promise<string | null>,
+  options: CachedVersionOptions = {},
+): Promise<string | null> {
+  const loadWithRetry = () => retryWithBackoff(loader, { maxRetries: options.maxRetries ?? 2 });
+  return versionCaches.for(options.ttlMs).load(key, loadWithRetry, {
+    namespace: "provider-version",
+  });
+}
+
+/** Drops all cached versions. Test-only. */
+export function resetVersionCacheForTesting(): void {
+  versionCaches.clear();
+}

package/src/providers/operations/provider-catalog-service.ts

@@ -10,7 +10,7 @@ import { ProviderRegistry } from "../provider-registry";
 import { OpenAICompatibleAdapter } from "../adapters/compatible-adapter";
 import { resolveByokWireProfile } from "./byok-wire-profile";
 import { BUNDLED_PROVIDER_MODULES } from "../default-registry";
-import { getCachedModels } from "./model-catalog-cache";
+import { getCachedModels } from "./provider-caches";
 import { seedBundledModels } from "./provider-catalog-seeder";
 import type { CartethyiaDatabase } from "../../persistence/db-handle";
 import { providers } from "../../persistence/schema";