cartethyia 2.6.6 → 2.7.0

package/src/providers/operations/account-health-service.ts

@@ -378,14 +378,57 @@ function accountDeadlineMs(
  * Matched on the code *and* the account-restriction wording, so an unrelated
  * upstream that happens to say "restricted" in prose cannot disable accounts.
  */
-function isBuddyAccountBanned(lower: string, providerCode: string): boolean {
+function isBuddyAccountBanned(
+  lower: string,
+  providerCode: string,
+  upstreamBody: unknown,
+): boolean {
   if (providerCode !== "11140") return false;
+  if (banWording(lower)) return true;
+  // The message alone is not enough, and this is the case that proved it: the
+  // provider's message for a ban is the same two words it uses for a refused
+  // request — "request illegal" — while the actual verdict ("Your account has
+  // been restricted due to unusual activity") lives in `displayMsg`, which the
+  // message extractor never reads. Matching only the message therefore read
+  // every ban as a mere request refusal: no category, no disable, and the
+  // account kept serving.
+  const body = isRecord(upstreamBody) ? upstreamBody : undefined;
+  if (body === undefined) return false;
+  for (const field of ["displayMsg", "displayTips", "msg", "message"]) {
+    const value = body[field];
+    if (typeof value === "string" && banWording(value.toLowerCase())) return true;
+    // `displayMsg` is a per-language map: `{ en: "…", zh: "…" }`. Any
+    // language states the same verdict, so all of them are read.
+    if (!isRecord(value)) continue;
+    for (const nested of Object.values(value)) {
+      if (typeof nested === "string" && banWording(nested.toLowerCase())) return true;
+    }
+  }
+  // `actions` names the remedy: an appeal can only be offered for a verdict
+  // about the account, and "edit input" only ever describes a request.
+  const actions = body["actions"];
+  if (Array.isArray(actions)) {
+    for (const action of actions) {
+      if (typeof action !== "string") continue;
+      if (action.toLowerCase().includes("appeal") || action.toLowerCase().includes("feedback")) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+/** The wording an upstream uses when it restricted the account, not the request. */
+function banWording(lower: string): boolean {
   return (
     lower.includes("restricted") ||
     lower.includes("has been banned") ||
     lower.includes("unusual activity") ||
     lower.includes("appeal this restriction") ||
-    lower.includes("submit feedback to appeal")
+    lower.includes("submit feedback to appeal") ||
+    lower.includes("account has been") ||
+    lower.includes("账号存在异常行为") ||
+    lower.includes("帳號存在異常行為")
   );
 }
 
@@ -619,6 +662,12 @@ function accountErrorContext(error: unknown, options?: AccountFailureEvidence):
   const bodyCooldown =
     parseUpstreamBodyReset(options?.upstreamBody) ??
     (rawUpstreamResponse === undefined ? null : parseUpstreamBodyReset(tryParseUpstreamBody(rawUpstreamResponse)));
+  // The body the ban is named in. The caller's parsed body when there is one,
+  // and the raw response text otherwise — the dispatch path stores the
+  // upstream's JSON there, and that is where `displayMsg` lives.
+  const banBody =
+    options?.upstreamBody ??
+    (rawUpstreamResponse === undefined ? undefined : tryParseUpstreamBody(rawUpstreamResponse));
   // Whether the provider itself named a window. Decided once, from the
   // evidence, so every branch of this classification agrees about it.
   const providerStated =
@@ -665,7 +714,7 @@ function accountErrorContext(error: unknown, options?: AccountFailureEvidence):
     canMutate,
     providerCode,
     statedCooldownMs,
-    buddyBanned: isBuddyAccountBanned(lower, providerCode),
+    buddyBanned: isBuddyAccountBanned(lower, providerCode, banBody),
     reason,
     result,
   };

package/src/security/abuse.ts

@@ -1,4 +1,6 @@
 import { GatewayError } from "../protocol/gateway-error";
+import { log } from "../observability/logger";
+import { metrics } from "../observability/metrics";
 import { redisEvalTuple, type RedisClient } from "../persistence/redis";
 
 export interface ClientIdentity {
@@ -419,7 +421,21 @@ export class IpAbuseProtectionService {
         banDurationMs: this.banDurationMs,
       });
 
+      // Logged once, at the transition. A ban is the one state here the
+      // operator cannot see from a metric and cannot wait out by guessing:
+      // every request afterwards answers 429 with the same message, whether
+      // the address is banned or merely over the window. Naming the address,
+      // the count that tripped it, and how long it lasts is what turns
+      // "429, sometimes" into something to act on.
+      if (outcome.bannedNow) {
+        log.warn(
+          `ip abuse: banned ${ip} for ${Math.round(this.banDurationMs / 60_000)}min` +
+            ` (route ${input.route}, ceiling ${this.banThreshold})`,
+        );
+      }
+
       if (outcome.banned) {
+        metrics.cartethyia_ip_abuse_refusals_total.inc(1, { reason: "ip-banned" });
         throw new GatewayError("quota_exceeded", 429, "This address is temporarily banned for too many failed requests. Wait for the ban to lift, then slow down.", {
           // Distinguishes this from the rate-limited 429 below and from an API
           // key's own budget. It used to be "admission-unavailable", which is
@@ -437,6 +453,7 @@ export class IpAbuseProtectionService {
       }
 
       if (outcome.count > this.maxRequestsPerWindow) {
+        metrics.cartethyia_ip_abuse_refusals_total.inc(1, { reason: "ip-rate-limited" });
         throw new GatewayError("quota_exceeded", 429, "This address is sending requests faster than the abuse limit allows. Back off and retry after the window rolls over.", {
           reason: "ip-rate-limited",
           ip,

package/src/transport/guards/auth-guard.ts

@@ -1,127 +1,109 @@
-import { log } from "../../observability/logger";
-// API-key authentication guard for `/v1/*`: token resolution, scope check,
-// client-router denylist, and the model-abuse ban gate.
-import { Elysia } from "elysia";
-import { GatewayError } from "../../protocol/gateway-error";
-import type { ProxyRequestStateStore } from "../request/state";
-import { fastPathname } from "../pathname";
-import type { CartethyiaDatabase } from "../../persistence/db-handle";
-import { requestToken, resolveApiKeyAuthorization, unknownApiKeyError } from "../../security/api-key-auth";
-import {
-  deniedClientRouter,
-  detectClientRouter,
-} from "../../security/client-router-fingerprint";
-import { createAccessDecision } from "../../security/access-control";
-import { modelAbuseBannedError, type ModelStrikeService } from "../../security/model-abuse";
-
-/**
- * Stateless Elysia `beforeHandle` gateway check: API-key authentication,
- * wired into the middleware chain in `app.ts`.
- */
-export function createApiKeyAuthenticationMiddleware(deps: {
-  readonly db: CartethyiaDatabase;
-  readonly stateStore: ProxyRequestStateStore;
-  /**
-   * Model-abuse ban gate. A banned IP or key is refused here, before the
-   * canonical parse and route preparation, and before `state.authorization` is
-   * assigned — so a banned caller's attempts produce no telemetry row and no
-   * console error, which is exactly the noise this layer exists to stop.
-   */
-  readonly modelStrikes?: Pick<ModelStrikeService, "check">;
-  /**
-   * Tenant invoke kill switch. Checked after the key resolves, because the
-   * tenant is a property of the key: the same gateway serves several tenants
-   * and one switching itself off must not stop the others.
-   */
-  readonly isInvokeDisabled?: (tenantId: string) => Promise<boolean | undefined>;
-}): Elysia {
-  // Single policy for every `/v1/*` gateway route: authenticated, scoped to
-  // `routing:invoke`, and tenant-bound (the snapshot's tenant is the only
-  // tenant this key may address). The check is inline rather than routed
-  // through a per-route policy table: Elysia's router already owns which
-  // paths exist, so a per-route table only added a second place to keep in
-  // sync.
-  return new Elysia()
-    .beforeHandle(async ({ request }) => {
-      const pathname = fastPathname(request.url);
-      if (!pathname.startsWith("/v1/")) return;
-      const token = requestToken(request.headers);
-      // A database failure here is not "the key is bad". Left to propagate it
-      // surfaced as a 500 whose log line carried the whole statement and its
-      // bound parameters — including the key hash — once per request. Under an
-      // overloaded database that is a credential leak written at request rate.
-      //
-      // It is also the wrong answer: the caller cannot fix it by sending a
-      // different key, so it is a capacity signal, not an auth one.
-      const authorization = await resolveApiKeyAuthorization(deps.db, token).catch((error: unknown) => {
-        // Only the error's identity is logged, never its message: a drizzle
-        // query error carries the full statement and its bound parameters, and
-        // on this path one of those parameters is the key hash.
-        // A sanitized stand-in, not the original: a drizzle query error's
-        // message is the statement plus its bound parameters, and one of those
-        // parameters is the key hash.
-        const safe = new Error("[redacted: credential-store query failed]");
-        safe.name = error instanceof Error ? error.name : "Error";
-        log.error("[auth] key lookup failed; treating as unavailable", safe);
-        throw new GatewayError(
-          "accounts_unavailable",
-          503,
-          "The gateway could not verify this API key because its credential store is unavailable. Retry shortly.",
-          {},
-          "cartethyia",
-        );
-      });
-      if (!authorization) throw unknownApiKeyError();
-      const decision = createAccessDecision({
-        id: authorization.id,
-        tenantId: authorization.tenantId,
-        scopes: authorization.scopes,
-      });
-      if (!decision.scopes.includes("routing:invoke"))
-        throw new GatewayError("invalid_request", 403, "This API key is not allowed to invoke routing. Grant it the routing:invoke scope, or use a key that has it.");
-      // Kill switch: refuse before any upstream is touched. Distinct from the
-      // scope check above, which is a property of the key — this is the tenant
-      // having stopped all traffic, and it applies to every key the tenant owns.
-      // 503 rather than 403: nothing about the request is invalid, and the
-      // caller should retry later instead of "fixing" a request that is fine.
-      if ((await deps.isInvokeDisabled?.(authorization.tenantId)) === true)
-        throw new GatewayError(
-          "platform_unavailable",
-          503,
-          "This gateway has invoke switched off. Nothing upstream is being dialed while the switch is off; turn it back on from the dashboard overview to resume routing.",
-        );
-      // A key may refuse a downstream router it has been resold to. This is a
-      // policy check on the resolved key, so it belongs with the other
-      // authorization decisions rather than in routing: the request is refused
-      // before a route is planned or an upstream is touched.
-      const denied = deniedClientRouter(
-        authorization.snapshot.client_router_denylist,
-        detectClientRouter({ headers: request.headers }),
-      );
-      if (denied !== null)
-        throw new GatewayError(
-          "client_router_denied",
-          403,
-          "No API invocation access for this client.",
-          { reason: "client_router_denied", clientRouter: denied },
-        );
-      // Abuse ban: refuse before parse/prepare and before authorization is
-      // recorded, so a banned caller cannot keep producing failed rows. The ban
-      // is keyed on the client address *and* the API key: one key can be shared
-      // by every recipient of a share link, so refusing the address alone would
-      // punish callers that did nothing, while refusing the key alone would let
-      // the caller that probed mint a fresh one.
-      if (deps.modelStrikes) {
-        const state = deps.stateStore.get(request);
-        const ip = state?.clientIdentity?.address;
-        if (ip !== undefined) {
-          const banned = await deps.modelStrikes
-            .check({ ip, apiKeyId: authorization.id })
-            .catch(() => false);
-          if (banned) throw modelAbuseBannedError();
-        }
-      }
-      deps.stateStore.require(request).authorization = authorization;
-    })
-    .as("plugin");
-}
+import { log } from "../../observability/logger";
+// API-key authentication guard for `/v1/*`: token resolution, scope check,
+// client-router denylist, and the model-abuse ban gate.
+import { Elysia } from "elysia";
+import { GatewayError } from "../../protocol/gateway-error";
+import type { ProxyRequestStateStore } from "../request/state";
+import { fastPathname } from "../pathname";
+import type { CartethyiaDatabase } from "../../persistence/db-handle";
+import { requestToken, resolveApiKeyAuthorization, unknownApiKeyError } from "../../security/api-key-auth";
+import {
+  deniedClientRouter,
+  detectClientRouter,
+} from "../../security/client-router-fingerprint";
+import { createAccessDecision } from "../../security/access-control";
+
+/**
+ * Stateless Elysia `beforeHandle` gateway check: API-key authentication,
+ * wired into the middleware chain in `app.ts`.
+ */
+export function createApiKeyAuthenticationMiddleware(deps: {
+  readonly db: CartethyiaDatabase;
+  readonly stateStore: ProxyRequestStateStore;
+  /**
+   * Model-abuse ban gate. A banned IP or key is refused here, before the
+   * canonical parse and route preparation, and before `state.authorization` is
+   * assigned — so a banned caller's attempts produce no telemetry row and no
+   * console error, which is exactly the noise this layer exists to stop.
+   */
+  /**
+   * Tenant invoke kill switch. Checked after the key resolves, because the
+   * tenant is a property of the key: the same gateway serves several tenants
+   * and one switching itself off must not stop the others.
+   */
+  readonly isInvokeDisabled?: (tenantId: string) => Promise<boolean | undefined>;
+}): Elysia {
+  // Single policy for every `/v1/*` gateway route: authenticated, scoped to
+  // `routing:invoke`, and tenant-bound (the snapshot's tenant is the only
+  // tenant this key may address). The check is inline rather than routed
+  // through a per-route policy table: Elysia's router already owns which
+  // paths exist, so a per-route table only added a second place to keep in
+  // sync.
+  return new Elysia()
+    .beforeHandle(async ({ request }) => {
+      const pathname = fastPathname(request.url);
+      if (!pathname.startsWith("/v1/")) return;
+      const token = requestToken(request.headers);
+      // A database failure here is not "the key is bad". Left to propagate it
+      // surfaced as a 500 whose log line carried the whole statement and its
+      // bound parameters — including the key hash — once per request. Under an
+      // overloaded database that is a credential leak written at request rate.
+      //
+      // It is also the wrong answer: the caller cannot fix it by sending a
+      // different key, so it is a capacity signal, not an auth one.
+      const authorization = await resolveApiKeyAuthorization(deps.db, token).catch((error: unknown) => {
+        // Only the error's identity is logged, never its message: a drizzle
+        // query error carries the full statement and its bound parameters, and
+        // on this path one of those parameters is the key hash.
+        // A sanitized stand-in, not the original: a drizzle query error's
+        // message is the statement plus its bound parameters, and one of those
+        // parameters is the key hash.
+        const safe = new Error("[redacted: credential-store query failed]");
+        safe.name = error instanceof Error ? error.name : "Error";
+        log.error("[auth] key lookup failed; treating as unavailable", safe);
+        throw new GatewayError(
+          "accounts_unavailable",
+          503,
+          "The gateway could not verify this API key because its credential store is unavailable. Retry shortly.",
+          {},
+          "cartethyia",
+        );
+      });
+      if (!authorization) throw unknownApiKeyError();
+      const decision = createAccessDecision({
+        id: authorization.id,
+        tenantId: authorization.tenantId,
+        scopes: authorization.scopes,
+      });
+      if (!decision.scopes.includes("routing:invoke"))
+        throw new GatewayError("invalid_request", 403, "This API key is not allowed to invoke routing. Grant it the routing:invoke scope, or use a key that has it.");
+      // Kill switch: refuse before any upstream is touched. Distinct from the
+      // scope check above, which is a property of the key — this is the tenant
+      // having stopped all traffic, and it applies to every key the tenant owns.
+      // 503 rather than 403: nothing about the request is invalid, and the
+      // caller should retry later instead of "fixing" a request that is fine.
+      if ((await deps.isInvokeDisabled?.(authorization.tenantId)) === true)
+        throw new GatewayError(
+          "platform_unavailable",
+          503,
+          "This gateway has invoke switched off. Nothing upstream is being dialed while the switch is off; turn it back on from the dashboard overview to resume routing.",
+        );
+      // A key may refuse a downstream router it has been resold to. This is a
+      // policy check on the resolved key, so it belongs with the other
+      // authorization decisions rather than in routing: the request is refused
+      // before a route is planned or an upstream is touched.
+      const denied = deniedClientRouter(
+        authorization.snapshot.client_router_denylist,
+        detectClientRouter({ headers: request.headers }),
+      );
+      if (denied !== null)
+        throw new GatewayError(
+          "client_router_denied",
+          403,
+          "No API invocation access for this client.",
+          { reason: "client_router_denied", clientRouter: denied },
+        );
+      deps.stateStore.require(request).authorization = authorization;
+    })
+    .as("plugin");
+}