cartethyia 2.6.3 → 2.6.5

package/src/providers/operations/account-health-service.ts

@@ -160,7 +160,29 @@ export type AccountErrorCategory =
   | "model_capacity"
   | "model_unreachable"
   | "auth_invalidated"
+  /**
+   * Deprecated: an upstream ban. Kept so existing health rows keep rendering
+   * instead of showing a blank cause. No longer produced — see
+   * `policy_blocked_banned` and `policy_blocked_invalid_request`.
+   */
   | "policy_blocked"
+  /**
+   * The upstream banned this credential: a verdict on the account, e.g. buddy
+   * `11140` "request illegal" with "Your account has been restricted due to
+   * unusual activity". Cooldown cannot help — the account is not throttled, it
+   * is refused. Disabling immediately is a *feature*: a client's retry budget
+   * is spent in seconds, so surviving attempts burn sibling accounts that are
+   * still healthy.
+   */
+  | "policy_blocked_banned"
+  /**
+   * The upstream refused this request: a verdict on the request. e.g. buddy
+   * `11-128` "request illegal" with no account restriction. Distinguished from
+   * {@link policy_blocked_banned} because the correct responses differ — a
+   * banned account must be disabled, a malformed request must only stop being
+   * retried verbatim.
+   */
+  | "policy_blocked_invalid_request"
   | "server_error"
   | "timeout"
   | "unknown";
@@ -338,6 +360,35 @@ function accountDeadlineMs(
   return latest > 0 ? new Date(latest) : null;
 }
 
+/**
+ * Whether the upstream named a restriction on the account itself.
+ *
+ * The buddy family reuses "request illegal" for two unrelated rejections, and
+ * the difference is only in the body:
+ *
+ * - `11140` carries `displayMsg` about the account being restricted ("Your
+ *   account has been restricted due to unusual activity") and `actions`
+ *   offering an appeal. That is a ban — the account is refused, not throttled.
+ * - `11-128` is a refusal of this request; the account is fine.
+ *
+ * Proven account-scoped (2026-10-16): two live share-page requests carrying
+ * byte-identical bodies and headers to the same model differed only in the
+ * pooled account, and the blocked one 400'd while the other 200'd.
+ *
+ * Matched on the code *and* the account-restriction wording, so an unrelated
+ * upstream that happens to say "restricted" in prose cannot disable accounts.
+ */
+function isBuddyAccountBanned(lower: string, providerCode: string): boolean {
+  if (providerCode !== "11140") return false;
+  return (
+    lower.includes("restricted") ||
+    lower.includes("has been banned") ||
+    lower.includes("unusual activity") ||
+    lower.includes("appeal this restriction") ||
+    lower.includes("submit feedback to appeal")
+  );
+}
+
 /**
  * Bounded to a day, matching `MAX_COOLDOWN_MS` in the failure policy: a stated
  * window longer than that is a monthly or plan-level limit, and parking an
@@ -421,6 +472,12 @@ interface AccountErrorContext {
   readonly canMutate: boolean;
   readonly providerCode: string;
   readonly statedCooldownMs: number | null;
+  /**
+   * The upstream named a restriction on the account, not a refusal of the
+   * request. Read by the policy arm to pick between disabling the account and
+   * merely refusing the request.
+   */
+  readonly buddyBanned: boolean;
   readonly reason: (prefix: string) => string;
   readonly result: (
     category: AccountErrorCategory,
@@ -476,6 +533,13 @@ interface AccountErrorSignals {
   readonly credentialKind: string | undefined;
   /** Buddy-family provider: the only family with provider-specific codes below. */
   readonly buddy: boolean;
+  /**
+   * The upstream named a restriction on this account, not a refusal of the
+   * request. Only the `11140` shape carries `displayMsg` about the account
+   * being restricted, so the policy arm can tell a ban from a bad request
+   * instead of treating both as one cooldown.
+   */
+  readonly buddyBanned: boolean;
   readonly priceRefusal: boolean;
   readonly deploymentInfraError: boolean;
   readonly quotaSignal: boolean;
@@ -601,6 +665,7 @@ function accountErrorContext(error: unknown, options?: AccountFailureEvidence):
     canMutate,
     providerCode,
     statedCooldownMs,
+    buddyBanned: isBuddyAccountBanned(lower, providerCode),
     reason,
     result,
   };
@@ -623,6 +688,7 @@ export function classifyAccountError(
     providerCode,
 
     statedCooldownMs,
+    buddyBanned,
     reason,
     result,
   } = accountErrorContext(error, options);
@@ -761,19 +827,46 @@ export function classifyAccountError(
     lower.includes("request illegal") ||
     lower.includes("content blocked") ||
     lower.includes("account suspended");
+  // A ban is a verdict on the credential, and cooldown cannot answer it: the
+  // account is refused, not throttled. Disabling is the fix, and disabling
+  // *immediately* is a feature — a client's retry budget is spent in seconds,
+  // so a banned account that merely cooled stayed in the plan and burned
+  // sibling accounts that were still healthy before the window expired.
+  //
+  // The two shapes are told apart by the body, not the code alone: `11140`
+  // names a restriction on the account, `11-128` refuses the request.
   const policyBlock =
     policyRejection &&
     options?.providerId !== undefined &&
     BUDDY_PROVIDER_IDS.has(options.providerId.toLowerCase());
   if (policyBlock) {
-    const policyBlockMs = resolveAccountPolicyBlockCooldownMs();
+    if (buddyBanned) {
+      return result(
+        "policy_blocked_banned",
+        "disabled",
+        0,
+        null,
+        reason("Account banned by upstream"),
+        // A ban is evidence about the credential even at provider scope, for
+        // the same reason a named revocation is: the message is the evidence.
+        // Without this a stream frame's missing credential detail would leave a
+        // banned account serving.
+        true,
+      );
+    }
+    // Not a ban: the upstream refused this request. The account is fine, so the
+    // only correct move is to stop retrying it verbatim. Cooling the account
+    // here was the bug that made one malformed request look like a dead
+    // credential.
     return result(
-      "policy_blocked",
+      "policy_blocked_invalid_request",
       "cooldown",
-      policyBlockMs,
-      new Date(Date.now() + policyBlockMs),
-      reason("Provider content-policy block"),
-      origin === "upstream",
+      0,
+      null,
+      reason("Upstream refused this request"),
+      // Never mutates: a malformed body fails identically on every account, so
+      // cooling would burn a healthy credential on the gateway's own fault.
+      false,
     );
   }
   // These codes are a verdict on the request, not the account. Cooling the
@@ -829,14 +922,30 @@ export function classifyAccountError(
     options?.providerId !== undefined &&
     BUDDY_PROVIDER_IDS.has(options.providerId.toLowerCase());
   if (channelBlock) {
-    const channelBlockMs = resolveAccountPolicyBlockCooldownMs();
+    // Same split as the policy arm above: a ban disables, a channel refusal
+    // only stops this request. `11-128` is the request shape, so this branch
+    // nearly always takes the refusal path — but it shares the ban test so a
+    // `11140`-shaped body arriving as a channel rejection still disables.
+    if (buddyBanned) {
+      return result(
+        "policy_blocked_banned",
+        "disabled",
+        0,
+        null,
+        reason("Account banned by upstream"),
+        true,
+      );
+    }
     return result(
-      "policy_blocked",
+      "policy_blocked_invalid_request",
       "cooldown",
-      channelBlockMs,
-      new Date(Date.now() + channelBlockMs),
+      0,
+      null,
       reason("Provider channel rejection"),
-      origin === "upstream",
+      // The credential is valid; the upstream refuses this calling channel.
+      // Cooling was the bug: it kept a refused account in the plan, where a
+      // client's retry budget burned healthy siblings for the whole window.
+      false,
     );
   }
   // server-side tool (web search, x search, web fetch), not evidence about the
@@ -907,6 +1016,7 @@ export function classifyAccountError(
     staticToken,
     credentialKind,
     buddy,
+    buddyBanned,
     priceRefusal,
     deploymentInfraError,
     quotaSignal,
@@ -941,6 +1051,62 @@ export function classifyAccountError(
  * {@link AccountErrorRule}.
  */
 const ACCOUNT_ERROR_RULES: readonly AccountErrorRule[] = [
+  {
+    // Must precede "rate-limit": this rejection is a 429 whose own text says
+    // "rate limit", so the broader arm claimed it and cooled the account for
+    // the full 15-minute rate-limit window.
+    //
+    // A full max-inflight is a statement about the gateway's own routing, not
+    // about the credential: every account serving the model is busy right now.
+    // Cooling them marks healthy accounts as `cooldown` for a limit their
+    // provider never applied, which is what made every account read busy while
+    // incoming requests piled up — the accounts were never the problem, the
+    // ceiling was.
+    name: "gateway-capacity-exhausted",
+    match: (s) =>
+      s.lower.includes("at its concurrency limit") ||
+      s.lower.includes("every proxy pool for this provider is at its concurrency limit") ||
+      s.lower.includes("the gateway is at its concurrency limit"),
+    decide: (s) =>
+      s.result(
+        "model_capacity",
+        // `status` is the status the account would move to; it is never
+        // applied because `mutatesAccount` is false below, so this names the
+        // shape rather than a transition. Zero cooldown is the point: no wait
+        // is attached to the account, so the next request may use it at once.
+        "cooldown",
+        0,
+        null,
+        s.reason("Gateway concurrency ceiling reached; no account was at fault"),
+        // Never mutates: there is nothing wrong with the account to record.
+        false,
+      ),
+  },
+  {
+    // Must precede the unclassified fallback: the fallback is account-blind and
+    // would stamp this as a provider-wide failure, which is how one proxy
+    // refusing CONNECT put every account of every provider behind that pool
+    // into cooldown — opencode's dead front door taking codebuddy down with it.
+    //
+    // A CONNECT refusal is a statement about the proxy pool, not about a
+    // credential: the tunnel was never established, so no upstream ever saw the
+    // request and no account can be at fault. Scoped to the pool so the
+    // provider's own accounts stay in rotation.
+    name: "proxy-connect-refusal",
+    match: (s) =>
+      s.lower.includes("proxy connect failed") ||
+      (s.lower.includes("connect") && s.lower.includes("tunnel") && s.lower.includes("refus")),
+    decide: (s) =>
+      s.result(
+        "model_unreachable",
+        "cooldown",
+        0,
+        null,
+        s.reason("Proxy pool refused the CONNECT tunnel"),
+        // Never mutates: the pool owns this failure, not the account.
+        false,
+      ),
+  },
   {
     name: "deployment-infrastructure",
     match: (s) => s.deploymentInfraError,
@@ -976,14 +1142,42 @@ const ACCOUNT_ERROR_RULES: readonly AccountErrorRule[] = [
     name: "content-policy-block",
     match: (s) => s.policyRejection && s.buddy,
     decide: (s) => {
-      const policyBlockMs = resolveAccountPolicyBlockCooldownMs();
+      // A ban is a verdict on the credential, and cooldown cannot answer it:
+      // the account is refused, not throttled. Disabling is the fix, and
+      // disabling *immediately* is a feature — a client's retry budget is spent
+      // in seconds, so a banned account that merely cooled stayed in the plan
+      // and burned sibling accounts that were still healthy before the window
+      // expired.
+      //
+      // The two shapes are told apart by the body, not the code alone: `11140`
+      // names a restriction on the account, `11-128` refuses the request.
+      if (s.buddyBanned) {
+        return s.result(
+          "policy_blocked_banned",
+          "disabled",
+          0,
+          null,
+          s.reason("Account banned by upstream"),
+          // A ban is evidence about the credential even at provider scope, for
+          // the same reason a named revocation is: the message is the evidence.
+          // Without this a stream frame's missing credential detail would leave
+          // a banned account serving.
+          true,
+        );
+      }
+      // Not a ban: the upstream refused this request. The account is fine, so
+      // the only correct move is to stop retrying it verbatim. Cooling the
+      // account here was the bug that made one malformed request look like a
+      // dead credential.
       return s.result(
-        "policy_blocked",
+        "policy_blocked_invalid_request",
         "cooldown",
-        policyBlockMs,
-        new Date(Date.now() + policyBlockMs),
-        s.reason("Provider content-policy block"),
-        s.origin === "upstream",
+        0,
+        null,
+        s.reason("Upstream refused this request"),
+        // Never mutates: a malformed body fails identically on every account,
+        // so cooling would burn a healthy credential on the gateway's own fault.
+        false,
       );
     },
   },

package/dist/dashboard/assets/HealthEventsModal-BRDc_wx8.js

@@ -1 +0,0 @@
-import{o as e,r as t,t as n}from"./jsx-runtime-CXUMngI_.js";import{i as r}from"./terminal-0_IjPWTk.js";import{c as i,s as a}from"./numbers-BOevJzdz.js";import{n as o,r as s,t as c}from"./state-2c5V4zo7.js";import{t as l}from"./dialog-C07hr5_G.js";import{t as u}from"./rotate-ccw-CufcqITi.js";import{t as d}from"./trash-Cppn5c3R.js";import{C as f,T as p,v as m}from"./assert-DU6qrMA_.js";import{o as h,t as g}from"./query-keys-UETqfytp.js";import{t as _}from"./useMutation-EDFRkn1i.js";import{k as v,n as y,v as b,y as x,z as S}from"./entry-8U_8IAW2.js";import{t as C}from"./badge-CKsF123d.js";import{n as w}from"./inline-C9dCYaBZ.js";import{t as T}from"./layout-V6y8DAkl.js";async function E(e){return h(`/quota/overview`,e?{signal:e}:void 0)}function D(){return f({queryKey:g.quota.all,queryFn:e=>E(m(e)),staleTime:1/0,refetchInterval:!1,refetchIntervalInBackground:!1,refetchOnMount:!1,refetchOnWindowFocus:!1,placeholderData:e=>e})}function O(e){let t=p(),n=g.quota.account(e);return _({mutationFn:async()=>h(`/accounts/${encodeURIComponent(e)}/quota/refresh`,{method:`POST`}),onSuccess:async()=>{await Promise.all([t.invalidateQueries({queryKey:n,exact:!0}),t.invalidateQueries({queryKey:g.quota.all}),t.invalidateQueries({queryKey:g.providers.all})])}})}function k(){let e=p();return _({mutationFn:async({accountIds:e,onProgress:t})=>{let n=e.length,r=0,i=await h(`/quota/refresh`,{method:`POST`,body:JSON.stringify({accountIds:e})});return r=i.queued,t?.(r,n),i},onSuccess:async(t,n)=>{let r=n.accountIds,i=new Set(t.failures.map(e=>e.accountId)),a=r.filter(e=>!i.has(e));await Promise.all([...a.map(t=>e.invalidateQueries({queryKey:g.quota.account(t),exact:!0})),e.invalidateQueries({queryKey:g.quota.all})])}})}function A(){let e=p();return _({mutationFn:({id:e,active:t})=>h(`/accounts/${encodeURIComponent(e)}`,{method:`PATCH`,body:JSON.stringify({active:t})}),onSuccess:async()=>{await Promise.all([e.invalidateQueries({queryKey:g.quota.all}),e.invalidateQueries({queryKey:g.providers.all})])}})}function j(){let e=p();return _({mutationFn:async({ids:e,active:t})=>{e.length!==0&&await h(`/accounts/batch`,{method:`PATCH`,body:JSON.stringify({ids:e,active:t})})},onSuccess:async()=>{await Promise.all([e.invalidateQueries({queryKey:g.quota.all}),e.invalidateQueries({queryKey:g.providers.all})])}})}function M(){let e=p();return _({mutationFn:async e=>{await h(`/accounts/${encodeURIComponent(e)}`,{method:`DELETE`})},onSuccess:async(t,n)=>{e.removeQueries({queryKey:g.quota.account(n),exact:!0}),await Promise.all([e.invalidateQueries({queryKey:g.quota.all}),e.invalidateQueries({queryKey:g.providers.all})])}})}var N=new Set([`workbuddy`,`workbuddycn`,`cb`,`cbcn`]);function P(e){return N.has(e.trim().toLowerCase())}function F(e){let t=p();return _({mutationFn:async()=>{if(e.length===0)throw Error(`Select a provider first`);return h(`/quota/daily-growth`,{method:`POST`,body:JSON.stringify({providerId:e})})},onSuccess:async()=>{await Promise.all([t.invalidateQueries({queryKey:g.quota.all}),t.invalidateQueries({queryKey:g.providers.all})])}})}function I(e){let t=p();return _({mutationFn:async()=>{let t=await h(`/accounts/${encodeURIComponent(e)}/activity-report`,{method:`POST`}),n=t.message??`Activity reported`,r=await h(`/accounts/${encodeURIComponent(e)}/checkin`,{method:`POST`}),i=r.message??`Daily check-in done`;return{ok:t.ok,message:t.ok?`Report success · Check-in ${i}`:`Report failed: ${n} · Check-in ${i}`,checkin:r,report:t}},onSuccess:async()=>{await Promise.all([t.invalidateQueries({queryKey:g.quota.all}),t.invalidateQueries({queryKey:g.providers.all})])}})}var L=new Set([`codex`,`claude`,`anthropic`]);function R(e){return L.has(e.trim().toLowerCase())}function z(e,t){return f({queryKey:g.quota.resets(e),queryFn:({signal:t})=>h(`/accounts/${encodeURIComponent(e)}/resets`,t?{signal:t}:void 0),enabled:t,staleTime:3e4})}function B(e){let t=p();return _({mutationFn:async t=>h(`/accounts/${encodeURIComponent(e)}/reset`,{method:`POST`,body:JSON.stringify(t??{})}),onSettled:async()=>{await Promise.all([t.invalidateQueries({queryKey:g.quota.all}),t.invalidateQueries({queryKey:g.providers.all})])}})}var V=e(t(),1),H=n();function U(e){return e===`cooldown`}function W(e){return e===`active`?`ok`:e===`disabled`?`disabled`:`warn`}function G(e,t){return e?{title:`Enable static-token mode?`,message:`${t} will be used exactly as issued and will not be refreshed. Re-enable refresh before this token expires.`,confirmLabel:`Enable static mode`}:{title:`Re-enable credential refresh?`,message:`${t} will return to the OAuth refresh sweep. Use this only when a refresh token is available.`,confirmLabel:`Re-enable refresh`}}function K({summary:e,onClose:t}){let{providerId:n,accountId:f,title:p,status:m,errorCategory:h,errorMessage:g}=e,_=y(n,f),E=x(),D=_.data??[],O=v(),k=e.credentialKind===`oauth`||e.credentialMode===`jwt`||e.staticToken===!0,[A,j]=(0,V.useState)(e.staticToken===!0),[M,N]=(0,V.useState)(null),[P,F]=(0,V.useState)(!1);(0,V.useEffect)(()=>j(e.staticToken===!0),[e.staticToken]);let I=b();return(0,H.jsx)(l,{open:!0,onClose:t,title:`Health & Error Log — ${p}`,size:`lg`,children:(0,H.jsxs)(`div`,{className:`flex flex-col gap-7 min-w-0`,children:[(0,H.jsxs)(`div`,{className:`flex flex-wrap items-center justify-between gap-5 rounded-control border border-inner-border bg-surface-2 py-5 px-7`,children:[(0,H.jsxs)(`div`,{className:`min-w-0`,children:[(0,H.jsxs)(w,{gap:`8px`,className:`flex-wrap`,children:[(0,H.jsx)(`span`,{className:`text-body font-semibold`,children:`Account status:`}),(0,H.jsx)(C,{tone:W(m),dot:!0,children:m.toUpperCase()}),h?(0,H.jsx)(C,{tone:`warn`,title:g??void 0,children:h}):null]}),g?(0,H.jsx)(`p`,{className:`text-badge text-red mt-2 wrap-anywhere`,children:g}):null]}),U(m)?(0,H.jsx)(i,{size:`sm`,variant:`secondary`,icon:(0,H.jsx)(u,{size:12,className:E.isPending?`animate-spin`:``}),disabled:E.isPending,onClick:()=>E.mutate({providerId:n,accountId:f},{onSuccess:()=>void _.refetch()}),children:E.isPending?`Recovering...`:`Recover Now`}):null]}),k?(0,H.jsxs)(`div`,{className:`flex items-center justify-between gap-6 rounded-control border border-inner-border bg-surface-2 py-5 px-7`,children:[(0,H.jsxs)(`div`,{children:[(0,H.jsx)(`div`,{className:`text-caption font-semibold`,children:`Credential management`}),(0,H.jsx)(`div`,{className:`text-badge text-text-tertiary`,children:A?`Static mode: no refresh request will be sent.`:`OAuth refresh is enabled for this account.`})]}),(0,H.jsx)(i,{size:`sm`,variant:A?`primary`:`secondary`,disabled:O.isPending,onClick:()=>N(!A),children:A?`Static · no refresh`:`Enable static mode`})]}):null,(0,H.jsxs)(`div`,{className:`min-w-0`,children:[(0,H.jsxs)(`div`,{className:`flex items-center justify-between gap-5 mb-4`,children:[(0,H.jsxs)(`h4`,{className:`text-caption font-semibold text-text-secondary`,children:[`Transition Events & Audit (`,D.length,`)`]}),(0,H.jsx)(i,{size:`sm`,variant:`secondary`,icon:(0,H.jsx)(d,{size:12}),disabled:D.length===0||I.isPending,onClick:()=>F(!0),title:`Deletes this account's recorded transitions and audit entries.`,children:`Purge log`})]}),_.isPending?(0,H.jsx)(s,{compact:!0,label:`Loading health events...`}):_.isError?(0,H.jsx)(o,{compact:!0,message:`Failed to load health events`,onRetry:()=>void _.refetch()}):D.length===0?(0,H.jsx)(c,{compact:!0,title:`No health events recorded`,message:e.emptyMessage,icon:(0,H.jsx)(r,{size:20})}):(0,H.jsx)(`div`,{className:`rounded-inset border border-inner-border`,children:(0,H.jsx)(T,{maxHeight:480,scrollRegion:!0,headers:[`Timestamp`,`From → To`,`Model`,`Category`,`Reason / Details`],children:D.map(e=>(0,H.jsxs)(`tr`,{children:[(0,H.jsx)(`td`,{className:`text-badge text-text-tertiary whitespace-nowrap`,children:new Date(e.createdAt).toLocaleString()}),(0,H.jsxs)(`td`,{className:`whitespace-nowrap`,children:[e.fromStatus!==null&&e.fromStatus!==e.toStatus?(0,H.jsxs)(`span`,{className:`text-badge text-text-secondary`,children:[e.fromStatus,` →`,` `]}):(0,H.jsx)(`span`,{className:`text-badge text-text-tertiary`,children:`logged · `}),(0,H.jsx)(C,{tone:W(e.toStatus),children:e.toStatus})]}),(0,H.jsx)(`td`,{className:`font-mono tabular-nums text-badge text-text-secondary wrap-anywhere`,children:e.modelId??`—`}),(0,H.jsx)(`td`,{className:`font-mono tabular-nums text-badge text-text-secondary`,children:e.errorCategory??`—`}),(0,H.jsx)(`td`,{className:`text-badge text-text-primary min-w-110 max-w-260 wrap-anywhere`,children:e.reason??`—`})]},e.id))})})]}),(0,H.jsx)(S,{open:M!==null,onClose:()=>N(null),onConfirm:()=>{let e=M===!0;O.mutate({providerId:n,accountId:f,request:{staticToken:e}},{onSuccess:()=>{j(e),N(null),a.success(e?`Static mode enabled`:`Credential refresh re-enabled`,p),_.refetch()},onError:e=>a.error(`Credential update failed`,e instanceof Error?e.message:`Unable to update credential mode`)})},title:G(M===!0,p).title,message:G(M===!0,p).message,confirmLabel:G(M===!0,p).confirmLabel,cancelLabel:`Cancel`}),(0,H.jsx)(S,{open:P,onClose:()=>F(!1),onConfirm:()=>{I.mutate({providerId:n,accountId:f,title:p},{onSuccess:()=>F(!1)})},title:`Purge the health & error log for ${p}?`,message:`Deletes every transition and audit entry recorded for this account. Its status, credentials, and counters are untouched. This cannot be undone.`,confirmLabel:`Purge log`,cancelLabel:`Cancel`}),(0,H.jsx)(w,{justify:`flex-end`,gap:`0px`,children:(0,H.jsx)(i,{variant:`secondary`,size:`sm`,onClick:t,children:`Close`})})]})})}export{M as a,k as c,I as d,F as f,z as i,j as l,P as n,D as o,A as p,R as r,O as s,K as t,B as u};