myapikey 0.35.0 → 0.36.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "myapikey",
-  "version": "0.35.0",
+  "version": "0.36.1",
   "type": "module",
   "description": "Personal LLM API gateway & proxy — one address + one API key for all your models. Forwards OpenAI & Anthropic calls to your backends with failover and a circuit breaker. Pure passthrough, no format translation. Self-hosted (CLI + web UI).",
   "keywords": [

package/packages/core/src/cli/index.ts

@@ -285,7 +285,7 @@ model.command("remove <name>").description("remove a model entirely (both format
 
 model
   .command("debug <name> [action] [index]")
-  .description("debug capture: on|off toggles recording of the last 50 actual upstream requests/responses; show lists them (add an index to dump one in full)")
+  .description("debug capture: on|off toggles recording of the last 50 actual requests/responses (failed attempts are ALWAYS recorded in a global net, last 50); show lists everything (add an index to dump one in full)")
   .action(async (name: string, action = "show", indexRaw?: string) => {
     const path = `/admin/models/${encodeURIComponent(name)}/debug`;
     if (action === "on" || action === "off") {
@@ -293,32 +293,39 @@ model
       console.log(
         r.enabled
           ? `Debug capture ON for ${name} — the last 50 upstream attempts (request + response) are recorded in memory; turn off to clear.`
-          : `Debug capture OFF for ${name}; captured content cleared.`,
+          : `Debug capture OFF for ${name}; captured content cleared (auto-recorded failures stay until they age out of the net).`,
       );
       return;
     }
     if (action !== "show") throw new Error(`Unknown action '${action}' (use on | off | show).`);
-    const r = (await api(ctx(), "GET", path)) as { enabled: boolean; captures: any[] };
-    if (!r.enabled) console.log(`Debug capture is OFF for ${name}.`);
-    const cs = r.captures ?? [];
+    const r = (await api(ctx(), "GET", path)) as { enabled: boolean; captures: any[]; failures: any[] };
+    if (!r.enabled) console.log(`Debug capture is OFF for ${name} (failed calls are still auto-recorded).`);
+    const isFail = (c: any) => c.status >= 400 || c.status === 0;
+    // One newest-first timeline; a failure made while the switch was on sits
+    // in both server buffers — show it once, from the net, tagged AUTO.
+    const rows = [
+      ...(r.failures ?? []).map((c) => ({ c, auto: true })),
+      ...(r.captures ?? []).filter((c) => !isFail(c)).map((c) => ({ c, auto: false })),
+    ].sort((a, b) => b.c.ts - a.c.ts);
     if (indexRaw !== undefined) {
-      const row = cs[Number(indexRaw) - 1];
-      if (!row) throw new Error(`No capture #${indexRaw} (list is newest-first, ${cs.length} captured).`);
-      console.log(`#${Number(indexRaw)}  ${new Date(row.ts).toLocaleTimeString("en-GB", { hour12: false })}  ${row.provider} [${row.format}]  status=${row.status}  ${row.ms}ms${row.upstreamModel ? `  model=${row.upstreamModel}` : ""}${row.error ? `  error=${row.error}` : ""}${row.truncated ? "  (truncated)" : ""}`);
+      const row = rows[Number(indexRaw) - 1];
+      if (!row) throw new Error(`No capture #${indexRaw} (list is newest-first, ${rows.length} recorded).`);
+      const c = row.c;
+      console.log(`#${Number(indexRaw)}${row.auto ? " (auto)" : ""}  ${new Date(c.ts).toLocaleTimeString("en-GB", { hour12: false })}  ${c.provider} [${c.format}]  status=${c.status}  ${c.ms}ms${c.upstreamModel ? `  model=${c.upstreamModel}` : ""}${c.error ? `  error=${c.error}` : ""}${c.truncated ? "  (truncated)" : ""}`);
       console.log("\n--- request (forwarded verbatim) ---");
-      console.log(prettyJson(row.request));
-      if (row.response !== undefined) {
+      console.log(prettyJson(c.request));
+      if (c.response !== undefined) {
         console.log("\n--- response (upstream) ---");
-        console.log(prettyJson(row.response));
+        console.log(prettyJson(c.response));
       }
       return;
     }
-    if (!cs.length) return console.log("No captures yet — call the model, then `show` again (or pass an index to dump one).");
-    console.log(`${name}: ${cs.length} captured (newest first)`);
-    for (let i = 0; i < cs.length; i++) {
-      const c = cs[i];
+    if (!rows.length) return console.log("Nothing recorded yet — call the model, then `show` again (or pass an index to dump one).");
+    console.log(`${name}: ${rows.length} recorded (newest first; AUTO = failed attempt, always recorded)`);
+    for (let i = 0; i < rows.length; i++) {
+      const { c, auto } = rows[i];
       console.log(
-        `  #${i + 1}  ${new Date(c.ts).toLocaleTimeString("en-GB", { hour12: false })}  ${c.provider} [${c.format}]  status=${c.status}  ${c.ms}ms  ${fmtBytes(c.request)} req / ${fmtBytes(c.response ?? "")} resp${c.upstreamModel ? `  model=${c.upstreamModel}` : ""}${c.error ? `  ${c.error}` : ""}`,
+        `  #${i + 1}${auto ? " AUTO" : "     "}  ${new Date(c.ts).toLocaleTimeString("en-GB", { hour12: false })}  ${c.provider} [${c.format}]  status=${c.status}  ${c.ms}ms  ${fmtBytes(c.request)} req / ${fmtBytes(c.response ?? "")} resp${c.upstreamModel ? `  model=${c.upstreamModel}` : ""}${c.error ? `  ${c.error}` : ""}`,
       );
     }
     console.log("\nDump one: myapikey model debug <name> show <index>");

package/packages/core/src/server/admin.ts

@@ -705,12 +705,15 @@ export function adminApi(store: Store, auth: MiddlewareHandler, openai: Hono, an
 // ON: dispatch records every upstream attempt for this model (exact forwarded
 // request + response bodies) into an in-memory ring buffer, last 50. OFF: the
 // buffer is cleared immediately — captured conversations shouldn't outlive the
-// debugging session. The switch itself is config state (persists in data.json);
-// the captured bodies do not (in-memory only, restart clears them too).
+// debugging session. Independently, FAILED attempts always land in a global
+// in-memory net (last 50 across all models) so an error can be inspected even
+// when the switch was never on; `failures` here is that net filtered to this
+// model. The switch is config state (persists in data.json); the captured
+// bodies do not (in-memory only, restart clears them too).
 app.get("/models/:name/debug", (c) => {
   const name = c.req.param("name");
   if (!store.get().models[name]) return c.json({ error: { message: "model not found" } }, 404);
-  return c.json({ enabled: store.isDebug(name), captures: store.getCaptures(name) });
+  return c.json({ enabled: store.isDebug(name), captures: store.getCaptures(name), failures: store.getFailCaptures(name) });
 });
 
 app.put("/models/:name/debug", async (c) => {
@@ -732,6 +735,16 @@ app.put("/models/:name/debug", async (c) => {
   return c.json({ ok: true, enabled });
 });
 
+// Manual scrub of this model's rows in the always-on failure net (the switch
+// only governs the per-model buffer; failures age out of the global ring on
+// their own — this drops them now).
+app.delete("/models/:name/debug/fails", (c) => {
+  const name = c.req.param("name");
+  if (!store.get().models[name]) return c.json({ error: { message: "model not found" } }, 404);
+  store.clearFailCaptures(name);
+  return c.json({ ok: true });
+});
+
   // Set (or clear) the upstream-model mapping for ONE chain slot (addressed by
   // `index`). An empty `model` clears it (back to identity — send the public
   // name). Index-addressed because a provider may occupy several slots, each

package/packages/core/src/server/proxy.ts

@@ -6,8 +6,43 @@ import { UsageCollector } from "./tokens";
 
 /** HTTP statuses that should trigger failover to the next provider. 401/403
  *  included: a banned/invalid credential (e.g. "User has been banned") is dead
- *  for THIS source only - the same request may be fine on the next one. */
-const RETRYABLE = new Set([401, 403, 408, 425, 429, 500, 502, 503, 504]);
+ *  for THIS source only - the same request may be fine on the next one. 402 is
+ *  always account-level (exhausted balance). */
+const RETRYABLE = new Set([401, 402, 403, 408, 425, 429, 500, 502, 503, 504]);
+
+/** Error-body markers that turn a NON-retryable 4xx into an account-level
+ *  failure worth failing over: dead credential / expired subscription / no
+ *  balance, which some vendors smuggle under a 400 instead of 401/403
+ *  (Volcengine Ark's expired coding-plan subscription is the case that bit
+ *  us). Deliberately narrow — a genuine request-shape 400 (bad params, context
+ *  overflow, "max_tokens too large") still returns to the client as-is, since
+ *  every other source would 400 identically. Matched case-insensitively. */
+const ACCOUNT_HINTS = [
+  "subscription",
+  "unauthorized",
+  "invalid_api_key",
+  "invalid api key",
+  "incorrect api key",
+  "quota",
+  "insufficient",
+  "balance",
+  "credit",
+  "expired",
+  "令牌",
+  "订阅",
+  "额度",
+  "余额",
+  "欠费",
+  "未开通",
+];
+
+/** True when `status`/`bodyText` describe an account-level failure on an
+ *  otherwise non-retryable 4xx (see ACCOUNT_HINTS). */
+function isAccountLevelError(status: number, bodyText: string): boolean {
+  if (RETRYABLE.has(status) || status < 400 || status >= 500) return false;
+  const t = bodyText.toLowerCase();
+  return ACCOUNT_HINTS.some((h) => t.includes(h));
+}
 
 /** Even pacing (per-model `paceRpm`) message constants. The queue itself lives
  *  in Store.paceClaim - 60s wait horizon, one release every 60/rpm seconds. */
@@ -259,11 +294,6 @@ function downHeaders(upstream: Response, servedBy?: string): Headers {
   return headers;
 }
 
-function passThrough(upstream: Response, servedBy?: string): Response {
-  // Stream the upstream body straight through (handles SSE + normal JSON).
-  return new Response(upstream.body, { status: upstream.status, headers: downHeaders(upstream, servedBy) });
-}
-
 /** Pull a short human-readable message out of an upstream error body. */
 export function shortError(text: string): string {
   try {
@@ -613,17 +643,19 @@ export function proxyApi(
         // Count this attempt toward the source's RPM window — but not for a pinned
         // probe, which (like circuit state) takes no routing side-effects.
         if (pinIndex == null) store.recordDispatch(provider.id);
-        // Debug capture (ModelEntry.debugCapture): when the model's switch is
-        // on, record EVERY upstream attempt — the exact forwarded body (this
-        // slot's model rewrite + thinking injection are already applied) and
-        // the response as it flowed. UI probes are excluded (not real
-        // conversations). One entry per attempt: a failover chain writes
-        // several, each showing what THAT source actually received.
-        const debugNow = !isProbe && store.isDebug(model);
+        // Debug capture: EVERY upstream attempt is offered to the store —
+        // failed attempts always land in the global failure net (the safety
+        // net for after-the-fact debugging), everything lands in the model's
+        // own buffer while its switch is on. The row carries the exact
+        // forwarded body (this slot's model rewrite + thinking injection are
+        // already applied) and the response as it flowed. UI probes are
+        // excluded (not real conversations). One entry per attempt: a
+        // failover chain writes several, each showing what THAT source
+        // actually received.
         const attemptStart = Date.now();
         const reqText = JSON.stringify(body);
         const capture = (status: number, response: string | undefined, truncated: boolean, error?: string) => {
-          if (!debugNow) return;
+          if (isProbe) return;
           const row: DebugCapture = {
             ts: Date.now(),
             model,
@@ -679,48 +711,49 @@ export function proxyApi(
           const ttfb = Date.now() - start;
           // Debug capture tee: accumulate the decoded chunks into a bounded
           // string (the proxy keeps forwarding bytes verbatim regardless).
-          const capAcc = debugNow ? { text: "", truncated: false } : undefined;
+          const capAcc = { text: "", truncated: false };
           const out = observedBody(upstream, {
             stream,
             key,
             requestMessages: body.messages,
-            ...(capAcc
-              ? {
-                  onText: (txt: string) => {
-                    const room = CAPTURE_BODY_MAX - capAcc.text.length;
-                    if (room <= 0) {
-                      capAcc.truncated = true;
-                      return;
-                    }
-                    if (txt.length > room) capAcc.truncated = true;
-                    capAcc.text += txt.slice(0, room);
-                  },
-                }
-              : {}),
+            onText: (txt: string) => {
+              const room = CAPTURE_BODY_MAX - capAcc.text.length;
+              if (room <= 0) {
+                capAcc.truncated = true;
+                return;
+              }
+              if (txt.length > room) capAcc.truncated = true;
+              capAcc.text += txt.slice(0, room);
+            },
             onSettle: (info) => {
               if (info.ok) {
                 store.recordCircuitSuccess(provider.id);
                 store.pushLog({ ts: Date.now(), model, upstreamModel, provider: provider.name, providerId: provider.id, format: wire, status: 200, ms: ttfb, stream, thinking: think, usage: info.usage });
-                capture(200, capAcc?.text, capAcc?.truncated ?? false);
+                capture(200, capAcc.text, capAcc.truncated);
               } else {
                 // A pinned per-source probe takes no circuit side-effects (a manual
                 // test must not trip the breaker) — mirrors the retryable branch.
                 if (pinIndex == null) store.recordCircuitFailure(provider.id, info.status, info.error || "stream failed");
                 if (!isProbe) rt.warn(`proxy stream failed: provider '${provider.name}' status=${info.status} (${info.error || "stream failed"})`);
                 store.pushLog({ ts: Date.now(), model, upstreamModel, provider: provider.name, providerId: provider.id, format: wire, status: info.status, ms: ttfb, stream, thinking: think, error: info.error });
-                capture(info.status, capAcc?.text, capAcc?.truncated ?? false, info.error);
+                capture(info.status, capAcc.text, capAcc.truncated, info.error);
               }
             },
           });
           return new Response(out, { status: upstream.status, headers: downHeaders(upstream, isProbe ? provider.name : undefined) });
         }
-        if (RETRYABLE.has(upstream.status)) {
-          lastStatus = upstream.status;
-          // Drain so the connection can be reused, then move on; capture the
-          // reason for the log (this branch never streams back to the client).
-          const txt = await upstream.text().catch(() => "");
-          lastErr = shortError(txt) || `HTTP ${upstream.status}`;
-          capture(upstream.status, txt, false, lastErr);
+        // Error from the upstream. Read the body ONCE (drains the connection
+        // for reuse), then either fail over or pass the error back verbatim —
+        // both branches share this text (log row + debug capture).
+        const txt = await upstream.text().catch(() => "");
+        lastStatus = upstream.status;
+        lastErr = shortError(txt) || `HTTP ${upstream.status}`;
+        capture(upstream.status, txt, false, lastErr);
+        // Fail over on a retryable status OR an account-level failure smuggled
+        // under a non-retryable 4xx (expired subscription / dead key / no
+        // balance returned as 400) — the source is dead for THIS request only,
+        // exactly like a 401/403.
+        if (RETRYABLE.has(upstream.status) || isAccountLevelError(upstream.status, txt)) {
           if (pinIndex != null) break; // per-source probe: fail fast, no circuit impact.
           sayFailover(provider, `HTTP ${lastStatus} (${lastErr})`);
           // A 429/overloaded upstream usually carries Retry-After; honoring it
@@ -737,12 +770,9 @@ export function proxyApi(
           }
           continue;
         }
-        // Non-retryable client error: return it to the caller as-is. Read the
-        // error text off a CLONE so the original body still streams back.
-        const errText = await upstream.clone().text().catch(() => "");
-        store.pushLog({ ts: Date.now(), model, upstreamModel, provider: provider.name, providerId: provider.id, format: wire, status: upstream.status, ms: Date.now() - start, stream, thinking: think, error: shortError(errText) || `HTTP ${upstream.status}` });
-        capture(upstream.status, errText, false, shortError(errText) || `HTTP ${upstream.status}`);
-        return passThrough(upstream, isProbe ? provider.name : undefined);
+        // Non-retryable client error: return it to the caller as-is.
+        store.pushLog({ ts: Date.now(), model, upstreamModel, provider: provider.name, providerId: provider.id, format: wire, status: upstream.status, ms: Date.now() - start, stream, thinking: think, error: lastErr });
+        return new Response(txt, { status: upstream.status, headers: downHeaders(upstream, isProbe ? provider.name : undefined) });
       }
 
       const last = order[order.length - 1];

package/packages/core/src/server/store.ts

@@ -45,12 +45,18 @@ const RPM_WINDOW_MS = 60_000;
  *  time and the queue depth (at N rpm / 60s wait, at most ~N requests queue). */
 const PACE_MAX_WAIT_MS = 60_000;
 
-/** Debug capture (per-model, in-memory ring buffer): how many upstream attempts
- *  to keep, and how many CHARS of each request/response body. 50 × 2 × 256KB ≈
- *  25MB worst case per debugged model — acceptable for a home-server debug aid.
- *  Bodies (conversations) are sensitive: the buffer lives only in memory, is
- *  dropped when the model's debug switch turns off, and never hits disk. */
+/** Debug capture (in-memory ring buffers, never persisted). Two tiers:
+ *  - FAIL_NET (global): the last CAPTURE_FAIL_MAX FAILED upstream attempts
+ *    across ALL models, recorded ALWAYS (that's the safety net — errors are
+ *    debugged after the fact, when nobody pre-armed a switch). 50 × 2 × 256KB
+ *    ≈ 25MB worst case, typical far less (failed responses are tiny).
+ *  - per-model switch buffer: last CAPTURE_MAX attempts of a model whose
+ *    ModelEntry.debugCapture is on — success AND failure.
+ *  Bodies (conversations) are sensitive: everything lives only in memory,
+ *  never touches disk, ages out of the rings, and the switch-off / clear
+ *  endpoints drop it on demand. */
 export const CAPTURE_MAX = 50;
+export const CAPTURE_FAIL_MAX = 50;
 export const CAPTURE_BODY_MAX = 256 * 1024;
 
 /** Per-provider circuit state (in-memory, never persisted). */
@@ -200,10 +206,14 @@ export class Store {
    *  release slot. In-memory, NOT persisted (resets on restart). */
   private pace = new Map<string, number>();
   /** Per-model debug-capture ring buffers: model name -> last CAPTURE_MAX
-   *  upstream attempts (oldest first). In-memory, NOT persisted — bodies are
-   *  conversation content, so they live only while the model's debug switch
-   *  is on and evaporate on restart or toggle-off. */
+   *  upstream attempts (oldest first) while the model's switch is on.
+   *  In-memory, NOT persisted — bodies are conversation content, so they live
+   *  only while the switch is on and evaporate on restart or toggle-off. */
   private captures = new Map<string, DebugCapture[]>();
+  /** The always-on failure net: last CAPTURE_FAIL_MAX FAILED upstream attempts
+   *  across all models (oldest first), regardless of any switch. In-memory,
+   *  NOT persisted; entries age out as new failures push them off. */
+  private failCaptures: DebugCapture[] = [];
 
   constructor(dataDir: string, opts: { logger?: Logger } = {}) {
     this.dataDir = dataDir;
@@ -669,12 +679,27 @@ export class Store {
     return !!this.data.models[model]?.debugCapture;
   }
 
-  /** Record one upstream attempt for a debugged model. Ignores silently when
-   *  the switch is off (defense against a stale caller); bodies are capped at
-   *  CAPTURE_BODY_MAX chars each (flagging `truncated`), and the buffer keeps
-   *  only the most recent CAPTURE_MAX attempts per model. */
+  /** Record one upstream attempt. Routing: FAILED attempts (4xx/5xx/network)
+   *  always land in the global failure net (the safety net for after-the-fact
+   *  debugging); EVERYTHING lands in the model's own buffer while its switch
+   *  is on. When the switch is off, successes are dropped. Bodies are capped
+   *  at CAPTURE_BODY_MAX chars each (flagging `truncated`). */
   pushCapture(model: string, entry: DebugCapture): void {
+    const e = this.capped({ ...entry, model }); // row's model = the routing key
+    const failed = e.status >= 400 || e.status === 0;
+    if (failed) {
+      this.failCaptures.push(e);
+      if (this.failCaptures.length > CAPTURE_FAIL_MAX) this.failCaptures.splice(0, this.failCaptures.length - CAPTURE_FAIL_MAX);
+    }
     if (!this.isDebug(model)) return;
+    let arr = this.captures.get(model);
+    if (!arr) this.captures.set(model, (arr = []));
+    arr.push(e);
+    if (arr.length > CAPTURE_MAX) arr.splice(0, arr.length - CAPTURE_MAX);
+  }
+
+  /** Copy of an entry with both bodies clamped to CAPTURE_BODY_MAX. */
+  private capped(entry: DebugCapture): DebugCapture {
     const e = { ...entry };
     if (e.request.length > CAPTURE_BODY_MAX) {
       e.request = e.request.slice(0, CAPTURE_BODY_MAX);
@@ -684,23 +709,31 @@ export class Store {
       e.response = e.response.slice(0, CAPTURE_BODY_MAX);
       e.truncated = true;
     }
-    let arr = this.captures.get(model);
-    if (!arr) this.captures.set(model, (arr = []));
-    arr.push(e);
-    if (arr.length > CAPTURE_MAX) arr.splice(0, arr.length - CAPTURE_MAX);
+    return e;
   }
 
-  /** The model's captured attempts, newest first. */
+  /** The model's switch-captured attempts, newest first. */
   getCaptures(model: string): DebugCapture[] {
     const arr = this.captures.get(model);
     return arr ? [...arr].reverse() : [];
   }
 
-  /** Drop a model's capture buffer (toggle-off, model delete/rename). */
+  /** The model's always-recorded failed attempts (from the global net),
+   *  newest first. */
+  getFailCaptures(model: string): DebugCapture[] {
+    return this.failCaptures.filter((c) => c.model === model).reverse();
+  }
+
+  /** Drop a model's switch-capture buffer (toggle-off, model delete/rename). */
   clearCaptures(model: string): void {
     this.captures.delete(model);
   }
 
+  /** Scrub a model's entries from the global failure net (manual clear). */
+  clearFailCaptures(model: string): void {
+    this.failCaptures = this.failCaptures.filter((c) => c.model !== model);
+  }
+
   /** Snapshot of every configured provider's circuit state for GET /admin/circuit.
    *  Healthy providers appear as state "open"; a provider deleted while cooling
    *  simply drops out (we iterate the live config, not the map). */

package/packages/core/src/shared/types.ts

@@ -104,7 +104,9 @@ export interface ModelEntry {
    *  records every UPSTREAM ATTEMPT for this model — the exact forwarded request
    *  body and the response body — into an in-memory ring buffer (last 50, see
    *  Store.pushCapture). Turning it off clears the buffer; restarts clear it too.
-   *  Nothing here ever touches logs.jsonl. */
+   *  Independently of this switch, FAILED attempts always land in a global
+   *  in-memory net (last 50 across all models) so an error can be inspected
+   *  after the fact. Nothing here ever touches logs.jsonl. */
   debugCapture?: boolean;
 }