myapikey 0.1.0 → 0.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "myapikey",
-  "version": "0.1.0",
+  "version": "0.3.0",
   "type": "module",
   "description": "Personal LLM API gateway & proxy — one address + one API key for all your models. Forwards OpenAI & Anthropic calls to your backends with failover and a circuit breaker. Pure passthrough, no format translation. Self-hosted (CLI + web UI).",
   "keywords": [

package/packages/core/src/cli/index.ts

@@ -110,15 +110,19 @@ provider
   .option("--base-url-anthropic <url>", "Anthropic base URL excl. /v1, e.g. https://api.anthropic.com", "")
   .option("--key <key>", "api key for the backend", "")
   .option("--formats <list>", "comma list: openai,anthropic", "openai")
-  .action(async (name: string, opts: { baseUrlOpenai: string; baseUrlAnthropic: string; key: string; formats: string }) => {
+  .option("--rpm <n>", "optional request-per-minute cap (pace a free/limited key)", "")
+  .action(async (name: string, opts: { baseUrlOpenai: string; baseUrlAnthropic: string; key: string; formats: string; rpm: string }) => {
     const formats = opts.formats.split(",").map((s) => s.trim()).filter(Boolean) as ("openai" | "anthropic")[];
-    const r = await api(ctx(), "POST", "/admin/providers", {
+    const rpm = Number(opts.rpm);
+    const body: Record<string, unknown> = {
       name,
       baseUrlOpenai: opts.baseUrlOpenai,
       baseUrlAnthropic: opts.baseUrlAnthropic,
       apiKey: opts.key,
       formats,
-    });
+    };
+    if (Number.isFinite(rpm) && rpm > 0) body.rpm = Math.floor(rpm);
+    const r = await api(ctx(), "POST", "/admin/providers", body);
     console.log(`Added provider ${(r as any).provider.name} (${(r as any).provider.id})`);
   });
 

package/packages/core/src/server/admin.ts

@@ -31,6 +31,13 @@ function formatsKey(f: Format[]): string {
   return [...f].sort().join(",");
 }
 
+/** Coerce an RPM cap to a positive integer, or undefined (0/blank/invalid =
+ *  unlimited). Accepts a number or a numeric string (form fields send strings). */
+function coerceRpm(v: unknown): number | undefined {
+  const n = typeof v === "string" ? Number(v.trim()) : v;
+  return typeof n === "number" && Number.isFinite(n) && n > 0 ? Math.floor(n) : undefined;
+}
+
 /** Whether a provider is a valid source for a routing slot: openai/anthropic
  *  require that wire format; responses requires supportsResponses. */
 function providerSpeaks(p: Provider, key: RouteKey): boolean {
@@ -58,6 +65,7 @@ function toPublic(p: Provider) {
     formats: p.formats,
     supportsResponses: p.supportsResponses ?? false,
     apiKey: mask(p.apiKey),
+    rpm: p.rpm ?? 0,
     discoveredModels: p.discoveredModels ?? [],
     discoveredAt: p.discoveredAt ?? null,
     createdAt: p.createdAt,
@@ -176,7 +184,7 @@ export function adminApi(store: Store, auth: MiddlewareHandler, v1: Hono): Hono
   app.get("/providers", (c) => c.json({ providers: store.get().providers.map(toPublic) }));
 
   app.post("/providers", async (c) => {
-    const body = await readJson<{ name?: string; baseUrlOpenai?: string; baseUrlAnthropic?: string; apiKey?: string; formats?: Format[]; supportsResponses?: boolean }>(c.req.raw);
+    const body = await readJson<{ name?: string; baseUrlOpenai?: string; baseUrlAnthropic?: string; apiKey?: string; formats?: Format[]; supportsResponses?: boolean; rpm?: number }>(c.req.raw);
     const formats = body?.formats ?? [];
     const needOpenai = formats.includes("openai");
     const needAnthropic = formats.includes("anthropic");
@@ -189,6 +197,7 @@ export function adminApi(store: Store, auth: MiddlewareHandler, v1: Hono): Hono
     const id = newProviderId();
     const baseUrlOpenai = trimBase(body.baseUrlOpenai ?? "");
     const baseUrlAnthropic = trimBase(body.baseUrlAnthropic ?? "");
+    const rpm = coerceRpm(body!.rpm);
     await store.update((d) => {
       d.providers.push({
         id,
@@ -198,6 +207,7 @@ export function adminApi(store: Store, auth: MiddlewareHandler, v1: Hono): Hono
         apiKey: body.apiKey!,
         formats,
         supportsResponses: body.supportsResponses === true,
+        ...(rpm ? { rpm } : {}),
         createdAt: Date.now(),
       });
     });
@@ -209,7 +219,7 @@ export function adminApi(store: Store, auth: MiddlewareHandler, v1: Hono): Hono
 
   app.put("/providers/:id", async (c) => {
     const id = c.req.param("id");
-    const body = await readJson<{ name?: string; baseUrlOpenai?: string; baseUrlAnthropic?: string; apiKey?: string; formats?: Format[]; supportsResponses?: boolean }>(c.req.raw);
+    const body = await readJson<{ name?: string; baseUrlOpenai?: string; baseUrlAnthropic?: string; apiKey?: string; formats?: Format[]; supportsResponses?: boolean; rpm?: number }>(c.req.raw);
     const formats = body?.formats ?? [];
     const needOpenai = formats.includes("openai");
     const needAnthropic = formats.includes("anthropic");
@@ -239,6 +249,12 @@ export function adminApi(store: Store, auth: MiddlewareHandler, v1: Hono): Hono
       p.apiKey = newKey;
       p.formats = formats;
       if (body.supportsResponses !== undefined) p.supportsResponses = body.supportsResponses;
+      // rpm: present in the body → set (0/invalid clears to unlimited); absent → keep.
+      if (body!.rpm !== undefined) {
+        const rpm = coerceRpm(body!.rpm);
+        if (rpm) p.rpm = rpm;
+        else delete p.rpm;
+      }
     });
     const found = store.get().providers.find((x) => x.id === id);
     if (!found) return c.json({ error: { message: "provider not found" } }, 404);
@@ -484,6 +500,56 @@ export function adminApi(store: Store, auth: MiddlewareHandler, v1: Hono): Hono
     return c.json({ result: { ok: false, status: res.status, provider, format, error: shortError(txt) || `HTTP ${res.status}` } });
   });
 
+  // Probe a SINGLE source for a model+format: the same end-to-end loopback as the
+  // whole-model /test above, but dispatch is pinned to this one provider (via the
+  // x-myapikey-probe-provider header), so there is no failover and no
+  // circuit-breaker impact — a manual "is THIS source up?" check that reports the
+  // provider's real upstream status. Validation misses (source not on this route,
+  // wrong wire format) come back as a ProbeResult body, not an HTTP error, so the
+  // caller reads `result` uniformly.
+  app.post("/models/:name/providers/:providerId/test", async (c) => {
+    const name = c.req.param("name");
+    const pid = c.req.param("providerId");
+    const cfg = store.get();
+    const entry = cfg.models[name];
+    if (!entry) return c.json({ error: { message: "model not found" } }, 404);
+    let format = c.req.query("format") as RouteKey | undefined;
+    if (!format) {
+      // No slot requested: pick the first one the model is enabled on.
+      for (const k of ["openai", "anthropic", "responses"] as RouteKey[]) if (entry[k]?.enabled) { format = k; break; }
+    }
+    if (!format || !entry[format]?.enabled) {
+      return c.json({ result: { ok: false, status: 0, format: format ?? "openai", error: "model not enabled on that routing slot" } });
+    }
+    if (!entry[format].providers.includes(pid)) {
+      return c.json({ result: { ok: false, status: 0, format, error: "source is not on this route" } });
+    }
+    const provider = cfg.providers.find((p) => p.id === pid);
+    if (!provider || !providerSpeaks(provider, format)) {
+      return c.json({ result: { ok: false, status: 0, format, error: "source does not speak this format" } });
+    }
+    const path = format === "anthropic" ? "/messages" : format === "responses" ? "/responses" : "/chat/completions";
+    // /responses is the OpenAI Responses API — it takes `input`, not `messages`.
+    const body =
+      format === "responses"
+        ? { model: name, input: "ping", stream: false }
+        : { model: name, messages: [{ role: "user", content: "ping" }], max_tokens: 1, stream: false };
+    let res: Response;
+    try {
+      res = await v1.request(path, {
+        method: "POST",
+        headers: { "content-type": "application/json", authorization: `Bearer ${cfg.apiKey}`, "x-myapikey-probe": "1", "x-myapikey-probe-provider": pid },
+        body: JSON.stringify(body),
+      });
+    } catch (e) {
+      return c.json({ result: { ok: false, status: 0, format, error: `gateway loopback failed: ${(e as Error).message}` } });
+    }
+    const answeredBy = res.headers.get("x-myapikey-provider") ?? undefined;
+    if (res.ok) return c.json({ result: { ok: true, status: res.status, provider: answeredBy, format } });
+    const txt = await res.text().catch(() => "");
+    return c.json({ result: { ok: false, status: res.status, provider: answeredBy, format, error: shortError(txt) || `HTTP ${res.status}` } });
+  });
+
   app.delete("/models/:name", async (c) => {
     const name = c.req.param("name");
     await store.update((d) => {

package/packages/core/src/server/proxy.ts

@@ -9,6 +9,26 @@ const RETRYABLE = new Set([408, 425, 429, 500, 502, 503, 504]);
 /** Headers copied from upstream back to the client. */
 const COPY_DOWN = ["content-type", "cache-control", "x-request-id", "openai-organization", "anthropic-ratelimit-requests-reset"];
 
+/** Parse a `Retry-After` response header (RFC 9110) into a millisecond delay.
+ *  Two legal forms: delta-seconds (`"30"`) or an HTTP-date
+ *  (`"Wed, 21 Oct 2026 07:28:00 GMT"`). Returns the raw ms — store clamps it to
+ *  [CB_MIN, CB_CAP]; returns undefined for absent/empty/invalid/future-negative
+ *  so the caller falls back to the escalating circuit backoff. OpenAI,
+ *  Anthropic, OpenRouter, NIM and the OpenAI-compatible backends all emit this
+ *  on a 429/overloaded, so honoring it gives an exact cooldown where the per-
+ *  vendor `*-reset` headers would each need bespoke parsing. */
+function parseRetryAfter(v: string | null | undefined): number | undefined {
+  if (!v) return undefined;
+  const s = Number(v);
+  if (Number.isFinite(s) && s > 0) return s * 1000;
+  const t = Date.parse(v);
+  if (Number.isFinite(t)) {
+    const ms = t - Date.now();
+    return ms > 0 ? ms : undefined;
+  }
+  return undefined;
+}
+
 /** Resolve the ordered, compatible provider list for a model on a routing slot. */
 function candidates(store: Store, model: string, key: RouteKey): Provider[] {
   const d = store.get();
@@ -68,18 +88,23 @@ function upstreamTarget(p: Provider, key: RouteKey): { url: string; wire: Format
   return { url: `${trimBase(p.baseUrlOpenai)}/${path}`, wire: "openai" };
 }
 
-function passThrough(upstream: Response, servedBy?: string): Response {
+/** Copy through the headers we reflect to the client (content-type, rate-limit
+ *  hints, request id, …) and optionally tag the in-process probe with which
+ *  source answered. The probe tag never reaches a real agent client (set only
+ *  on isProbe calls — see dispatch). */
+function downHeaders(upstream: Response, servedBy?: string): Headers {
   const headers = new Headers();
   for (const h of COPY_DOWN) {
     const v = upstream.headers.get(h);
     if (v) headers.set(h, v);
   }
-  // Internal hook for the model-page "test": report which provider answered.
-  // Only set on in-process probe calls (see isProbe in dispatch), so it never
-  // appears on responses to real agent clients.
   if (servedBy) headers.set("x-myapikey-provider", servedBy);
+  return headers;
+}
+
+function passThrough(upstream: Response, servedBy?: string): Response {
   // Stream the upstream body straight through (handles SSE + normal JSON).
-  return new Response(upstream.body, { status: upstream.status, headers });
+  return new Response(upstream.body, { status: upstream.status, headers: downHeaders(upstream, servedBy) });
 }
 
 /** Pull a short human-readable message out of an upstream error body. */
@@ -92,6 +117,127 @@ export function shortError(text: string): string {
   }
 }
 
+/** Outcome of observing an upstream body to completion. A 200 at the headers is
+ *  not proof the call succeeded — some backends 200 then truncate the stream or
+ *  emit nothing for request shapes they mishandle. `ok` means the stream truly
+ *  ended cleanly (terminal marker seen for streaming; clean close otherwise). */
+interface SettleInfo {
+  ok: boolean;
+  status: number;
+  error?: string;
+}
+
+/** Wrap an upstream body so every byte is forwarded to the client VERBATIM while
+ *  we watch — out of band — for whether the stream completed cleanly. The 200
+ *  status is already committed before the body flows, so on a bad end we can't
+ *  change THAT; instead we (a) [anthropic] inject a synthetic SSE `error` event
+ *  so the client learns the stream died rather than seeing a silent EOF, and
+ *  (b) settle {ok:false} so dispatch logs a 502 and trips the circuit (the NEXT
+ *  call then fails over — this call can't be salvaged once streaming started).
+ *
+ *  Detection keys on the stream's terminal marker (anthropic message_stop /
+ *  openai [DONE] / responses response.completed), buffered across chunk
+ *  boundaries — NOT on content, which would false-positive on legitimate
+ *  tool-use responses that carry only input_json_delta. A client cancel settles
+ *  nothing (the client walked away — not a provider failure, don't log/cool). */
+/** Substrings whose presence proves a streaming response reached a REAL
+ *  terminal event — so an absent marker at stream-end means truncation. Keyed by
+ *  routing slot: anthropic ends on message_stop; /chat/completions on [DONE];
+ *  /responses on any of its terminal events (completed/failed/incomplete/
+ *  cancelled — a clean upstream FAILURE is not a truncation, just a failed call,
+ *  so we don't cool the source for it). Empty for a non-streaming body, where
+ *  only a reader error counts. */
+function terminalMarkers(key: RouteKey, stream: boolean): string[] {
+  if (!stream) return [];
+  if (key === "anthropic") return ["message_stop"];
+  if (key === "responses") return ["response.completed", "response.failed", "response.incomplete", "response.cancelled"];
+  return ["[DONE]"]; // openai /chat/completions
+}
+
+/** Best-effort synthetic terminal error frame, so a client learns a stream died
+ *  instead of seeing a silent EOF. Each wire's own convention:
+ *  - anthropic + /responses use typed `event: error` (spec'd);
+ *  - /chat/completions is a data-only SSE stream with NO spec'd mid-stream error
+ *    event, so we emit the de-facto `data: {"error":…}` shape most compatible
+ *    backends/SDKs raise on.
+ *  Never emits `[DONE]` (that signals success). */
+function errorFrame(key: RouteKey, reason: string): string {
+  const msg = reason.slice(0, 200);
+  if (key === "anthropic") {
+    return `event: error\ndata: ${JSON.stringify({ type: "error", error: { type: "api_error", message: msg } })}\n\n`;
+  }
+  if (key === "responses") {
+    // Plain transport-error `event: error` (response.failed would need a full
+    // Response object we don't have). Best-effort — /responses is opt-in.
+    return `event: error\ndata: ${JSON.stringify({ type: "error", message: msg })}\n\n`;
+  }
+  return `data: ${JSON.stringify({ error: { message: msg, type: "server_error" } })}\n\n`;
+}
+
+function observedBody(
+  upstream: Response,
+  opts: { stream: boolean; key: RouteKey; onSettle: (info: SettleInfo) => void },
+): ReadableStream<Uint8Array> {
+  const reader = upstream.body?.getReader();
+  const enc = new TextEncoder();
+  const dec = new TextDecoder();
+  const markers = terminalMarkers(opts.key, opts.stream);
+  let tail = ""; // rolling window so a marker split across chunks is still caught
+  let terminal = false;
+  let settled = false;
+
+  const settle = (info: SettleInfo) => {
+    if (settled) return;
+    settled = true;
+    opts.onSettle(info);
+  };
+  const injectError = (controller: ReadableStreamDefaultController<Uint8Array>, reason: string) => {
+    controller.enqueue(enc.encode(errorFrame(opts.key, reason)));
+  };
+
+  return new ReadableStream<Uint8Array>({
+    async pull(controller) {
+      if (!reader) {
+        settle({ ok: true, status: 200 });
+        controller.close();
+        return;
+      }
+      try {
+        const { done, value } = await reader.read();
+        if (done) {
+          if (opts.stream && !terminal) {
+            const reason = "upstream stream truncated (no terminal marker)";
+            injectError(controller, reason);
+            settle({ ok: false, status: 502, error: reason });
+          } else {
+            settle({ ok: true, status: 200 });
+          }
+          controller.close();
+          return;
+        }
+        if (!terminal && markers.length) {
+          const txt = dec.decode(value, { stream: true });
+          const win = tail + txt;
+          if (markers.some((m) => win.includes(m))) terminal = true;
+          tail = win.slice(-128);
+        }
+        controller.enqueue(value);
+      } catch (e) {
+        const reason = `upstream stream error: ${e instanceof Error ? e.message : String(e)}`;
+        injectError(controller, reason);
+        settle({ ok: false, status: 502, error: reason });
+        controller.close();
+      }
+    },
+    cancel() {
+      // Client abort (Esc / disconnect) — not a provider failure. Suppress the
+      // settle so we neither log nor cool down a source the client simply left.
+      settled = true;
+      reader?.cancel().catch(() => {});
+    },
+  });
+}
+
 export function proxyApi(store: Store, auth: MiddlewareHandler): Hono {
   const app = new Hono();
   app.use("*", auth);
@@ -112,8 +258,15 @@ export function proxyApi(store: Store, auth: MiddlewareHandler): Hono {
     // answered back to the test handler (x-myapikey-provider), without leaking
     // that header to real agent clients.
     const isProbe = c.req.header("x-myapikey-probe") === "1";
+    // The per-source "test this source" variant pins dispatch to ONE provider:
+    // the candidate chain is reduced to just it, and on failure we stop
+    // immediately (no failover) WITHOUT recording a circuit failure — a manual
+    // probe must not trip the breaker. Failure surfaces the real upstream status
+    // (429/500/…), not a collapsed 502, so the badge shows what really happened.
+    const pinId = c.req.header("x-myapikey-probe-provider") || "";
     // candidates() already restricts the responses chain to supportsResponses sources.
-    const list = candidates(store, model, key);
+    let list = candidates(store, model, key);
+    if (pinId) list = list.filter((p) => p.id === pinId);
     if (!list.length) {
       if (key === "responses") {
         return c.json(
@@ -134,11 +287,14 @@ export function proxyApi(store: Store, auth: MiddlewareHandler): Hono {
     let lastStatus = 502;
     let lastErr = "";
 
-    // Skip providers currently in circuit-breaker cooldown. If every candidate
-    // is cooling, fall back to the full list anyway — cooldown is a heuristic,
-    // and one real attempt beats a guaranteed 502 (a cooled provider that now
-    // succeeds also resets its circuit).
-    const live = list.filter((p) => !store.isCooling(p.id));
+    // Skip providers that are either in circuit-breaker cooldown OR over their
+    // RPM pacing cap. Both are heuristics: if every candidate is skipped, fall
+    // back to the full list anyway — one real attempt beats a guaranteed 502
+    // (a skipped provider that now succeeds also resets its state). A pinned
+    // (per-source) probe ignores both — the user is testing THIS source now,
+    // whatever its breaker/pacing state.
+    const skipped = (p: Provider) => store.isCooling(p.id) || (!!p.rpm && store.rpmUsed(p.id) >= p.rpm);
+    const live = pinId ? list : list.filter((p) => !skipped(p));
     const order = live.length ? live : list;
 
     for (const provider of order) {
@@ -148,6 +304,9 @@ export function proxyApi(store: Store, auth: MiddlewareHandler): Hono {
       // previous provider's upstream name. Absent map/key → send the public name.
       const mapped = store.get().models[model]?.[key]?.modelMap?.[provider.id];
       body.model = mapped ?? model;
+      // Count this attempt toward the source's RPM window — but not for a pinned
+      // probe, which (like circuit state) takes no routing side-effects.
+      if (!pinId) store.recordDispatch(provider.id);
       let upstream: Response;
       try {
         upstream = await fetch(upstreamTarget(provider, key).url, {
@@ -159,6 +318,7 @@ export function proxyApi(store: Store, auth: MiddlewareHandler): Hono {
         // Network error / DNS / timeout → try next provider.
         lastStatus = 502;
         lastErr = "network error";
+        if (pinId) break; // per-source probe: fail fast, no circuit impact.
         const r = store.recordCircuitFailure(provider.id, lastStatus, lastErr);
         if (r.entered) {
           store.pushLog({ ts: Date.now(), model, provider: provider.name, providerId: provider.id, format: wire, status: lastStatus, ms: Date.now() - start, stream, kind: "cooldown", cooldownMs: r.cooldownMs, fails: r.fails, error: lastErr });
@@ -167,10 +327,34 @@ export function proxyApi(store: Store, auth: MiddlewareHandler): Hono {
       }
 
       if (upstream.ok) {
-        // A success closes the circuit (provider is healthy again).
-        store.recordCircuitSuccess(provider.id);
-        store.pushLog({ ts: Date.now(), model, provider: provider.name, providerId: provider.id, format: wire, status: upstream.status, ms: Date.now() - start, stream });
-        return passThrough(upstream, isProbe ? provider.name : undefined);
+        // A 200 from the upstream is NOT proof the call succeeded: some backends
+        // return 200 then truncate the stream (or emit no content) for request
+        // shapes they mishandle. We commit the 200 status to the client right
+        // away (headers are already sent) but OBSERVE the body as it flows and
+        // settle once — on a clean, fully-terminated stream we close the circuit
+        // + log 200; on a truncated/errored stream we log 502, trip the circuit
+        // (so the NEXT call fails over), and — on the anthropic wire — inject a
+        // synthetic SSE error event so the client learns the stream died instead
+        // of seeing a silent EOF. TTFB is captured now; logging is deferred to
+        // the body's end (so the row reflects the real outcome, not just the
+        // headers). See observedBody() for the detection rules.
+        const ttfb = Date.now() - start;
+        const body = observedBody(upstream, {
+          stream,
+          key,
+          onSettle: (info) => {
+            if (info.ok) {
+              store.recordCircuitSuccess(provider.id);
+              store.pushLog({ ts: Date.now(), model, provider: provider.name, providerId: provider.id, format: wire, status: 200, ms: ttfb, stream });
+            } else {
+              // A pinned per-source probe takes no circuit side-effects (a manual
+              // test must not trip the breaker) — mirrors the retryable branch.
+              if (!pinId) store.recordCircuitFailure(provider.id, info.status, info.error || "stream failed");
+              store.pushLog({ ts: Date.now(), model, provider: provider.name, providerId: provider.id, format: wire, status: info.status, ms: ttfb, stream, error: info.error });
+            }
+          },
+        });
+        return new Response(body, { status: upstream.status, headers: downHeaders(upstream, isProbe ? provider.name : undefined) });
       }
       if (RETRYABLE.has(upstream.status)) {
         lastStatus = upstream.status;
@@ -178,7 +362,12 @@ export function proxyApi(store: Store, auth: MiddlewareHandler): Hono {
         // reason for the log (this branch never streams back to the client).
         const txt = await upstream.text().catch(() => "");
         lastErr = shortError(txt) || `HTTP ${upstream.status}`;
-        const r = store.recordCircuitFailure(provider.id, lastStatus, lastErr);
+        if (pinId) break; // per-source probe: fail fast, no circuit impact.
+        // A 429/overloaded upstream usually carries Retry-After; honoring it
+        // cools for exactly as long as asked (clamped) instead of the escalating
+        // guess. Absent (5xx often, or a proxy that stripped it) → escalate.
+        const retryAfterMs = parseRetryAfter(upstream.headers.get("retry-after"));
+        const r = store.recordCircuitFailure(provider.id, lastStatus, lastErr, retryAfterMs);
         if (r.entered) {
           store.pushLog({ ts: Date.now(), model, provider: provider.name, providerId: provider.id, format: wire, status: lastStatus, ms: Date.now() - start, stream, kind: "cooldown", cooldownMs: r.cooldownMs, fails: r.fails, error: lastErr });
         }
@@ -193,6 +382,17 @@ export function proxyApi(store: Store, auth: MiddlewareHandler): Hono {
 
     const last = order[order.length - 1];
     store.pushLog({ ts: Date.now(), model, provider: last.name, providerId: last.id, format: wire, status: lastStatus, ms: Date.now() - start, stream, error: lastErr || `all providers failed (last status ${lastStatus})` });
+    // A pinned (per-source) probe failed: surface the REAL upstream status the
+    // one provider returned (429/500/…), not a collapsed 502, and tag it with
+    // x-myapikey-provider so the source-row badge names the tested source.
+    if (pinId) {
+      const h = new Headers({ "content-type": "application/json" });
+      if (isProbe) h.set("x-myapikey-provider", last.name);
+      return new Response(
+        JSON.stringify({ error: { message: lastErr || `provider failed (status ${lastStatus})`, type: "upstream_error" } }),
+        { status: lastStatus, headers: h },
+      );
+    }
     return c.json(
       { error: { message: `all providers for '${model}' failed (last status ${lastStatus})`, type: "upstream_error" } },
       502,

package/packages/core/src/server/store.ts

@@ -23,6 +23,14 @@ const LOG_TAIL_BYTES = 512 * 1024;
  *  doubling each consecutive failure up to CAP. Resets on the next success. */
 const CB_BASE = 30_000;
 const CB_CAP = 300_000;
+// Floor for honoring an upstream Retry-After hint — small positive values (a
+// lenient "Retry-After: 0"/sub-second) shouldn't read as "no cooldown" and let
+// us re-hammer a just-rate-limited source in a tight loop.
+const CB_MIN = 1_000;
+
+/** RPM pacing window: a source's `rpm` cap counts calls within this trailing
+ *  window. 60s matches the usual "requests per minute" limit. */
+const RPM_WINDOW_MS = 60_000;
 
 /** Per-provider circuit state (in-memory, never persisted). */
 interface CircuitEntry {
@@ -45,6 +53,10 @@ export interface CircuitView {
   lastStatus: number;
   lastReason: string;
   lastTs: number;
+  /** Configured RPM cap (0 = unlimited). */
+  rpm: number;
+  /** Calls forwarded to this source within the trailing 60s window. */
+  rpmUsed: number;
 }
 
 /** One bucket in a stats breakdown (by model / provider / format). `id` is set
@@ -108,6 +120,9 @@ export class Store {
    *  NOT persisted (resets on restart). Mutated via the circuit* methods only,
    *  never through update()/persist(). */
   private circuit = new Map<string, CircuitEntry>();
+  /** Per-provider dispatch timestamps within the RPM pacing window. In-memory,
+   *  NOT persisted (resets on restart). Pruned as `rpmUsed` reads. */
+  private rpm = new Map<string, number[]>();
 
   constructor(dataDir: string) {
     this.dataDir = dataDir;
@@ -401,15 +416,21 @@ export class Store {
    *  consecutive failure (BASE * 2^(fails-1), capped at CAP); `fails` persists
    *  across cooldown expirations and is reset only by success — unless the
    *  provider has been quiet for > CAP, in which case it starts fresh at 1.
+   *  When the upstream told us exactly how long to back off (`retryAfterMs`,
+   *  parsed from a 429/overloaded Retry-After header), honor it — clamped to
+   *  [CB_MIN, CAP] — instead of the escalating guess: the source isn't sicker,
+   *  it just said when it'll be ready. `fails` still increments either way so a
+   *  later hint-less failure continues the escalation from where it left off.
    *  Returns `entered` = transitioned from healthy → cooling this call (the
    *  caller logs a cooldown row only then, to avoid timeline spam), plus the
    *  fails count and cooldown duration for that row. */
-  recordCircuitFailure(id: string, status: number, reason: string): { entered: boolean; fails: number; cooldownMs: number } {
+  recordCircuitFailure(id: string, status: number, reason: string, retryAfterMs?: number): { entered: boolean; fails: number; cooldownMs: number } {
     const now = Date.now();
     const cur = this.circuit.get(id);
     const stale = !cur || now - cur.lastTs > CB_CAP;
     const fails = stale ? 1 : cur!.fails + 1;
-    const cooldownMs = Math.min(CB_CAP, CB_BASE * 2 ** (fails - 1));
+    const hint = retryAfterMs && Number.isFinite(retryAfterMs) && retryAfterMs > 0 ? retryAfterMs : 0;
+    const cooldownMs = hint ? Math.min(CB_CAP, Math.max(CB_MIN, Math.round(hint))) : Math.min(CB_CAP, CB_BASE * 2 ** (fails - 1));
     const until = now + cooldownMs;
     const wasCooling = !!cur && cur.until > now;
     this.circuit.set(id, { fails, until, lastStatus: status, lastReason: reason, lastTs: now });
@@ -431,6 +452,30 @@ export class Store {
     this.circuit.set(id, { ...cur, fails: 0, until: 0 });
   }
 
+  // --- RPM pacing (in-memory sliding window, never persisted) ---
+
+  /** Count this source's forwarded calls in the trailing RPM_WINDOW_MS, pruning
+   *  expired entries as it reads (timestamps are appended oldest-first). Returns
+   *  0 for a source with no recent activity. */
+  rpmUsed(id: string): number {
+    const arr = this.rpm.get(id);
+    if (!arr || !arr.length) return 0;
+    const cutoff = Date.now() - RPM_WINDOW_MS;
+    let i = 0;
+    while (i < arr.length && arr[i] < cutoff) i++;
+    if (i > 0) arr.splice(0, i);
+    if (!arr.length) this.rpm.delete(id);
+    return arr.length;
+  }
+
+  /** Record that we forwarded a call to this source (called by dispatch right
+   *  before the upstream fetch). Lets the next pacing check count this attempt. */
+  recordDispatch(id: string): void {
+    const arr = this.rpm.get(id);
+    if (arr) arr.push(Date.now());
+    else this.rpm.set(id, [Date.now()]);
+  }
+
   /** Snapshot of every configured provider's circuit state for GET /admin/circuit.
    *  Healthy providers appear as state "open"; a provider deleted while cooling
    *  simply drops out (we iterate the live config, not the map). */
@@ -449,6 +494,8 @@ export class Store {
         lastStatus: c?.lastStatus ?? 0,
         lastReason: c?.lastReason ?? "",
         lastTs: c?.lastTs ?? 0,
+        rpm: p.rpm ?? 0,
+        rpmUsed: this.rpmUsed(p.id),
       };
     });
   }

package/packages/core/src/shared/types.ts

@@ -19,6 +19,13 @@ export interface Provider {
   apiKey: string;
   /** Which wire formats this backend responds to. */
   formats: Format[];
+  /** Optional request-per-minute cap (RPM pacing). When set, dispatch skips this
+   *  source once it has forwarded `rpm` calls in the trailing 60s window — the
+   *  request fails over to the next source instead of racing the upstream's own
+   *  rate limit (and burning a free/quota-bound key). 0/absent = unlimited.
+   *  The limit is on the key, so it's per-source and shared across every model
+   *  routed through it. Tracked in-memory only (see Store.rpmUsed). */
+  rpm?: number;
   /** Whether this backend also implements the OpenAI Responses API (/responses).
    *  NOT implied by `formats` — many openai-compatible backends lack it. Opt-in. */
   supportsResponses?: boolean;