myapikey 0.17.0 → 0.19.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "myapikey",
-  "version": "0.17.0",
+  "version": "0.19.0",
   "type": "module",
   "description": "Personal LLM API gateway & proxy — one address + one API key for all your models. Forwards OpenAI & Anthropic calls to your backends with failover and a circuit breaker. Pure passthrough, no format translation. Self-hosted (CLI + web UI).",
   "keywords": [

package/packages/core/src/cli/index.ts

@@ -181,7 +181,7 @@ model.command("list").action(async () => {
     console.log(m.name);
     for (const f of fmts) {
       const fe = m[f];
-      const chain = fe.providers.map((p: any) => (p.model ? `${p.name}→${p.model}` : p.name)).join(" → ") || "(none)";
+      const chain = fe.providers.map((p: any) => `${p.model ? `${p.name}→${p.model}` : p.name}${p.thinking ? ` thinking=${p.thinking}` : ""}`).join(" → ") || "(none)";
       console.log(`  ${f.padEnd(9)} ${fe.enabled ? "✓" : "·"} ${chain}`);
     }
     if (m.paceRpm) console.log(`  pace      ${m.paceRpm}/min (one every ${Math.round(60 / m.paceRpm)}s)`);
@@ -264,6 +264,20 @@ model
     console.log(r.paceRpm ? `Pacing ${name} at ${r.paceRpm}/min (one every ${Math.round(60 / r.paceRpm)}s).` : `Pacing cleared for ${name} (unlimited).`);
   });
 
+model
+  .command("thinking <name> <index> [value]")
+  .description("set/clear the default thinking level of one chain slot (effort on openai/responses, budget tokens on anthropic; empty = clear)")
+  .addOption(fmtOption())
+  .action(async (name: string, indexRaw: string, value: string | undefined, opts: { format: "openai" | "anthropic" | "responses" }) => {
+    const index = Number(indexRaw);
+    const r = (await api(ctx(), "PUT", `/admin/models/${encodeURIComponent(name)}/thinking`, { format: opts.format, index, thinking: value ?? "" })) as { thinking?: string };
+    console.log(
+      r.thinking
+        ? `Default thinking for ${name} [${opts.format}] slot ${index}: ${r.thinking}.`
+        : `Default thinking cleared for ${name} [${opts.format}] slot ${index}.`,
+    );
+  });
+
 model.command("remove <name>").description("remove a model entirely (both formats)").action(async (name: string) => {
   await api(ctx(), "DELETE", `/admin/models/${encodeURIComponent(name)}`);
   console.log(`Removed ${name}.`);

package/packages/core/src/server/admin.ts

@@ -312,6 +312,8 @@ export function adminApi(store: Store, auth: MiddlewareHandler, openai: Hono, an
         // send the public model name verbatim). Carried inline on each slot so
         // a provider can appear more than once with different upstream names.
         model: s.model,
+        // Default thinking level for this slot (undefined = pure passthrough).
+        thinking: s.thinking,
       })),
     });
     const models = Object.entries(d.models).map(([name, e]) => ({
@@ -540,6 +542,52 @@ export function adminApi(store: Store, auth: MiddlewareHandler, openai: Hono, an
     return c.json({ ok: true });
   });
 
+  // Set (or clear) the default thinking level for ONE chain slot (addressed by
+  // `index`, like /map above). The value is whatever the slot's wire format
+  // takes natively: an effort token ("low"/"medium"/"high"/…) on openai/
+  // responses slots; a thinking budget in TOKENS (positive integer) on
+  // anthropic slots (Anthropic has no named levels). An empty value clears it
+  // (back to pure passthrough). Dispatch applies it only when the request
+  // carries no thinking parameter of its own.
+  app.put("/models/:name/thinking", async (c) => {
+    const name = c.req.param("name");
+    const body = await readJson<{ format?: RouteKey; index?: number; thinking?: string | number }>(c.req.raw);
+    if (!body?.format) return c.json({ error: { message: "format is required" } }, 400);
+    if (!Number.isInteger(body?.index) || (body?.index ?? -1) < 0)
+      return c.json({ error: { message: "index (non-negative integer) is required" } }, 400);
+    const format = body.format;
+    const index = body.index!;
+    const raw = body.thinking === undefined ? "" : String(body.thinking).trim();
+    if (raw && format === "anthropic" && (!/^\d+$/.test(raw) || Number(raw) < 1)) {
+      return c.json({ error: { message: "anthropic thinking default must be a positive integer (thinking budget tokens, e.g. 8192)" } }, 400);
+    }
+    if (raw && format !== "anthropic" && raw.length > 32) {
+      return c.json({ error: { message: "thinking default too long (max 32 chars)" } }, 400);
+    }
+    const value = raw && format === "anthropic" ? String(Number(raw)) : raw;
+    let errStatus = 0;
+    let errMsg = "";
+    await store.update((d) => {
+      const entry = d.models[name];
+      if (!entry) {
+        errStatus = 404;
+        errMsg = "model not found";
+        return;
+      }
+      const fe = entry[format];
+      if (index >= fe.providers.length) {
+        errStatus = 400;
+        errMsg = "index out of range for this model's chain";
+        return;
+      }
+      if (value) fe.providers[index].thinking = value;
+      else delete fe.providers[index].thinking;
+    });
+    if (errStatus === 404) return c.json({ error: { message: errMsg } }, 404);
+    if (errStatus === 400) return c.json({ error: { message: errMsg } }, 400);
+    return c.json({ ok: true, thinking: value || undefined });
+  });
+
   app.post("/models/:name/disable", async (c) => {
     const name = c.req.param("name");
     const body = await readJson<{ format?: RouteKey }>(c.req.raw);

package/packages/core/src/server/proxy.ts

@@ -62,12 +62,14 @@ function parseResetFromBody(text: string): number | undefined {
 }
 
 /** One resolved routing slot: the provider to forward to plus THIS slot's
- *  optional upstream model name (absent = send the public model name). The
- *  same provider may occupy several slots in a chain — each is an independent
- *  failover slot carrying its own upstream model. */
+ *  optional upstream model name (absent = send the public model name) and
+ *  default thinking level. The same provider may occupy several slots in a
+ *  chain — each is an independent failover slot carrying its own upstream
+ *  model. */
 interface CandidateSlot {
   provider: Provider;
   model?: string;
+  thinking?: string;
 }
 
 /** Resolve the ordered, compatible provider slots for a model on a routing slot. */
@@ -83,7 +85,7 @@ function candidates(store: Store, model: string, key: RouteKey): CandidateSlot[]
   return fe.providers
     .map((s): CandidateSlot | null => {
       const p = byId.get(s.id);
-      return p ? { provider: p, model: s.model } : null;
+      return p ? { provider: p, model: s.model, thinking: s.thinking } : null;
     })
     .filter((slot): slot is CandidateSlot => {
       if (!slot) return false;
@@ -104,6 +106,84 @@ function notFound(c: Context, model: string) {
   );
 }
 
+// --- per-slot default thinking level -----------------------------------------
+// Each routing slot can carry a default thinking level, injected ONLY when the
+// request itself carries no thinking parameter (a request's own setting always
+// wins). No cross-format translation — the default is applied in the wire's own
+// dialect, matching how the model rewrite stays a pure passthrough body edit:
+//   openai /chat/completions → `reasoning_effort: "<token>"` (low/medium/high/…)
+//   /responses               → `reasoning: { effort: "<token>" }`
+//   anthropic /v1/messages   → `thinking: { type: "enabled", budget_tokens: N }`
+//     (Anthropic has no named levels — the stored default IS the budget, a
+//     positive integer. The API also demands max_tokens > budget_tokens, so an
+//     at-or-below cap is lifted to budget+1; otherwise the call would 400 —
+//     including our own max_tokens:1 probe.)
+
+/** The body field this slot's thinking parameter travels in. */
+function thinkingField(key: RouteKey): "reasoning_effort" | "reasoning" | "thinking" {
+  if (key === "anthropic") return "thinking";
+  if (key === "responses") return "reasoning";
+  return "reasoning_effort";
+}
+
+/** Whether the CLIENT's own body already steers thinking on this slot, plus a
+ *  short best-effort display of what it asked for (for the log row). On openai
+ *  chat, any of the ecosystem's switches counts — compat backends also honor
+ *  `thinking` / `enable_thinking`, and double-specifying alongside a default
+ *  would only confuse them. */
+function clientThinking(body: Record<string, unknown>, key: RouteKey): { set: boolean; value?: string } {
+  const show = (v: unknown): string | undefined => (typeof v === "object" && v !== null ? JSON.stringify(v) : String(v));
+  if (key === "responses") {
+    const r = body.reasoning;
+    if (r === undefined || r === null) return { set: false };
+    const effort = (r as { effort?: unknown }).effort;
+    return { set: true, value: effort === undefined ? show(r) : show(effort) };
+  }
+  if (key === "anthropic") {
+    const t = body.thinking;
+    if (t === undefined || t === null) return { set: false };
+    const budget = (t as { budget_tokens?: unknown }).budget_tokens;
+    return { set: true, value: budget === undefined ? show(t) : show(budget) };
+  }
+  const found = [
+    body.reasoning_effort,
+    (body.reasoning as { effort?: unknown } | undefined)?.effort,
+    (body.thinking as { budget_tokens?: unknown } | undefined)?.budget_tokens,
+    (body.thinking as { type?: unknown } | undefined)?.type,
+    body.enable_thinking,
+  ].find((v) => v !== undefined && v !== null);
+  return { set: found !== undefined, value: found === undefined ? undefined : show(found) };
+}
+
+/** Apply THIS slot's thinking default to the body (mutating it), and return the
+ *  log row's `thinking` field: the level that applied and where it came from.
+ *  Undefined = nothing applied. A slot without a default CLEARS whatever an
+ *  earlier failover slot injected, so a chain never leaks slot A's level into
+ *  slot B (the model rewrite has the same recompute-per-attempt discipline). */
+function applySlotThinking(
+  body: Record<string, unknown>,
+  key: RouteKey,
+  def: string | undefined,
+  client: { set: boolean; value?: string },
+): { value: string; from: "client" | "default" } | undefined {
+  if (client.set) return { value: client.value ?? "", from: "client" };
+  const field = thinkingField(key);
+  if (!def) {
+    delete body[field]; // no-op unless a previous slot injected one
+    return undefined;
+  }
+  if (key === "anthropic") {
+    const budget = Number(def);
+    body.thinking = { type: "enabled", budget_tokens: budget };
+    if (typeof body.max_tokens === "number" && body.max_tokens <= budget) body.max_tokens = budget + 1;
+  } else if (key === "responses") {
+    body.reasoning = { effort: def };
+  } else {
+    body.reasoning_effort = def;
+  }
+  return { value: def, from: "default" };
+}
+
 /** Auth headers for the Anthropic wire format. Sends BOTH x-api-key and
  *  Authorization: Bearer (same key). Native Anthropic (api.anthropic.com)
  *  accepts either; anthropic- COMPATIBLE surfaces (sensenova, Volcengine Ark,
@@ -353,6 +433,9 @@ export function proxyApi(
     const model: string = body.model;
     const wire: Format = key === "anthropic" ? "anthropic" : "openai";
     const stream = body.stream === true;
+    // The client's own thinking setting, read ONCE from the original body (the
+    // per-slot default injection below would otherwise read back as "client").
+    const clientThink = clientThinking(body, key);
     // The model-page "test" button drives dispatch via an in-process loopback
     // (adminApi calls v1.request). The probe is a real call in every respect —
     // including being logged — so we only tag it to report WHICH provider
@@ -393,6 +476,9 @@ export function proxyApi(
     const start = Date.now();
     let lastStatus = 502;
     let lastErr = "";
+    // Thinking level of the most recent attempt (for the all-failed row after
+    // the rounds loop; per-attempt rows capture the loop's own `think` const).
+    let lastThink: { value: string; from: "client" | "default" } | undefined;
     // Runtime log (console + server.log). Errors and notable events only — the
     // per-call history these lines summarize goes to pushLog/logs.jsonl anyway.
     // UI-triggered probes are excluded: their outcome is shown inline already.
@@ -433,135 +519,172 @@ export function proxyApi(
       }
     }
 
-    // Skip providers that are either in circuit-breaker cooldown OR over their
-    // RPM pacing cap. Both are heuristics: if every candidate is skipped, fall
-    // back to the full list anyway — one real attempt beats a guaranteed 502
-    // (a skipped provider that now succeeds also resets its state). A pinned
-    // (per-source) probe ignores both — the user is testing THIS source now,
-    // whatever its breaker/pacing state.
-    const skipped = (slot: CandidateSlot) => {
-      const p = slot.provider;
-      return store.isCooling(p.id) || (!!p.rpm && store.rpmUsed(p.id) >= p.rpm);
+    // Selection + failover run in ROUNDS. Each round attempts the candidates
+    // that are neither in circuit-breaker cooldown nor over their rpm cap; a
+    // capped-but-healthy source still spills to the next free source (failover
+    // first). Only when NOTHING is immediately usable but some candidate is
+    // merely over its rpm cap does the request QUEUE: sleep until that source's
+    // soonest window slot frees and run another round — the client perceives
+    // only the wait (or its own timeout), never an rpm error. There is no wait
+    // cap: rounds terminate anyway because retryable failures escalate the
+    // circuit breaker, so sources converge to cooling and the final round is
+    // the old try-anyway fallback (full list — one real attempt beats a
+    // guaranteed 502, and a skipped provider that now succeeds also resets its
+    // state). A pinned (per-source) probe ignores all of this — the user is
+    // testing THIS source now, whatever its breaker/pacing state.
+    const cooling = (slot: CandidateSlot) => store.isCooling(slot.provider.id);
+    const overRpm = (slot: CandidateSlot) => !!slot.provider.rpm && store.rpmUsed(slot.provider.id) >= slot.provider.rpm;
+    /** Sleep until the soonest rpm slot frees among `capped`. Returns false
+     *  when the client hung up while queued — the caller drops the request
+     *  (nothing was sent upstream, so no log row either). */
+    const waitForSlot = async (capped: CandidateSlot[]): Promise<boolean> => {
+      const wait = Math.max(1, Math.min(...capped.map((s) => store.rpmNextFreeMs(s.provider.id))));
+      if (!isProbe) rt.warn(`proxy model=${model}: all sources over rpm cap — queued, next slot in ~${Math.round(wait / 1000)}s`);
+      await new Promise((r) => setTimeout(r, wait));
+      return !c.req.raw.signal?.aborted;
     };
-    const live = pinIndex != null ? list : list.filter((slot) => !skipped(slot));
-    const order = live.length ? live : list;
-
-    for (const slot of order) {
-      const provider = slot.provider;
-      // Per-slot upstream model name (absent = send the public name). Read from
-      // the slot each iteration, so failover never carries the previous slot's
-      // upstream name.
-      body.model = slot.model ?? model;
-      // The actual upstream model forwarded this attempt (after the per-slot
-      // rewrite). Recorded on the log row so history shows which real model a
-      // routed call landed on when a source remaps the public name. `undefined`
-      // when the public name went through verbatim — JSON.stringify drops it, so
-      // identity + legacy rows stay clean.
-      const upstreamModel = slot.model && slot.model !== model ? slot.model : undefined;
-      // Count this attempt toward the source's RPM window — but not for a pinned
-      // probe, which (like circuit state) takes no routing side-effects.
-      if (pinIndex == null) store.recordDispatch(provider.id);
-      let upstream: Response;
-      try {
-        upstream = await fetch(upstreamTarget(provider, key).url, {
-          method: "POST",
-          headers: upstreamHeaders(provider, wire, clientVersion),
-          body: JSON.stringify(body),
-        });
-      } catch {
-        // Network error / DNS / timeout → try next provider.
-        lastStatus = 502;
-        lastErr = "network error";
-        if (pinIndex != null) break; // per-source probe: fail fast, no circuit impact.
-        sayFailover(provider, "network error");
-        const r = store.recordCircuitFailure(provider.id, lastStatus, lastErr);
-        if (r.entered) {
-          store.pushLog({ ts: Date.now(), model, upstreamModel, provider: provider.name, providerId: provider.id, format: wire, status: lastStatus, ms: Date.now() - start, stream, kind: "cooldown", cooldownMs: r.cooldownMs, fails: r.fails, error: lastErr });
-          sayCooldown(provider, r);
-        }
-        continue;
-      }
 
-      if (upstream.ok) {
-        // A 200 from the upstream is NOT proof the call succeeded: some backends
-        // return 200 then truncate the stream (or emit no content) for request
-        // shapes they mishandle. We commit the 200 status to the client right
-        // away (headers are already sent) but OBSERVE the body as it flows and
-        // settle once — on a clean, fully-terminated stream we close the circuit
-        // + log 200; on a truncated/errored stream we log 502, trip the circuit
-        // (so the NEXT call fails over), and — on the anthropic wire — inject a
-        // synthetic SSE error event so the client learns the stream died instead
-        // of seeing a silent EOF. TTFB is captured now; logging is deferred to
-        // the body's end (so the row reflects the real outcome, not just the
-        // headers). See observedBody() for the detection rules.
-        const ttfb = Date.now() - start;
-        const out = observedBody(upstream, {
-          stream,
-          key,
-          requestMessages: body.messages,
-          onSettle: (info) => {
-            if (info.ok) {
-              store.recordCircuitSuccess(provider.id);
-              store.pushLog({ ts: Date.now(), model, upstreamModel, provider: provider.name, providerId: provider.id, format: wire, status: 200, ms: ttfb, stream, usage: info.usage });
-            } else {
-              // A pinned per-source probe takes no circuit side-effects (a manual
-              // test must not trip the breaker) — mirrors the retryable branch.
-              if (pinIndex == null) store.recordCircuitFailure(provider.id, info.status, info.error || "stream failed");
-              if (!isProbe) rt.warn(`proxy stream failed: provider '${provider.name}' status=${info.status} (${info.error || "stream failed"})`);
-              store.pushLog({ ts: Date.now(), model, upstreamModel, provider: provider.name, providerId: provider.id, format: wire, status: info.status, ms: ttfb, stream, error: info.error });
-            }
-          },
-        });
-        return new Response(out, { status: upstream.status, headers: downHeaders(upstream, isProbe ? provider.name : undefined) });
+    for (;;) {
+      // Re-read the chain each round: config and circuit state move while queued.
+      const cur = pinIndex != null ? list : candidates(store, model, key);
+      if (pinIndex == null && !cur.length) return notFound(c, model); // disabled while queued
+      const live = pinIndex != null ? cur : cur.filter((s) => !cooling(s) && !overRpm(s));
+      const capped = pinIndex != null ? [] : cur.filter((s) => !cooling(s) && overRpm(s));
+      if (pinIndex == null && !live.length && capped.length) {
+        if (await waitForSlot(capped)) continue;
+        return new Response(null, { status: 499 });
       }
-      if (RETRYABLE.has(upstream.status)) {
-        lastStatus = upstream.status;
-        // Drain so the connection can be reused, then move on; capture the
-        // reason for the log (this branch never streams back to the client).
-        const txt = await upstream.text().catch(() => "");
-        lastErr = shortError(txt) || `HTTP ${upstream.status}`;
-        if (pinIndex != null) break; // per-source probe: fail fast, no circuit impact.
-        sayFailover(provider, `HTTP ${lastStatus} (${lastErr})`);
-        // A 429/overloaded upstream usually carries Retry-After; honoring it
-        // cools for exactly as long as asked (clamped) instead of the escalating
-        // guess. Absent (5xx often, OR a quota error that buried the reset time
-        // in the BODY — e.g. Volcengine Ark's 1308 "您的限额将在 <datetime> 重置")
-        // → parse that deadline out of the body, else fall back to escalating.
-        const retryAfterMs = parseRetryAfter(upstream.headers.get("retry-after"));
-        const resetInMs = retryAfterMs ? undefined : parseResetFromBody(txt);
-        const r = store.recordCircuitFailure(provider.id, lastStatus, lastErr, retryAfterMs ?? resetInMs, !!resetInMs);
-        if (r.entered) {
-          store.pushLog({ ts: Date.now(), model, upstreamModel, provider: provider.name, providerId: provider.id, format: wire, status: lastStatus, ms: Date.now() - start, stream, kind: "cooldown", cooldownMs: r.cooldownMs, fails: r.fails, error: lastErr });
-          sayCooldown(provider, r);
+      const order = live.length ? live : cur;
+
+      for (const slot of order) {
+        const provider = slot.provider;
+        // Per-slot upstream model name (absent = send the public name). Read from
+        // the slot each iteration, so failover never carries the previous slot's
+        // upstream name.
+        body.model = slot.model ?? model;
+        // The actual upstream model forwarded this attempt (after the per-slot
+        // rewrite). Recorded on the log row so history shows which real model a
+        // routed call landed on when a source remaps the public name. `undefined`
+        // when the public name went through verbatim — JSON.stringify drops it, so
+        // identity + legacy rows stay clean.
+        const upstreamModel = slot.model && slot.model !== model ? slot.model : undefined;
+        // Per-slot default thinking level: re-read from the slot and re-applied
+        // every attempt (an earlier slot's injection is cleared when this one
+        // has no default — see applySlotThinking). The request's own thinking
+        // parameter, when present, was captured once above and always wins.
+        const think = applySlotThinking(body, key, slot.thinking, clientThink);
+        lastThink = think;
+        // Count this attempt toward the source's RPM window — but not for a pinned
+        // probe, which (like circuit state) takes no routing side-effects.
+        if (pinIndex == null) store.recordDispatch(provider.id);
+        let upstream: Response;
+        try {
+          upstream = await fetch(upstreamTarget(provider, key).url, {
+            method: "POST",
+            headers: upstreamHeaders(provider, wire, clientVersion),
+            body: JSON.stringify(body),
+          });
+        } catch {
+          // Network error / DNS / timeout → try next provider.
+          lastStatus = 502;
+          lastErr = "network error";
+          if (pinIndex != null) break; // per-source probe: fail fast, no circuit impact.
+          sayFailover(provider, "network error");
+          const r = store.recordCircuitFailure(provider.id, lastStatus, lastErr);
+          if (r.entered) {
+            store.pushLog({ ts: Date.now(), model, upstreamModel, provider: provider.name, providerId: provider.id, format: wire, status: lastStatus, ms: Date.now() - start, stream, thinking: think, kind: "cooldown", cooldownMs: r.cooldownMs, fails: r.fails, error: lastErr });
+            sayCooldown(provider, r);
+          }
+          continue;
+        }
+
+        if (upstream.ok) {
+          // A 200 from the upstream is NOT proof the call succeeded: some backends
+          // return 200 then truncate the stream (or emit no content) for request
+          // shapes they mishandle. We commit the 200 status to the client right
+          // away (headers are already sent) but OBSERVE the body as it flows and
+          // settle once — on a clean, fully-terminated stream we close the circuit
+          // + log 200; on a truncated/errored stream we log 502, trip the circuit
+          // (so the NEXT call fails over), and — on the anthropic wire — inject a
+          // synthetic SSE error event so the client learns the stream died instead
+          // of seeing a silent EOF. TTFB is captured now; logging is deferred to
+          // the body's end (so the row reflects the real outcome, not just the
+          // headers). See observedBody() for the detection rules.
+          const ttfb = Date.now() - start;
+          const out = observedBody(upstream, {
+            stream,
+            key,
+            requestMessages: body.messages,
+            onSettle: (info) => {
+              if (info.ok) {
+                store.recordCircuitSuccess(provider.id);
+                store.pushLog({ ts: Date.now(), model, upstreamModel, provider: provider.name, providerId: provider.id, format: wire, status: 200, ms: ttfb, stream, thinking: think, usage: info.usage });
+              } else {
+                // A pinned per-source probe takes no circuit side-effects (a manual
+                // test must not trip the breaker) — mirrors the retryable branch.
+                if (pinIndex == null) store.recordCircuitFailure(provider.id, info.status, info.error || "stream failed");
+                if (!isProbe) rt.warn(`proxy stream failed: provider '${provider.name}' status=${info.status} (${info.error || "stream failed"})`);
+                store.pushLog({ ts: Date.now(), model, upstreamModel, provider: provider.name, providerId: provider.id, format: wire, status: info.status, ms: ttfb, stream, thinking: think, error: info.error });
+              }
+            },
+          });
+          return new Response(out, { status: upstream.status, headers: downHeaders(upstream, isProbe ? provider.name : undefined) });
         }
-        continue;
+        if (RETRYABLE.has(upstream.status)) {
+          lastStatus = upstream.status;
+          // Drain so the connection can be reused, then move on; capture the
+          // reason for the log (this branch never streams back to the client).
+          const txt = await upstream.text().catch(() => "");
+          lastErr = shortError(txt) || `HTTP ${upstream.status}`;
+          if (pinIndex != null) break; // per-source probe: fail fast, no circuit impact.
+          sayFailover(provider, `HTTP ${lastStatus} (${lastErr})`);
+          // A 429/overloaded upstream usually carries Retry-After; honoring it
+          // cools for exactly as long as asked (clamped) instead of the escalating
+          // guess. Absent (5xx often, OR a quota error that buried the reset time
+          // in the BODY — e.g. Volcengine Ark's 1308 "您的限额将在 <datetime> 重置")
+          // → parse that deadline out of the body, else fall back to escalating.
+          const retryAfterMs = parseRetryAfter(upstream.headers.get("retry-after"));
+          const resetInMs = retryAfterMs ? undefined : parseResetFromBody(txt);
+          const r = store.recordCircuitFailure(provider.id, lastStatus, lastErr, retryAfterMs ?? resetInMs, !!resetInMs);
+          if (r.entered) {
+            store.pushLog({ ts: Date.now(), model, upstreamModel, provider: provider.name, providerId: provider.id, format: wire, status: lastStatus, ms: Date.now() - start, stream, thinking: think, kind: "cooldown", cooldownMs: r.cooldownMs, fails: r.fails, error: lastErr });
+            sayCooldown(provider, r);
+          }
+          continue;
+        }
+        // Non-retryable client error: return it to the caller as-is. Read the
+        // error text off a CLONE so the original body still streams back.
+        const errText = await upstream.clone().text().catch(() => "");
+        store.pushLog({ ts: Date.now(), model, upstreamModel, provider: provider.name, providerId: provider.id, format: wire, status: upstream.status, ms: Date.now() - start, stream, thinking: think, error: shortError(errText) || `HTTP ${upstream.status}` });
+        return passThrough(upstream, isProbe ? provider.name : undefined);
       }
-      // Non-retryable client error: return it to the caller as-is. Read the
-      // error text off a CLONE so the original body still streams back.
-      const errText = await upstream.clone().text().catch(() => "");
-      store.pushLog({ ts: Date.now(), model, upstreamModel, provider: provider.name, providerId: provider.id, format: wire, status: upstream.status, ms: Date.now() - start, stream, error: shortError(errText) || `HTTP ${upstream.status}` });
-      return passThrough(upstream, isProbe ? provider.name : undefined);
-    }
 
-    const last = order[order.length - 1];
-    const lastUpstreamModel = last.model && last.model !== model ? last.model : undefined;
-    if (!isProbe) rt.error(`proxy all providers failed model=${model} (last status ${lastStatus})`);
-    store.pushLog({ ts: Date.now(), model, upstreamModel: lastUpstreamModel, provider: last.provider.name, providerId: last.provider.id, format: wire, status: lastStatus, ms: Date.now() - start, stream, error: lastErr || `all providers failed (last status ${lastStatus})` });
-    // A pinned (per-source) probe failed: surface the REAL upstream status the
-    // one slot returned (429/500/…), not a collapsed 502, and tag it with
-    // x-myapikey-provider so the source-row badge names the tested source.
-    if (pinIndex != null) {
-      const h = new Headers({ "content-type": "application/json" });
-      if (isProbe) h.set("x-myapikey-provider", encodeTag(last.provider.name));
-      return new Response(
-        JSON.stringify({ error: { message: lastErr || `provider failed (status ${lastStatus})`, type: "upstream_error" } }),
-        { status: lastStatus, headers: h },
+      const last = order[order.length - 1];
+      const lastUpstreamModel = last.model && last.model !== model ? last.model : undefined;
+      // Every slot in this round failed retryably. rpm-capped candidates
+      // remain - queue for their next slot instead of erroring the client.
+      if (pinIndex == null && capped.length) {
+        if (await waitForSlot(capped)) continue;
+        return new Response(null, { status: 499 });
+      }
+      if (!isProbe) rt.error(`proxy all providers failed model=${model} (last status ${lastStatus})`);
+      store.pushLog({ ts: Date.now(), model, upstreamModel: lastUpstreamModel, provider: last.provider.name, providerId: last.provider.id, format: wire, status: lastStatus, ms: Date.now() - start, stream, thinking: lastThink, error: lastErr || `all providers failed (last status ${lastStatus})` });
+      // A pinned (per-source) probe failed: surface the REAL upstream status the
+      // one slot returned (429/500/…), not a collapsed 502, and tag it with
+      // x-myapikey-provider so the source-row badge names the tested source.
+      if (pinIndex != null) {
+        const h = new Headers({ "content-type": "application/json" });
+        if (isProbe) h.set("x-myapikey-provider", encodeTag(last.provider.name));
+        return new Response(
+          JSON.stringify({ error: { message: lastErr || `provider failed (status ${lastStatus})`, type: "upstream_error" } }),
+          { status: lastStatus, headers: h },
+        );
+      }
+      return c.json(
+        { error: { message: `all providers for '${model}' failed (last status ${lastStatus})`, type: "upstream_error" } },
+        502,
       );
     }
-    return c.json(
-      { error: { message: `all providers for '${model}' failed (last status ${lastStatus})`, type: "upstream_error" } },
-      502,
-    );
   };
 
   // OpenAI surface: chat/completions + responses (/models is registered above,

package/packages/core/src/server/store.ts

@@ -607,6 +607,28 @@ export class Store {
     else this.rpm.set(id, [Date.now()]);
   }
 
+  /** Ms until one slot frees in this source's trailing window (i.e. until its
+   *  OLDEST recorded dispatch ages out) - the soonest a call over the source's
+   *  rpm cap could go through. 0 when the window's first entry has already
+   *  expired or the source has no recent activity (whether a slot is actually
+   *  free is the caller's rpmUsed-vs-rpm check). Prunes expired entries as it
+   *  reads, like rpmUsed. Used by dispatch to sleep out an rpm cap when every
+   *  candidate source is over its limit. */
+  rpmNextFreeMs(id: string): number {
+    const arr = this.rpm.get(id);
+    if (!arr || !arr.length) return 0;
+    const now = Date.now();
+    const cutoff = now - RPM_WINDOW_MS;
+    let i = 0;
+    while (i < arr.length && arr[i] < cutoff) i++;
+    if (i > 0) arr.splice(0, i);
+    if (!arr.length) {
+      this.rpm.delete(id);
+      return 0;
+    }
+    return Math.max(0, arr[0] + RPM_WINDOW_MS - now);
+  }
+
   // --- even pacing (per model, in-memory leaky-bucket queue) ---
 
   /** Reserve the next release slot for a call to `model`, paced at `rpm`

package/packages/core/src/shared/types.ts

@@ -68,6 +68,13 @@ export interface ChainSlot {
   /** Upstream model name to send to this provider. Absent = forward the public
    *  model name (the key in GateConfig.models) unchanged. */
   model?: string;
+  /** Default thinking level for this slot, applied when the REQUEST carries no
+   *  thinking parameter of its own (a request's own setting always wins). The
+   *  value is whatever the slot's wire format takes natively — no translation:
+   *  an effort token ("low"/"medium"/"high"/…) on openai/responses slots, a
+   *  thinking budget in tokens (positive integer) on anthropic slots. Absent =
+   *  pure passthrough. */
+  thinking?: string;
 }
 
 /** A model's routing dimensions — one per forwarding endpoint. /chat/completions
@@ -148,6 +155,12 @@ export interface LogEntry {
   usage?: Usage;
   /** Short upstream error text on non-2xx (omitted on success). */
   error?: string;
+  /** The thinking level this call ran with, and where it came from: "client" =
+   *  the request carried its own thinking parameter (forwarded untouched);
+   *  "default" = the gateway injected the routing slot's `thinking` default
+   *  because the request had none. Absent when neither applied (no default
+   *  configured, request carried nothing). */
+  thinking?: { value: string; from: "client" | "default" };
   /** Row kind. Absent on legacy lines → treated as a normal call. "cooldown"
    *  marks a circuit-breaker event (a provider just entered cooldown), shown
    *  distinctly in the timeline alongside the failures that caused it. */