myapikey 0.16.0 → 0.17.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "myapikey",
-  "version": "0.16.0",
+  "version": "0.17.0",
   "type": "module",
   "description": "Personal LLM API gateway & proxy — one address + one API key for all your models. Forwards OpenAI & Anthropic calls to your backends with failover and a circuit breaker. Pure passthrough, no format translation. Self-hosted (CLI + web UI).",
   "keywords": [

package/packages/core/src/cli/index.ts

@@ -184,6 +184,7 @@ model.command("list").action(async () => {
       const chain = fe.providers.map((p: any) => (p.model ? `${p.name}→${p.model}` : p.name)).join(" → ") || "(none)";
       console.log(`  ${f.padEnd(9)} ${fe.enabled ? "✓" : "·"} ${chain}`);
     }
+    if (m.paceRpm) console.log(`  pace      ${m.paceRpm}/min (one every ${Math.round(60 / m.paceRpm)}s)`);
   }
 });
 
@@ -254,6 +255,15 @@ model
     console.log(`Priority for ${name} [${opts.format}]: ${refs.join(" → ")}`);
   });
 
+model
+  .command("pace <name> [rpm]")
+  .description("set/clear the per-model even-pacing limit (requests/min, spread one every 60/rpm s; 0 = unlimited)")
+  .action(async (name: string, rpmRaw: string) => {
+    const rpm = Math.floor(Number(rpmRaw ?? 0)) || 0;
+    const r = (await api(ctx(), "PUT", `/admin/models/${encodeURIComponent(name)}/pace`, { rpm })) as { paceRpm: number };
+    console.log(r.paceRpm ? `Pacing ${name} at ${r.paceRpm}/min (one every ${Math.round(60 / r.paceRpm)}s).` : `Pacing cleared for ${name} (unlimited).`);
+  });
+
 model.command("remove <name>").description("remove a model entirely (both formats)").action(async (name: string) => {
   await api(ctx(), "DELETE", `/admin/models/${encodeURIComponent(name)}`);
   console.log(`Removed ${name}.`);

package/packages/core/src/server/admin.ts

@@ -319,6 +319,7 @@ export function adminApi(store: Store, auth: MiddlewareHandler, openai: Hono, an
       openai: proj(e.openai),
       anthropic: proj(e.anthropic),
       responses: proj(e.responses),
+      paceRpm: e.paceRpm ?? 0,
     }));
     return c.json({ models });
   });
@@ -481,6 +482,28 @@ export function adminApi(store: Store, auth: MiddlewareHandler, openai: Hono, an
     return c.json({ ok: true });
   });
 
+  // Set (or clear) the per-model even-pacing limit: `rpm` requests/min spread
+  // evenly (one every 60/rpm seconds; excess calls queue at the gateway, capped
+  // at a 60s wait). 0/blank/invalid clears. Model-wide - shared by all three
+  // route slots - and independent of the per-source Provider.rpm spill-over.
+  app.put("/models/:name/pace", async (c) => {
+    const name = c.req.param("name");
+    const body = await readJson<{ rpm?: number }>(c.req.raw);
+    const rpm = coerceRpm(body?.rpm);
+    let errStatus = 0;
+    await store.update((d) => {
+      const entry = d.models[name];
+      if (!entry) {
+        errStatus = 404;
+        return;
+      }
+      if (rpm) entry.paceRpm = rpm;
+      else delete entry.paceRpm;
+    });
+    if (errStatus === 404) return c.json({ error: { message: "model not found" } }, 404);
+    return c.json({ ok: true, paceRpm: rpm ?? 0 });
+  });
+
   // Set (or clear) the upstream-model mapping for ONE chain slot (addressed by
   // `index`). An empty `model` clears it (back to identity — send the public
   // name). Index-addressed because a provider may occupy several slots, each

package/packages/core/src/server/proxy.ts

@@ -4,8 +4,15 @@ import type { Format, Provider, RouteKey, Usage } from "../shared/types";
 import type { Store } from "./store";
 import { UsageCollector } from "./tokens";
 
-/** HTTP statuses that should trigger failover to the next provider. */
-const RETRYABLE = new Set([408, 425, 429, 500, 502, 503, 504]);
+/** HTTP statuses that should trigger failover to the next provider. 401/403
+ *  included: a banned/invalid credential (e.g. "User has been banned") is dead
+ *  for THIS source only - the same request may be fine on the next one. */
+const RETRYABLE = new Set([401, 403, 408, 425, 429, 500, 502, 503, 504]);
+
+/** Even pacing (per-model `paceRpm`) message constants. The queue itself lives
+ *  in Store.paceClaim - 60s wait horizon, one release every 60/rpm seconds. */
+const PACE_MAX_WAIT_S = 60;
+const RPM_SECONDS = 60;
 
 /** Headers copied from upstream back to the client. */
 const COPY_DOWN = ["content-type", "cache-control", "x-request-id", "openai-organization", "anthropic-ratelimit-requests-reset"];
@@ -397,6 +404,35 @@ export function proxyApi(
       if (r.entered && !isProbe) rt.warn(`proxy circuit open: provider '${p.name}' cooldown=${r.cooldownMs}ms fails=${r.fails}`);
     };
 
+    // Even pacing (per-model leaky bucket, `paceRpm`): spread the model's calls
+    // one every 60/rpm seconds - excess requests QUEUE here (bounded by a 60s
+    // wait horizon; past it they're rejected 429 in the wire's own error shape).
+    // One slot per REQUEST, claimed before the failover loop, so a call that
+    // fails over to later sources never queues twice. Independent of the
+    // per-source Provider.rpm skip (that one spills to the next source). Probes
+    // go through the same queue - they are real calls and claim real slots.
+    const paceRpm = store.get().models[model]?.paceRpm;
+    if (paceRpm) {
+      const wait = store.paceClaim(model, paceRpm);
+      if (wait < 0) {
+        const retryAfter = Math.max(1, Math.ceil(RPM_SECONDS / paceRpm));
+        const message = `rate limited: even-pacing queue for '${model}' is full (max wait ${Math.round(PACE_MAX_WAIT_S)}s; retry in ~${retryAfter}s)`;
+        if (!isProbe) rt.warn(`proxy model=${model}: paced out (429)`);
+        store.pushLog({ ts: Date.now(), model, provider: "", format: wire, status: 429, ms: Date.now() - start, stream, error: "even-pacing queue full" });
+        const headers = { "content-type": "application/json", "retry-after": String(retryAfter) };
+        if (wire === "anthropic") {
+          return c.json({ type: "error", error: { type: "rate_limit_error", message } }, 429, headers);
+        }
+        return c.json({ error: { message, type: "rate_limit_error", code: "rate_limit_exceeded" } }, 429, headers);
+      }
+      if (wait > 0) {
+        await new Promise((r) => setTimeout(r, wait));
+        // The client may have hung up while queued - release nothing upstream
+        // (the slot is already spent, but no need to burn provider quota too).
+        if (c.req.raw.signal?.aborted) return new Response(null, { status: 499 });
+      }
+    }
+
     // Skip providers that are either in circuit-breaker cooldown OR over their
     // RPM pacing cap. Both are heuristics: if every candidate is skipped, fall
     // back to the full list anyway — one real attempt beats a guaranteed 502

package/packages/core/src/server/store.ts

@@ -40,6 +40,11 @@ const RESET_CAP_MS = 6 * 60 * 60 * 1000;
  *  window. 60s matches the usual "requests per minute" limit. */
 const RPM_WINDOW_MS = 60_000;
 
+/** Even-pacing queue cap: a request whose reserved slot is further out than
+ *  this gets rejected (429) instead of queueing. Bounds both the client's hang
+ *  time and the queue depth (at N rpm / 60s wait, at most ~N requests queue). */
+const PACE_MAX_WAIT_MS = 60_000;
+
 /** Per-provider circuit state (in-memory, never persisted). */
 interface CircuitEntry {
   fails: number;
@@ -183,6 +188,9 @@ export class Store {
   /** Per-provider dispatch timestamps within the RPM pacing window. In-memory,
    *  NOT persisted (resets on restart). Pruned as `rpmUsed` reads. */
   private rpm = new Map<string, number[]>();
+  /** Per-model even-pacing queue: model name -> epoch ms of the next free
+   *  release slot. In-memory, NOT persisted (resets on restart). */
+  private pace = new Map<string, number>();
 
   constructor(dataDir: string, opts: { logger?: Logger } = {}) {
     this.dataDir = dataDir;
@@ -599,6 +607,26 @@ export class Store {
     else this.rpm.set(id, [Date.now()]);
   }
 
+  // --- even pacing (per model, in-memory leaky-bucket queue) ---
+
+  /** Reserve the next release slot for a call to `model`, paced at `rpm`
+   *  requests/min (one every 60/rpm seconds). Returns the ms the caller should
+   *  sleep BEFORE forwarding (0 = go now), or -1 when the next free slot is
+   *  further out than PACE_MAX_WAIT_MS (caller rejects with 429 - the slot is
+   *  left unclaimed so rejections never push the queue further back). Slot
+   *  claiming is synchronous, so concurrent dispatches get strictly FIFO slots;
+   *  after an idle period the stale slot is clamped to now (first request goes
+   *  through immediately). */
+  paceClaim(model: string, rpm: number): number {
+    const interval = RPM_WINDOW_MS / rpm;
+    const now = Date.now();
+    const next = Math.max(this.pace.get(model) ?? 0, now);
+    const wait = next - now;
+    if (wait > PACE_MAX_WAIT_MS) return -1;
+    this.pace.set(model, next + interval);
+    return wait;
+  }
+
   /** Snapshot of every configured provider's circuit state for GET /admin/circuit.
    *  Healthy providers appear as state "open"; a provider deleted while cooling
    *  simply drops out (we iterate the live config, not the map). */

package/packages/core/src/shared/types.ts

@@ -81,6 +81,12 @@ export interface ModelEntry {
   openai: FormatEntry;
   anthropic: FormatEntry;
   responses: FormatEntry;
+  /** Even-pacing rate limit (requests/min, leaky-bucket style). When set, calls
+   *  to this model are released one every 60/rpm seconds - excess requests QUEUE
+   *  at the gateway (up to 60s wait) instead of overflowing. Completely
+   *  independent of Provider.rpm (that one skips a busy source and fails over);
+   *  this one paces the model across all three route slots. Absent = unlimited. */
+  paceRpm?: number;
 }
 
 export interface Account {