myapikey 0.4.1 → 0.5.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "myapikey",
-  "version": "0.4.1",
+  "version": "0.5.0",
   "type": "module",
   "description": "Personal LLM API gateway & proxy — one address + one API key for all your models. Forwards OpenAI & Anthropic calls to your backends with failover and a circuit breaker. Pure passthrough, no format translation. Self-hosted (CLI + web UI).",
   "keywords": [
@@ -69,6 +69,7 @@
     "@hono/node-server": "^1.13.5",
     "@hono/zod-validator": "^0.4.2",
     "commander": "^12.1.0",
+    "gpt-tokenizer": "^3.4.0",
     "hono": "^4.6.12",
     "tsx": "^4.19.0",
     "zod": "^3.23.8"

package/packages/core/src/server/proxy.ts

@@ -1,7 +1,8 @@
 import { Hono, type Context, type MiddlewareHandler } from "hono";
 import { trimBase } from "../shared/config";
-import type { Format, Provider, RouteKey } from "../shared/types";
+import type { Format, Provider, RouteKey, Usage } from "../shared/types";
 import type { Store } from "./store";
+import { UsageCollector } from "./tokens";
 
 /** HTTP statuses that should trigger failover to the next provider. */
 const RETRYABLE = new Set([408, 425, 429, 500, 502, 503, 504]);
@@ -149,6 +150,9 @@ interface SettleInfo {
   ok: boolean;
   status: number;
   error?: string;
+  /** Token usage captured from the body as it flowed (success rows only).
+   *  Undefined for failed/truncated streams and for a body with no usage. */
+  usage?: Usage;
 }
 
 /** Wrap an upstream body so every byte is forwarded to the client VERBATIM while
@@ -200,7 +204,14 @@ function errorFrame(key: RouteKey, reason: string): string {
 
 function observedBody(
   upstream: Response,
-  opts: { stream: boolean; key: RouteKey; onSettle: (info: SettleInfo) => void },
+  opts: {
+    stream: boolean;
+    key: RouteKey;
+    /** The original request's `messages`, used only to estimate prompt tokens
+     *  on the openai-chat-stream fallback path (see tokens.ts). */
+    requestMessages?: unknown;
+    onSettle: (info: SettleInfo) => void;
+  },
 ): ReadableStream<Uint8Array> {
   const reader = upstream.body?.getReader();
   const enc = new TextEncoder();
@@ -209,6 +220,7 @@ function observedBody(
   let tail = ""; // rolling window so a marker split across chunks is still caught
   let terminal = false;
   let settled = false;
+  const usage = new UsageCollector();
 
   const settle = (info: SettleInfo) => {
     if (settled) return;
@@ -234,13 +246,14 @@ function observedBody(
             injectError(controller, reason);
             settle({ ok: false, status: 502, error: reason });
           } else {
-            settle({ ok: true, status: 200 });
+            settle({ ok: true, status: 200, usage: usage.finalize({ stream: opts.stream, key: opts.key, requestMessages: opts.requestMessages }) });
           }
           controller.close();
           return;
         }
+        const txt = dec.decode(value, { stream: true });
+        usage.feed(txt, { stream: opts.stream, key: opts.key });
         if (!terminal && markers.length) {
-          const txt = dec.decode(value, { stream: true });
           const win = tail + txt;
           if (markers.some((m) => win.includes(m))) terminal = true;
           tail = win.slice(-128);
@@ -363,13 +376,14 @@ export function proxyApi(store: Store, auth: MiddlewareHandler): Hono {
         // the body's end (so the row reflects the real outcome, not just the
         // headers). See observedBody() for the detection rules.
         const ttfb = Date.now() - start;
-        const body = observedBody(upstream, {
+        const out = observedBody(upstream, {
           stream,
           key,
+          requestMessages: body.messages,
           onSettle: (info) => {
             if (info.ok) {
               store.recordCircuitSuccess(provider.id);
-              store.pushLog({ ts: Date.now(), model, provider: provider.name, providerId: provider.id, format: wire, status: 200, ms: ttfb, stream });
+              store.pushLog({ ts: Date.now(), model, provider: provider.name, providerId: provider.id, format: wire, status: 200, ms: ttfb, stream, usage: info.usage });
             } else {
               // A pinned per-source probe takes no circuit side-effects (a manual
               // test must not trip the breaker) — mirrors the retryable branch.
@@ -378,7 +392,7 @@ export function proxyApi(store: Store, auth: MiddlewareHandler): Hono {
             }
           },
         });
-        return new Response(body, { status: upstream.status, headers: downHeaders(upstream, isProbe ? provider.name : undefined) });
+        return new Response(out, { status: upstream.status, headers: downHeaders(upstream, isProbe ? provider.name : undefined) });
       }
       if (RETRYABLE.has(upstream.status)) {
         lastStatus = upstream.status;

package/packages/core/src/server/store.ts

@@ -75,6 +75,9 @@ export interface StatBucket {
   success: number;
   error: number;
   avgMs: number;
+  /** Token totals for this bucket (success rows only). */
+  inputTokens: number;
+  outputTokens: number;
 }
 
 /** One day in the stats time series. */
@@ -98,6 +101,15 @@ export interface StatsResult {
     avgMs: number;
     p50Ms: number;
     p95Ms: number;
+    /** Token totals across the window (success rows only). cacheRead/
+     *  cacheCreation are prompt-cache hits (Anthropic) — the cached input
+     *  tokens, counted separately from `inputTokens`. A small fraction of
+     *  rows (OpenAI chat streams where the upstream omitted usage) contribute
+     *  local tokenizer ESTIMATES rather than billed counts. */
+    inputTokens: number;
+    outputTokens: number;
+    cacheRead: number;
+    cacheCreation: number;
   };
   byModel: StatBucket[];
   byProvider: StatBucket[];
@@ -327,7 +339,7 @@ export class Store {
     const empty: StatsResult = {
       from,
       to,
-      totals: { calls: 0, success: 0, error: 0, errorRate: 0, avgMs: 0, p50Ms: 0, p95Ms: 0 },
+      totals: { calls: 0, success: 0, error: 0, errorRate: 0, avgMs: 0, p50Ms: 0, p95Ms: 0, inputTokens: 0, outputTokens: 0, cacheRead: 0, cacheCreation: 0 },
       byModel: [],
       byProvider: [],
       byFormat: [],
@@ -358,12 +370,12 @@ export class Store {
       const ok = e.status >= 200 && e.status < 300;
       const err = e.status >= 400;
       const ms = e.ms || 0;
-      bump(tot, ok, err, ms);
+      bump(tot, ok, err, e);
       latencies.push(ms);
-      bump(acc(model, e.model), ok, err, ms);
-      bump(acc(provider, e.providerId ?? e.provider ?? "?"), ok, err, ms);
-      bump(acc(format, e.format ?? "?"), ok, err, ms);
-      bump(acc(day, dayKey(e.ts)), ok, err, ms);
+      bump(acc(model, e.model), ok, err, e);
+      bump(acc(provider, e.providerId ?? e.provider ?? "?"), ok, err, e);
+      bump(acc(format, e.format ?? "?"), ok, err, e);
+      bump(acc(day, dayKey(e.ts)), ok, err, e);
     }
 
     latencies.sort((a, b) => a - b);
@@ -398,6 +410,10 @@ export class Store {
         avgMs: tot.calls ? Math.round(tot.sumMs / tot.calls) : 0,
         p50Ms: pick(0.5),
         p95Ms: pick(0.95),
+        inputTokens: tot.sumInput,
+        outputTokens: tot.sumOutput,
+        cacheRead: tot.sumCacheRead,
+        cacheCreation: tot.sumCacheCreation,
       },
       byModel: [...model].map(([k, a]) => ({ key: k, ...fields(a) })).sort(sortDesc),
       byProvider: [...provider]
@@ -523,15 +539,26 @@ interface Acc {
   success: number;
   error: number;
   sumMs: number;
+  sumInput: number;
+  sumOutput: number;
+  sumCacheRead: number;
+  sumCacheCreation: number;
 }
 function newAcc(): Acc {
-  return { calls: 0, success: 0, error: 0, sumMs: 0 };
+  return { calls: 0, success: 0, error: 0, sumMs: 0, sumInput: 0, sumOutput: 0, sumCacheRead: 0, sumCacheCreation: 0 };
 }
-function bump(a: Acc, ok: boolean, err: boolean, ms: number): void {
+function bump(a: Acc, ok: boolean, err: boolean, e: LogEntry): void {
   a.calls++;
   if (ok) a.success++;
   if (err) a.error++;
-  a.sumMs += ms;
+  a.sumMs += e.ms || 0;
+  const u = e.usage;
+  if (u) {
+    a.sumInput += u.input || 0;
+    a.sumOutput += u.output || 0;
+    a.sumCacheRead += u.cacheRead || 0;
+    a.sumCacheCreation += u.cacheCreation || 0;
+  }
 }
 /** Get-or-create a bucket entry in a stats map. */
 function acc(m: Map<string, Acc>, k: string): Acc {
@@ -544,8 +571,15 @@ function dayKey(ts: number): string {
   const d = new Date(ts);
   return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}-${String(d.getDate()).padStart(2, "0")}`;
 }
-function fields(a: Acc): Pick<StatBucket, "calls" | "success" | "error" | "avgMs"> {
-  return { calls: a.calls, success: a.success, error: a.error, avgMs: a.calls ? Math.round(a.sumMs / a.calls) : 0 };
+function fields(a: Acc): Pick<StatBucket, "calls" | "success" | "error" | "avgMs" | "inputTokens" | "outputTokens"> {
+  return {
+    calls: a.calls,
+    success: a.success,
+    error: a.error,
+    avgMs: a.calls ? Math.round(a.sumMs / a.calls) : 0,
+    inputTokens: a.sumInput,
+    outputTokens: a.sumOutput,
+  };
 }
 
 /**

package/packages/core/src/server/tokens.ts

@@ -0,0 +1,210 @@
+/** Token-usage capture for the proxy's stream bucket.
+ *
+ *  `observedBody()` in proxy.ts already wraps the upstream body in a
+ *  `ReadableStream` that forwards every byte to the client VERBATIM while
+ *  watching for clean termination. This module hangs a second tap off that same
+ *  loop: as each decoded chunk flows past, a `UsageCollector` extracts token
+ *  usage out of band. The bytes themselves are never held back or altered.
+ *
+ *  Strategy is HYBRID, by design (see shared/types.ts `Usage`):
+ *  - UPSTREAM-REPORTED usage wins wherever the backend sends it — that's the
+ *    exact billed count. Anthropic streams (message_start/message_delta), every
+ *    non-streaming JSON body, and /responses (response.completed) all carry it.
+ *  - OpenAI /chat/completions STREAMS usually omit usage (most agents don't set
+ *    stream_options.include_usage). For THAT one wire we fall back to a local
+ *    tokenizer estimate (gpt-tokenizer, o200k_base) of the request messages
+ *    (prompt) + the accumulated completion text, flagged `estimated: true`. */
+import { countTokens as bpeCountTokens } from "gpt-tokenizer";
+import type { RouteKey, Usage } from "../shared/types";
+
+/** Count tokens with gpt-tokenizer's default o200k_base encoding (gpt-4o /
+ *  gpt-4.1 / o1 / …). Used ONLY for the OpenAI chat streaming fallback estimate;
+ *  everywhere else the upstream's exact usage is used. o200k_base is a fair ≈
+ *  for modern OpenAI-family models and rougher for non-OpenAI OpenAI-compatible
+ *  backends (DeepSeek/Qwen/Ark) — which is why these rows are marked estimated.
+ *  Never throws: a tokenizer hiccup must not break the passthrough. */
+export function countTokens(text: string): number {
+  if (!text) return 0;
+  try {
+    return bpeCountTokens(text);
+  } catch {
+    return 0;
+  }
+}
+
+/** Rough local estimate of the PROMPT token count for an OpenAI chat request,
+ *  used only on the streaming fallback path. Sums the token counts of every
+ *  string `content` (and multi-part `text` blocks) across the messages array —
+ *  non-text parts (images, tool I/O) are undercounted, acceptable for an
+ *  estimate marked ≈. */
+export function estimatePromptTokens(messages: unknown): number {
+  if (!Array.isArray(messages)) return 0;
+  let n = 0;
+  for (const m of messages) {
+    if (!m || typeof m !== "object") continue;
+    const content = (m as { content?: unknown }).content;
+    if (typeof content === "string") {
+      n += countTokens(content);
+    } else if (Array.isArray(content)) {
+      for (const part of content) {
+        const text = (part as { text?: unknown } | null)?.text;
+        if (typeof text === "string") n += countTokens(text);
+      }
+    }
+  }
+  return n;
+}
+
+/** Coerce a JSON value to a finite non-negative integer, or undefined. */
+function num(v: unknown): number | undefined {
+  return typeof v === "number" && Number.isFinite(v) && v >= 0 ? Math.trunc(v) : undefined;
+}
+
+type UsageFields = Partial<Pick<Usage, "input" | "output" | "cacheRead" | "cacheCreation">>;
+
+/** Pull whatever usage fields are present out of one parsed SSE `data:` payload
+ *  (streaming) or the whole JSON body (non-streaming). The collector merges
+ *  these across events per-field (last writer wins) — each protocol's terminal
+ *  usage is authoritative:
+ *  - anthropic: `message_start.message.usage` (input + cache) and the running
+ *    `message_delta.usage.output_tokens` (output); a non-streaming Message
+ *    carries `usage` directly (both).
+ *  - openai chat: only the final chunk (with stream_options.include_usage) has
+ *    `usage` (prompt/completion tokens); the non-streaming body has it too.
+ *  - responses: `response.usage` on response.completed/in-progress, or top-level
+ *    `usage` on a non-streaming Response. Accepts both the `*_tokens` and bare
+ *    `input`/`output` spellings the API has used over time. */
+function extractUsage(obj: any, key: RouteKey): UsageFields | null {
+  if (!obj || typeof obj !== "object") return null;
+
+  if (key === "anthropic") {
+    const u = obj.message?.usage ?? obj.usage;
+    if (u && (typeof u.input_tokens === "number" || typeof u.output_tokens === "number")) {
+      return {
+        input: num(u.input_tokens),
+        output: num(u.output_tokens),
+        cacheRead: num(u.cache_read_input_tokens),
+        cacheCreation: num(u.cache_creation_input_tokens),
+      };
+    }
+    return null;
+  }
+  if (key === "responses") {
+    const u = obj.response?.usage ?? obj.usage;
+    if (u) {
+      const input = num(u.input_tokens) ?? num(u.input);
+      const output = num(u.output_tokens) ?? num(u.output);
+      if (typeof input === "number" || typeof output === "number") return { input, output };
+    }
+    return null;
+  }
+  // openai /chat/completions
+  if (obj.usage) {
+    return { input: num(obj.usage.prompt_tokens), output: num(obj.usage.completion_tokens) };
+  }
+  return null;
+}
+
+/** Accumulates usage out of the bytes flowing through the stream bucket. Feed it
+ *  every decoded chunk as it passes; call `finalize` once at stream end. Bounded
+ *  memory: streaming holds only the partial line currently being assembled plus
+ *  (openai-chat only) the accumulated completion text — which is the assistant
+ *  message itself, no larger than what the client already receives. */
+export class UsageCollector {
+  private partial = ""; // streaming: the in-progress line without a trailing \n
+  private buf = ""; // non-streaming: the whole body, parsed once at finalize
+  private input?: number;
+  private output?: number;
+  private cacheRead?: number;
+  private cacheCreation?: number;
+  /** True once ANY upstream-reported usage has been seen — selects the exact
+   *  path in finalize() over the local-estimate fallback. */
+  private upstream = false;
+  /** openai-chat-stream only: concatenated `choices[].delta.content`, for the
+   *  local completion-token estimate when the upstream omits usage. */
+  private completionText = "";
+
+  feed(text: string, opts: { stream: boolean; key: RouteKey }): void {
+    if (!opts.stream) {
+      this.buf += text;
+      return;
+    }
+    // Streaming SSE: parse complete lines now, keep the tail partial.
+    this.partial += text;
+    let nl: number;
+    while ((nl = this.partial.indexOf("\n")) >= 0) {
+      const line = this.partial.slice(0, nl);
+      this.partial = this.partial.slice(nl + 1);
+      this.parseLine(line, opts.key);
+    }
+  }
+
+  private parseLine(line: string, key: RouteKey): void {
+    const colon = line.indexOf(":");
+    if (colon < 0) return;
+    if (line.slice(0, colon).trim() !== "data") return; // only `data:` lines carry JSON
+    const payload = line.slice(colon + 1).trimStart();
+    if (!payload || payload === "[DONE]") return;
+    let obj: any;
+    try {
+      obj = JSON.parse(payload);
+    } catch {
+      return; // a non-JSON data line (e.g. a backend's bespoke marker) — ignore
+    }
+    const u = extractUsage(obj, key);
+    if (u) {
+      this.upstream = true;
+      if (typeof u.input === "number") this.input = u.input;
+      if (typeof u.output === "number") this.output = u.output;
+      if (typeof u.cacheRead === "number") this.cacheRead = u.cacheRead;
+      if (typeof u.cacheCreation === "number") this.cacheCreation = u.cacheCreation;
+    }
+    // Accumulate completion text for the openai-chat-stream estimate fallback.
+    // Stop once the upstream reported usage (the terminal chunk) — there's
+    // nothing after it and no estimate will be needed.
+    if (key === "openai" && !this.upstream && Array.isArray(obj.choices)) {
+      for (const ch of obj.choices) {
+        const c = ch?.delta?.content;
+        if (typeof c === "string") this.completionText += c;
+      }
+    }
+  }
+
+  /** Resolve the final usage (or undefined if none could be determined). Call
+   *  once, at stream end. `requestMessages` is the original chat request's
+   *  `messages` — used only to estimate prompt tokens on the openai-chat-stream
+   *  fallback path. */
+  finalize(opts: { stream: boolean; key: RouteKey; requestMessages?: unknown }): Usage | undefined {
+    if (!opts.stream) {
+      // Non-streaming: parse the whole JSON body once.
+      let obj: any;
+      try {
+        obj = JSON.parse(this.buf);
+      } catch {
+        return undefined;
+      }
+      const u = extractUsage(obj, opts.key);
+      if (!u) return undefined;
+      return { input: u.input ?? 0, output: u.output ?? 0, cacheRead: u.cacheRead, cacheCreation: u.cacheCreation };
+    }
+    // Flush any trailing line that had no trailing newline.
+    if (this.partial.trim()) this.parseLine(this.partial, opts.key);
+    if (this.upstream) {
+      return {
+        input: this.input ?? 0,
+        output: this.output ?? 0,
+        cacheRead: this.cacheRead,
+        cacheCreation: this.cacheCreation,
+      };
+    }
+    // Fallback: local estimate, ONLY for the openai chat streaming path (the one
+    // wire whose streams routinely omit usage). Anthropic / /responses streams
+    // always report usage, so they never reach here.
+    if (opts.key === "openai") {
+      const out = countTokens(this.completionText);
+      const inp = estimatePromptTokens(opts.requestMessages);
+      if (out || inp) return { input: inp, output: out, estimated: true };
+    }
+    return undefined;
+  }
+}

package/packages/core/src/shared/types.ts

@@ -86,6 +86,22 @@ export interface GateConfig {
   models: Record<string, ModelEntry>;
 }
 
+/** Token usage for a single call. Captured from the upstream's reported usage
+ *  when available — that's the exact billed count (Anthropic streams, non-
+ *  streaming bodies, /responses all carry it). For OpenAI /chat/completions
+ *  streams where the upstream omits usage (most agents don't set
+ *  stream_options.include_usage), `estimated` is set and input/output come from
+ *  a local tokenizer approximation (gpt-tokenizer, o200k_base) instead — the UI
+ *  renders those with a ≈ marker. cacheRead/cacheCreation (prompt-caching hits,
+ *  Anthropic-only) are surfaced separately from `input`. */
+export interface Usage {
+  input: number;
+  output: number;
+  cacheRead?: number;
+  cacheCreation?: number;
+  estimated?: boolean;
+}
+
 /** One persisted call-history entry (stored in logs.jsonl, one JSON object per line). */
 export interface LogEntry {
   ts: number;
@@ -101,6 +117,10 @@ export interface LogEntry {
   ms: number;
   /** Whether the request asked for streaming. */
   stream: boolean;
+  /** Token usage for this call (success rows only). Absent when the upstream
+   *  reported none AND no local estimate was possible (e.g. a failed/truncated
+   *  stream), or on legacy log lines written before usage tracking. */
+  usage?: Usage;
   /** Short upstream error text on non-2xx (omitted on success). */
   error?: string;
   /** Row kind. Absent on legacy lines → treated as a normal call. "cooldown"