cachegate 1.3.1 → 1.4.0

package/providers/openrouter.js

@@ -0,0 +1,233 @@
+// model-router/providers/openrouter.js
+//
+// OpenRouter is an aggregator: one OpenAI-compatible endpoint in front of many
+// vendors, addressed by `vendor/model` ids (e.g. `deepseek/deepseek-chat`,
+// `anthropic/claude-sonnet-4.5`). Two things make it different from the direct
+// providers in this directory:
+//
+//   1. ATTRIBUTION HEADERS. Requests may carry HTTP-Referer and X-Title so the
+//      app shows up correctly in OpenRouter's own dashboard. They are optional
+//      on the wire, so they are only sent when configured - inventing a
+//      referer for someone's deployment would be worse than omitting it.
+//
+//   2. PRICING CANNOT BE A CONSTANT TABLE HERE. providers/openai.js hardcodes
+//      two rates and providers/deepseek.js a handful, which is defensible for a
+//      vendor with a handful of models. OpenRouter fronts hundreds, and their
+//      prices change without a release of this project - a hardcoded table
+//      would rot into wrong money silently, which is the one failure mode this
+//      project's cost tracking exists to prevent. So the catalog is fetched
+//      from OpenRouter's own /models endpoint and cached in-process.
+//
+//      WHEN PRICING IS UNKNOWN, cost is null - NEVER 0. A zero would be read as
+//      "free" by anything that sorts candidates by cost, and this router's whole
+//      pitch is routing to the cheapest healthy provider; a silent zero would
+//      route everything to whichever model we failed to price. Callers must
+//      treat null as "unknown, do not compare" (see providers/index.js).
+const { OpenAI } = require('openai');
+
+const BASE_URL = process.env.OPENROUTER_BASE_URL || 'https://openrouter.ai/api/v1';
+const MODELS_PATH = '/models';
+const PRICING_TTL_MS = Number(process.env.OPENROUTER_PRICING_TTL_MS) || 6 * 60 * 60 * 1000;
+
+// model id -> { input, output } in USD per 1M tokens. Populated by
+// refreshPricing() (network) or setPricingTable() (tests / a pinned table).
+let pricingTable = null;
+let pricingFetchedAt = 0;
+
+/**
+ * OpenRouter's documented optional attribution headers, from the environment.
+ * A pure function so the "only send what was configured" rule is testable
+ * without constructing an SDK client.
+ */
+function attributionHeaders(env = process.env) {
+  const headers = {};
+  if (env.OPENROUTER_SITE_URL) headers['HTTP-Referer'] = env.OPENROUTER_SITE_URL;
+  if (env.OPENROUTER_SITE_NAME) headers['X-Title'] = env.OPENROUTER_SITE_NAME;
+  return headers;
+}
+
+function buildClient(apiKey) {
+  const headers = attributionHeaders();
+
+  return new OpenAI({
+    apiKey,
+    baseURL: BASE_URL,
+    defaultHeaders: Object.keys(headers).length ? headers : undefined,
+    timeout: Number(process.env.OPENROUTER_TIMEOUT_MS) || 60000
+  });
+}
+
+// `vendor/model`, optionally written as `openrouter/vendor/model` so a caller
+// can be explicit. Anything else is not an OpenRouter id.
+function isOpenRouterModel(model) {
+  if (typeof model !== 'string') return false;
+  if (model.startsWith('openrouter/')) return true;
+  return /^[a-z0-9][a-z0-9._-]*\/[A-Za-z0-9][A-Za-z0-9._:-]*$/.test(model);
+}
+
+function normalizeModel(model) {
+  return typeof model === 'string' && model.startsWith('openrouter/') ? model.slice('openrouter/'.length) : model;
+}
+
+/** OpenRouter reports per-token prices as strings; convert to per-1M USD. */
+function toPerMillion(value) {
+  const n = Number(value);
+  return Number.isFinite(n) ? n * 1_000_000 : null;
+}
+
+function setPricingTable(entries) {
+  // setPricingTable(null) resets to "price unknown" - what a test, or a caller
+  // forcing a refresh, needs. An empty object would instead be a loaded-but-
+  // empty table, which reads as "every model is unpriced" rather than "not
+  // loaded yet" and would make the null-vs-zero distinction untestable.
+  if (!entries) { pricingTable = null; pricingFetchedAt = 0; return null; }
+  const table = {};
+  for (const e of entries || []) {
+    const id = e && (e.id || e.model);
+    if (!id || !e.pricing) continue;
+    const input = toPerMillion(e.pricing.prompt ?? e.pricing.input);
+    const output = toPerMillion(e.pricing.completion ?? e.pricing.output);
+    if (input === null) continue;
+    table[id] = { input, output: output === null ? input : output };
+  }
+  pricingTable = table;
+  pricingFetchedAt = Date.now();
+  return table;
+}
+
+async function refreshPricing({ fetchImpl, force = false } = {}) {
+  if (!force && pricingTable && (Date.now() - pricingFetchedAt) < PRICING_TTL_MS) return pricingTable;
+  const doFetch = fetchImpl || globalThis.fetch;
+  if (typeof doFetch !== 'function') return pricingTable;
+  try {
+    const res = await doFetch(`${BASE_URL}${MODELS_PATH}`);
+    if (!res || !res.ok) return pricingTable;
+    const body = await res.json();
+    setPricingTable(body && body.data);
+  } catch {
+    // A pricing fetch failure must never break dispatch - the request itself
+    // does not need prices. Cost comes back null and is reported as unknown.
+  }
+  return pricingTable;
+}
+
+/**
+ * USD for one call, or null when the model's price is not known yet.
+ * (null, not 0 - see the header.)
+ */
+function estimateCost(model, inputTokens, outputTokens) {
+  if (!pricingTable) return null;
+  const rate = pricingTable[normalizeModel(model)];
+  if (!rate) return null;
+  return (((inputTokens || 0) * rate.input) + ((outputTokens || 0) * rate.output)) / 1_000_000;
+}
+
+function usageFrom(usage = {}) {
+  return {
+    input_tokens: usage.prompt_tokens || 0,
+    output_tokens: usage.completion_tokens || 0,
+    // Some upstreams behind OpenRouter report their own cache tiers. Passed
+    // through when present, never invented when absent.
+    ...(typeof usage.prompt_tokens_details?.cached_tokens === 'number'
+      && { cached_input_tokens: usage.prompt_tokens_details.cached_tokens })
+  };
+}
+
+async function chat(client, payload, options = {}) {
+  const request = {
+    model: normalizeModel(payload.model),
+    messages: payload.messages,
+    temperature: typeof payload.temperature === 'number' ? payload.temperature : 0.0,
+    max_tokens: payload.max_tokens || 1024,
+    ...(payload.tools && { tools: payload.tools }),
+    ...(payload.tool_choice && { tool_choice: payload.tool_choice }),
+    ...(payload.response_format && { response_format: payload.response_format }),
+    // Ask for the upstream provider's identity + cost accounting in the same
+    // response, so the dashboard can attribute spend per vendor.
+    usage: { include: true },
+    ...(options.requestLogprobs && { logprobs: true, top_logprobs: 1 })
+  };
+
+  const start = Date.now();
+  const response = await client.chat.completions.create(request);
+  const latencyMs = Date.now() - start;
+
+  const choice = response.choices[0];
+  const usage = usageFrom(response.usage);
+  // Prefer OpenRouter's own accounting when it sends it: it is the billed
+  // amount, including any provider-specific pricing we did not model.
+  const billed = typeof response.usage?.cost === 'number' ? response.usage.cost : null;
+  const costUsd = billed !== null ? billed : estimateCost(payload.model, usage.input_tokens, usage.output_tokens);
+
+  return {
+    provider: 'openrouter',
+    model: payload.model,
+    latency_ms: latencyMs,
+    usage,
+    cost_usd: costUsd,
+    content: choice.message.content || '',
+    tool_calls: choice.message.tool_calls,
+    // Which upstream actually served it (OpenRouter routes among several).
+    upstream_provider: response.provider || undefined,
+    raw: response
+  };
+}
+
+function applyStreamChunk(state, chunk, onDelta) {
+  const choice = chunk.choices && chunk.choices[0];
+  if (choice && choice.delta && choice.delta.content) {
+    state.content += choice.delta.content;
+    onDelta(choice.delta.content);
+  }
+  if (chunk.usage) {
+    state.inputTokens = chunk.usage.prompt_tokens || 0;
+    state.outputTokens = chunk.usage.completion_tokens || 0;
+    if (typeof chunk.usage.cost === 'number') state.billedCost = chunk.usage.cost;
+    if (typeof chunk.usage.prompt_tokens_details?.cached_tokens === 'number') {
+      state.cachedInputTokens = chunk.usage.prompt_tokens_details.cached_tokens;
+    }
+  }
+}
+
+async function chatStream(client, payload, { onDelta, signal } = {}) {
+  const request = {
+    model: normalizeModel(payload.model),
+    messages: payload.messages,
+    temperature: typeof payload.temperature === 'number' ? payload.temperature : 0.0,
+    max_tokens: payload.max_tokens || 1024,
+    stream: true,
+    stream_options: { include_usage: true },
+    usage: { include: true }
+  };
+
+  const start = Date.now();
+  const stream = await client.chat.completions.create(request, signal ? { signal } : undefined);
+
+  const state = { content: '', inputTokens: 0, outputTokens: 0, billedCost: null, cachedInputTokens: 0 };
+  for await (const chunk of stream) {
+    applyStreamChunk(state, chunk, onDelta || (() => {}));
+  }
+
+  const latencyMs = Date.now() - start;
+  const usage = usageFrom({
+    prompt_tokens: state.inputTokens,
+    completion_tokens: state.outputTokens,
+    ...(state.cachedInputTokens ? { prompt_tokens_details: { cached_tokens: state.cachedInputTokens } } : {})
+  });
+  const costUsd = state.billedCost !== null ? state.billedCost : estimateCost(payload.model, usage.input_tokens, usage.output_tokens);
+
+  return {
+    provider: 'openrouter',
+    model: payload.model,
+    latency_ms: latencyMs,
+    usage,
+    cost_usd: costUsd,
+    content: state.content,
+    tool_calls: undefined
+  };
+}
+
+module.exports = {
+  buildClient, chat, chatStream, applyStreamChunk, estimateCost,
+  isOpenRouterModel, normalizeModel, refreshPricing, setPricingTable, attributionHeaders, BASE_URL
+};

package/public/dashboard.html

@@ -594,6 +594,7 @@
   function renderKpis(data) {
     var tiles = [
       { label: 'Total cost (' + data.days + 'd)', value: formatUsd(data.total_cost_usd), spark: data.daily.map(function (d) { return d.cost_usd; }) },
+      { label: 'Saved (' + data.days + 'd)', value: formatUsd(data.saved_usd), spark: data.daily.map(function (d) { return d.saved_usd; }) },
       { label: 'Total requests', value: formatCount(data.sample_size) },
       { label: 'Exact hit rate', value: formatPercent(data.cache_hit_rate.exact) },
       { label: 'Semantic hit rate', value: formatPercent(data.cache_hit_rate.semantic) },

package/router.js

@@ -147,6 +147,16 @@ const COMPARISON_OUTPUT_TOKENS = 500;
 // (in which case we still have to pick one - see pickCandidate).
 const UNHEALTHY_ERROR_RATE = 0.5;
 
+// Step 33 (health scoring + circuit breakers): a second, STRICTER tier
+// above "unhealthy" - "shed". A candidate at or above SHED_ERROR_RATE is
+// clearly down (not just having a rough patch), so it's fully excluded
+// this request rather than merely deprioritized. avgQualityScore (step 32)
+// is a second, independent confirming signal: a candidate that keeps
+// needing failover to nominally "succeed" sheds even before its raw
+// error-rate count crosses the threshold. Both thresholds are env-tunable.
+const SHED_ERROR_RATE = Number(process.env.ROUTER_SHED_ERROR_RATE) || 0.9;
+const SHED_QUALITY_SCORE = Number(process.env.ROUTER_SHED_QUALITY_SCORE) || 0.2;
+
 // Minimum number of recent requests before a provider's error rate is
 // treated as meaningful. Without this, a brand-new provider (or one whose
 // traffic just resumed) with a SINGLE request that happened to error has
@@ -205,18 +215,36 @@ async function pickCandidate(virtualModel, scope) {
     const estimatedCostUsd = estimate
       ? estimate(candidate.model, COMPARISON_INPUT_TOKENS, COMPARISON_OUTPUT_TOKENS)
       : Infinity;
-    const providerStat = stats[candidate.provider] || { errorRate: 0, avgLatencyMs: null, sampleSize: 0 };
+    const providerStat = stats[candidate.provider] || { errorRate: 0, avgLatencyMs: null, avgQualityScore: null, sampleSize: 0 };
+    const hasEnoughSamples = providerStat.sampleSize >= MIN_HEALTH_SAMPLES;
+    // shed = clearly down (33.1): high error rate, OR low avgQualityScore
+    // (keeps failing over to "succeed"). avgQualityScore is null when there
+    // is no quality data yet - insufficient data must never shed.
+    const shed =
+      hasEnoughSamples &&
+      (providerStat.errorRate >= SHED_ERROR_RATE ||
+        (typeof providerStat.avgQualityScore === 'number' && providerStat.avgQualityScore < SHED_QUALITY_SCORE));
     return {
       ...candidate,
       estimatedCostUsd,
       errorRate: providerStat.errorRate,
       avgLatencyMs: providerStat.avgLatencyMs,
-      healthy: providerStat.sampleSize < MIN_HEALTH_SAMPLES || providerStat.errorRate < UNHEALTHY_ERROR_RATE
+      avgQualityScore: providerStat.avgQualityScore,
+      shed,
+      healthy: !hasEnoughSamples || providerStat.errorRate < UNHEALTHY_ERROR_RATE
     };
   });
 
-  const healthy = scored.filter((c) => c.healthy);
-  const pool = healthy.length > 0 ? healthy : scored; // all unhealthy: pick the least-bad rather than fail outright
+  // 33.1: shed the clearly-down candidates first (stricter than unhealthy).
+  const notShed = scored.filter((c) => !c.shed);
+  // 33.2: all-down fallback - if every candidate is shed, fall back to the
+  // full list and try anyway (least-bad rather than fail outright), the
+  // same philosophy as the all-unhealthy fallback below.
+  const afterShed = notShed.length > 0 ? notShed : scored;
+  // Existing unhealthy filter (a rough patch, 0.5 <= errorRate < 0.9) on
+  // what remains after shedding.
+  const healthy = afterShed.filter((c) => c.healthy);
+  const pool = healthy.length > 0 ? healthy : afterShed; // all unhealthy: pick the least-bad rather than fail outright
 
   const strategy = loadStrategy();
   let ranked;
@@ -245,6 +273,7 @@ async function pickCandidate(virtualModel, scope) {
       consideredTier: virtualModel,
       strategy,
       candidates: scored,
+      shedExcludedACandidate: notShed.length < scored.length,
       allUnhealthy: healthy.length === 0,
       latencyGuardExcludedACandidate: guardApplied
     }

package/semanticCache.js

@@ -20,6 +20,19 @@
 // meant to scale past that cap. A real vector index is the honest next
 // step if traffic outgrows it.
 //
+// What that brute force actually COSTS was worth measuring rather than
+// assuming, and it was not the arithmetic: each entry used to carry its
+// embedding as JSON text, so a 1536-dim vector arrived as ~30 KB of
+// digits and a lookup parsed up to 200 of them - 5.89 MB of JSON
+// number-parsing per semantic miss, against 200 x 1536 multiply-adds to
+// score them (48.7 ms, measured). The vector is now stored as base64
+// Float32 with its L2 norm beside it: 1.58 MB and 3.6 ms for the same
+// work, 13x faster. Entries written before this change are still read
+// correctly - both formats live in the same list until the old ones age
+// out - and the cap, the threshold and the honesty framing above are
+// unchanged. This is an encoding improvement, NOT the vector index the
+// last paragraph is still waiting for.
+//
 // The threshold is a probabilistic judgment call, not a guarantee: a
 // "hit" above the threshold is the router's best guess that two
 // prompts want the same answer, not proof they do. Set it too low and
@@ -32,6 +45,15 @@
 
 const redis = require('./redisClient');
 const embeddingsDefault = require('./embeddings');
+// The answer-shape definition is shared with the exact cache ON PURPOSE: the two paths diverged once -
+// cache.js keyed response_format and this file never did - and a field added to one of them silently
+// stops being enforced by the other. No cycle: cache.js does not require this file.
+const { shapeFields } = require('./cache');
+
+// The shape of a request that demands nothing about the answer's form: no response_format, no tools,
+// no tool_choice, no seed. The semantic shape gate uses it to decide whether an entry carrying no
+// recorded shape may still be served - see the gate in findMatch.
+const SHAPE_OF_NEUTRAL = JSON.stringify(shapeFields({}));
 
 const MAX_CANDIDATES_PER_MODEL = Number(process.env.SEMANTIC_CACHE_MAX_CANDIDATES) || 200;
 const DEFAULT_TTL_SECONDS = Number(process.env.SEMANTIC_CACHE_TTL_SECONDS) || 3600;
@@ -45,6 +67,44 @@ function listKey(scope, model) {
   return scope != null ? `SEMANTIC_LIST:${scope}:${model}` : `SEMANTIC_LIST:${model}`;
 }
 
+// Embeddings are stored as base64 Float32 rather than JSON numbers. Float32 loses ~7 significant digits
+// versus the JSON doubles the old format held, and cosine similarity over 1536 dimensions cannot see the
+// difference - test/semanticCache.test.js asserts both paths agree, and the legacy branch below is what
+// keeps entries written before this change matching correctly.
+function encodeEmbedding(vec) {
+  const f = Float32Array.from(vec);
+  return Buffer.from(f.buffer, f.byteOffset, f.byteLength).toString('base64');
+}
+
+function decodeEmbedding(record) {
+  const e = record && record.embedding;
+  if (Array.isArray(e)) return e;                       // legacy: JSON doubles
+  if (typeof e !== 'string') return null;
+  const buf = Buffer.from(e, 'base64');
+  // A Float32 vector is 4 bytes per element; anything else is not one. Checked rather than assumed so a
+  // corrupt or truncated entry is skipped (as a malformed JSON entry already was) instead of producing a
+  // wrong-length array that silently scores 0 against everything.
+  if (buf.length === 0 || buf.length % 4 !== 0) return null;
+  return new Float32Array(buf.buffer, buf.byteOffset, buf.length / 4);
+}
+
+function normOf(v) {
+  let n = 0;
+  for (let i = 0; i < v.length; i++) n += v[i] * v[i];
+  return Math.sqrt(n);
+}
+
+// Same result as cosineSimilarity(), given the two norms the caller already has or has stored. The stored
+// norm is the point: without it, every lookup recomputes the norm of every candidate - a third of the float
+// work per candidate, for a value that never changes.
+function similarityWithNorms(a, aNorm, b, bNorm) {
+  if (!a || !b || a.length !== b.length) return 0;
+  if (aNorm === 0 || bNorm === 0) return 0;
+  let dot = 0;
+  for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
+  return dot / (aNorm * bNorm);
+}
+
 function cosineSimilarity(a, b) {
   if (!a || !b || a.length !== b.length) return 0;
   let dot = 0;
@@ -79,6 +139,11 @@ function isEnabled(embeddings = embeddingsDefault) {
 // approximate text match can't guarantee the exact argument values a
 // tool call needs, and returning a plausible-but-wrong tool call is a
 // worse failure than a cache miss.
+//
+// response_format is NOT excluded, it is FILTERED (see findMatch): the
+// answer is still worth caching, it just may not be served to a caller
+// who asked for a different shape. Excluding it outright would throw away
+// hit rate to solve a matching problem.
 function isCacheable(payload) {
   return !payload.tools;
 }
@@ -86,6 +151,11 @@ function isCacheable(payload) {
 async function findMatch(scope, payload, { threshold = DEFAULT_THRESHOLD, embeddings = embeddingsDefault } = {}) {
   if (!isEnabled(embeddings) || !isCacheable(payload)) return null;
 
+  // The shape this request needs. A semantic match is approximate about the PROMPT, never about the
+  // answer's shape: a json_object caller served a cached plain-text answer fails to parse it, which
+  // surfaces as an upstream outage rather than a cache miss. Same reasoning that excludes tool calls.
+  const shape = JSON.stringify(shapeFields(payload));
+
   let queryEmbedding;
   try {
     queryEmbedding = await embeddings.embed(extractPromptText(payload));
@@ -102,7 +172,9 @@ async function findMatch(scope, payload, { threshold = DEFAULT_THRESHOLD, embedd
     return null;
   }
 
+  const queryNorm = normOf(queryEmbedding);
   let best = null;
+  let shapeSkipped = 0;
   for (const line of raw) {
     let record;
     try {
@@ -110,11 +182,36 @@ async function findMatch(scope, payload, { threshold = DEFAULT_THRESHOLD, embedd
     } catch {
       continue; // a malformed entry is skipped, not fatal
     }
-    const similarity = cosineSimilarity(queryEmbedding, record.embedding);
+    // Shape gate. Refuse a candidate that cannot PROVE the shape matches.
+    //
+    // The asymmetry is deliberate, and it reconciles two requirements that both have to hold:
+    //   * A request that demands nothing about the answer's shape can consume either form, so an entry
+    //     with no recorded shape - stored before this gate existed - is still served. That is what
+    //     keeps "an upgrade is not a cache flush" true for the plain-prose case, which is the vast
+    //     majority of traffic.
+    //   * A request that DOES demand a shape may only be served by an entry that proves it matches.
+    //     Prose handed to a json_object caller fails to parse in the caller and surfaces as an upstream
+    //     outage - worse than a miss - and that direction is the entire reason this gate exists.
+    // A recorded shape that differs is refused in both directions: the reverse (json handed to a prose
+    // caller) does not crash anything, but it is still the wrong answer to the question asked.
+    const shapeNeutral = shape === SHAPE_OF_NEUTRAL;
+    if (!(record.shape === shape || (record.shape === undefined && shapeNeutral))) {
+      shapeSkipped += 1;
+      continue;
+    }
+    const vector = decodeEmbedding(record);
+    if (!vector) continue; // ditto an entry whose vector cannot be read
+    const candidateNorm = typeof record.norm === 'number' ? record.norm : normOf(vector);
+    const similarity = similarityWithNorms(queryEmbedding, queryNorm, vector, candidateNorm);
     if (similarity >= threshold && (!best || similarity > best.similarity)) {
       best = { entry: record.entry, similarity };
     }
   }
+  if (!best && shapeSkipped > 0) {
+    // Visible, because "the semantic cache stopped hitting" otherwise looks like a tuning problem
+    // rather than the shape gate doing its job.
+    console.log(`[semantic] ${shapeSkipped} candidate(s) skipped: answer shape differs from this request`);
+  }
   return best;
 }
 
@@ -130,8 +227,17 @@ async function store(scope, payload, entry, { ttlSeconds = DEFAULT_TTL_SECONDS,
   }
 
   const key = listKey(scope, payload.model);
+  // The shape is stored BESIDE the entry rather than folded into the key: matching here is by
+  // embedding, so there is no key to fold it into. findMatch refuses a candidate whose shape differs.
+  const shape = JSON.stringify(shapeFields(payload));
   try {
-    await redis.client.lPush(key, JSON.stringify({ embedding, entry, storedAt: Date.now() }));
+    await redis.client.lPush(key, JSON.stringify({
+      embedding: encodeEmbedding(embedding),
+      norm: normOf(embedding),
+      shape,
+      entry,
+      storedAt: Date.now()
+    }));
     await redis.client.lTrim(key, 0, MAX_CANDIDATES_PER_MODEL - 1);
     // A rolling TTL on the whole per-model bucket, reset on every
     // store - simple and predictable (as long as there's traffic to
@@ -155,5 +261,11 @@ module.exports = {
   listKey,
   cosineSimilarity,
   extractPromptText,
-  DEFAULT_THRESHOLD
+  DEFAULT_THRESHOLD,
+  // exported for test/semanticCache.test.js: the storage format is a compatibility surface (both formats
+  // must keep matching), so its round-trip is asserted rather than assumed
+  encodeEmbedding,
+  decodeEmbedding,
+  normOf,
+  similarityWithNorms
 };