cachegate 1.3.1 → 1.4.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cachegate",
-  "version": "1.3.1",
+  "version": "1.4.1",
   "description": "Self-hostable, OpenAI-compatible LLM proxy: routes to the cheapest healthy provider, caches responses exactly and semantically, tracks cost and latency per call.",
   "license": "MIT",
   "main": "server.js",
@@ -10,13 +10,18 @@
   "files": [
     "server.js",
     "cache.js",
+    "cascade.js",
+    "coalescing.js",
     "embeddings.js",
     "failover.js",
+    "guardrails.js",
     "metrics.js",
+    "pii.js",
     "redisClient.js",
     "router.js",
     "semanticCache.js",
     "streaming.js",
+    "tracing.js",
     "providers/",
     "public/",
     ".env.example"
@@ -26,10 +31,15 @@
   },
   "scripts": {
     "start": "node server.js",
-    "test": "node --test"
+    "test": "node --test",
+    "eval:semantic-cache": "node eval/semantic-cache-eval.js"
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.115.0",
+    "@huggingface/transformers": "^3.0.0",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.222.0",
+    "@opentelemetry/sdk-node": "^0.222.0",
     "dotenv": "^16.3.1",
     "express": "^4.18.2",
     "express-rate-limit": "^8.6.2",

package/pii.js

@@ -0,0 +1,128 @@
+// model-router/pii.js
+//
+// Pattern-based PII detection + redaction (roadmap step 25, PII track).
+// Gated behind GUARDRAILS_PII_REDACTION (default OFF, same off-by-default
+// convention as step 22's local embeddings) - ships fully inert until a
+// deployment opts in. Standalone for now: NOT wired into server.js yet
+// (see the step 25 kickoff directive) - this module only exposes the
+// detection/redaction primitive; the pre-dispatch request-path wiring
+// (and the coordination with the injection/policy track it needs to
+// share a hook shape with) is a separate, joint follow-up change.
+//
+// Deliberately pattern/regex based, not a model call: PII redaction has
+// to be synchronous, fast (it would run on every request, not just
+// cache misses), and free of its own network dependency - a step whose
+// whole job is stripping sensitive content out of a request shouldn't
+// itself be a network hop that could leak that content to a third party.
+//
+// Redaction never surfaces the actual matched value anywhere, even in
+// its own return value - only { type, count } counters. A false
+// positive costs an unnecessary redaction (annoying); a leaked value in
+// a log or a metrics row costs an actual PII exposure. This module is
+// built to make the cheaper mistake.
+
+// Order matters: more specific/longer patterns run first, so a token
+// that could satisfy two shapes (e.g. a 16-digit run also containing a
+// phone-shaped substring) is claimed by the more specific match before
+// a looser pattern gets a chance to partially match what's left of it.
+const PATTERNS = [
+  {
+    type: 'email',
+    regex: /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g
+  },
+  {
+    type: 'credit_card',
+    // 13-19 digits, optionally grouped by single spaces or dashes
+    // between digits (covers both "4111111111111111" and
+    // "4111 1111 1111 1111"/"4111-1111-1111-1111"). Matched by shape
+    // first, then narrowed by a Luhn checksum below - shape alone would
+    // false-positive on any long unrelated digit run (an order id, a
+    // padded invoice number).
+    regex: /\b(?:\d[ -]?){12,18}\d\b/g,
+    validate: (match) => luhnValid(match.replace(/[ -]/g, ''))
+  },
+  {
+    type: 'ssn',
+    // US SSN: NNN-NN-NNNN. Dashes required - a bare 9-digit run is too
+    // easy to confuse with an account/phone number to redact safely
+    // without a much higher false-positive rate.
+    regex: /\b\d{3}-\d{2}-\d{4}\b/g
+  },
+  {
+    type: 'phone',
+    // NA-style, requiring at least one separator (space/dash/dot/
+    // parens) - a bare 10-digit run is left to the credit-card pattern
+    // above (which needs 13+ digits, so no real overlap) rather than
+    // guessed at here. Uses digit lookaround, not \b, at both ends: a
+    // leading "(" is itself a non-word character, so a \b right before
+    // it never matches (word-boundary needs one word char and one
+    // non-word char either side) - \b would let the engine skip past a
+    // real "(555)" opening paren and leave it un-redacted outside the
+    // match. (?<!\d)/(?!\d) only cares that the run isn't glued to more
+    // digits, which is what actually needs guarding against here.
+    regex: /(?<!\d)(?:\+?1[ .-]?)?\(?\d{3}\)?[ .-]\d{3}[ .-]\d{4}(?!\d)/g
+  },
+  {
+    type: 'api_key',
+    // Common vendor secret shapes (OpenAI classic "sk-...", OpenAI
+    // project-scoped "sk-proj-...", Anthropic "sk-ant-...", AWS,
+    // GitHub, Slack, Google) - opaque tokens like these are exactly the
+    // kind of thing that ends up pasted into a prompt by accident. The
+    // "sk-" body allows dashes/underscores, not just alphanumerics, so
+    // it covers the dash-separated "-proj-"/"-ant-" variants too rather
+    // than needing one alternative per vendor prefix.
+    regex: /\b(?:sk-[a-zA-Z0-9_-]{20,}|AKIA[0-9A-Z]{16}|gh[pousr]_[a-zA-Z0-9]{20,}|xox[baprs]-[a-zA-Z0-9-]{10,}|AIza[0-9A-Za-z_-]{35})\b/g
+  }
+];
+
+function luhnValid(digits) {
+  let sum = 0;
+  let alternate = false;
+  for (let i = digits.length - 1; i >= 0; i--) {
+    let n = Number(digits[i]);
+    if (alternate) {
+      n *= 2;
+      if (n > 9) n -= 9;
+    }
+    sum += n;
+    alternate = !alternate;
+  }
+  return sum % 10 === 0;
+}
+
+function isEnabled() {
+  return process.env.GUARDRAILS_PII_REDACTION === 'true';
+}
+
+// redact(text) -> { text, redactions: [{ type, count }] }.
+// Side-effect free and safe to call unconditionally - when the flag is
+// off, returns the text UNCHANGED (not an error, not a throw), same
+// "always callable, flag decides" shape as embeddings.js's isEnabled().
+function redact(text) {
+  if (!isEnabled() || typeof text !== 'string' || text.length === 0) {
+    return { text, redactions: [] };
+  }
+
+  let result = text;
+  const counts = new Map();
+
+  for (const { type, regex, validate } of PATTERNS) {
+    // A fresh RegExp per pattern per call: the source patterns are
+    // global (/g), and a shared stateful regex's lastIndex would
+    // corrupt matching across concurrent/repeated calls in a
+    // long-running process handling many requests.
+    const re = new RegExp(regex.source, regex.flags);
+    result = result.replace(re, (match) => {
+      if (validate && !validate(match)) return match; // shape matched but failed validation - leave as-is
+      counts.set(type, (counts.get(type) || 0) + 1);
+      return `[REDACTED_${type.toUpperCase()}]`;
+    });
+  }
+
+  return {
+    text: result,
+    redactions: Array.from(counts.entries()).map(([type, count]) => ({ type, count }))
+  };
+}
+
+module.exports = { isEnabled, redact };

package/providers/deepseek.js

@@ -0,0 +1,194 @@
+// model-router/providers/deepseek.js
+//
+// DeepSeek speaks the OpenAI wire format, so this is the `openai` SDK pointed
+// at DeepSeek's own base URL - no second HTTP client, no hand-rolled fetch.
+// What is NOT identical is the money: DeepSeek bills input in two tiers
+// (cache hit vs cache miss) and every rate has a PEAK and an OFF-PEAK value,
+// so a single flat rate table - the shape providers/openai.js can get away
+// with - would misprice most requests. Both facts come from the vendor's own
+// pages, read 2026-09-10:
+//   base URL + models + rates: api-docs.deepseek.com/quick_start/pricing
+//   usage fields:              api-docs.deepseek.com/guides/kv_cache
+//
+// The review that prompted this file said "DeepSeek features incredibly cheap
+// API calls and aggressive server-side prompt caching" and told the reader to
+// "parse cached_tokens from DeepSeek's usage response blocks". The second half
+// is wrong in a way that would have silently zeroed the savings: DeepSeek does
+// not report OpenAI's nested prompt_tokens_details.cached_tokens, it reports a
+// flat prompt_cache_hit_tokens / prompt_cache_miss_tokens pair. Cached input
+// here is ~50x cheaper than a miss, so getting that mapping wrong is the
+// difference between an accurate cost dashboard and a decorative one.
+const { OpenAI } = require('openai');
+
+const BASE_URL = process.env.DEEPSEEK_BASE_URL || 'https://api.deepseek.com';
+
+// USD per 1M tokens, PEAK rates. Off-peak is exactly half (vendor's own note:
+// "Off-peak rates are half of the peak rates"). Peak = 01:00-04:00 and
+// 06:00-10:00 UTC, Monday through Friday; everything else is off-peak.
+const PEAK_RATES = {
+  'deepseek-flash':                 { hit: 0.006, miss: 0.30, output: 1.20 },
+  'deepseek-v4-pro':                { hit: 0.044, miss: 1.32, output: 3.96 },
+  // Legacy names the API still accepts. The vendor states these are served by
+  // V4.1-Flash and BILLED AT THE FLASH PRICE, so they must not fall through to
+  // a "unknown model" default with different numbers.
+  'deepseek-v4-flash':              { hit: 0.006, miss: 0.30, output: 1.20 },
+  'deepseek-v4-flash-vision-exp':   { hit: 0.006, miss: 0.30, output: 1.20 }
+};
+// `deepseek-v4-pro` is being retired: from 2026-09-14 requests to it are routed
+// to V4.1-Flash and billed as Flash. Until that date the pro rate is real, so
+// both are priced and the switch happens on the vendor's side, not ours.
+const DEFAULT_RATE = PEAK_RATES['deepseek-flash'];
+
+function isPeak(now = new Date()) {
+  const day = now.getUTCDay();            // 0 = Sunday, 6 = Saturday
+  if (day === 0 || day === 6) return false;
+  const hour = now.getUTCHours();
+  return (hour >= 1 && hour < 4) || (hour >= 6 && hour < 10);
+}
+
+function rateFor(model, now = new Date()) {
+  const base = PEAK_RATES[model] || DEFAULT_RATE;
+  return isPeak(now) ? base : { hit: base.hit / 2, miss: base.miss / 2, output: base.output / 2 };
+}
+
+/**
+ * Cost for one call. Input tokens are SPLIT, not uniform: `cacheHitTokens` of
+ * them were served from DeepSeek's disk cache at the hit rate, the remainder at
+ * the miss rate. Getting this wrong is not a rounding error - at Flash's peak
+ * rates a fully-cached 1M-token input costs $0.006 instead of $0.30.
+ *
+ * The 3-argument form (no options) is kept working for callers that have no
+ * cache data: it prices everything as a miss, which is the conservative
+ * (never-understate) direction for a cost dashboard.
+ *
+ * `now` is injectable so tests can pin the billing window instead of passing
+ * or failing depending on what time of day the suite runs.
+ */
+function estimateCost(model, inputTokens, outputTokens, { cacheHitTokens = 0, now = new Date() } = {}) {
+  const rate = rateFor(model, now);
+  const hit = Math.max(0, Math.min(cacheHitTokens || 0, inputTokens || 0));
+  const miss = Math.max(0, (inputTokens || 0) - hit);
+  return ((hit * rate.hit) + (miss * rate.miss) + ((outputTokens || 0) * rate.output)) / 1_000_000;
+}
+
+function buildClient(apiKey) {
+  return new OpenAI({
+    apiKey,
+    baseURL: BASE_URL,
+    // DeepSeek requires no extra headers; the SDK's defaults are fine.
+    timeout: Number(process.env.DEEPSEEK_TIMEOUT_MS) || 60000
+  });
+}
+
+function usageFrom(usage = {}) {
+  const input = usage.prompt_tokens || 0;
+  const hit = usage.prompt_cache_hit_tokens || 0;
+  const miss = typeof usage.prompt_cache_miss_tokens === 'number'
+    ? usage.prompt_cache_miss_tokens
+    : Math.max(0, input - hit);
+  return { input_tokens: input, output_tokens: usage.completion_tokens || 0, cache_hit_tokens: hit, cache_miss_tokens: miss };
+}
+
+async function chat(client, payload, options = {}) {
+  const request = {
+    model: payload.model,
+    messages: payload.messages,
+    temperature: typeof payload.temperature === 'number' ? payload.temperature : 0.0,
+    max_tokens: payload.max_tokens || 1024,
+    ...(payload.tools && { tools: payload.tools }),
+    ...(payload.tool_choice && { tool_choice: payload.tool_choice }),
+    ...(payload.response_format && { response_format: payload.response_format })
+  };
+
+  const start = Date.now();
+  const response = await client.chat.completions.create(request);
+  const latencyMs = Date.now() - start;
+
+  const choice = response.choices[0];
+  const usage = usageFrom(response.usage);
+  const costUsd = estimateCost(payload.model, usage.input_tokens, usage.output_tokens, { cacheHitTokens: usage.cache_hit_tokens });
+
+  return {
+    provider: 'deepseek',
+    model: payload.model,
+    latency_ms: latencyMs,
+    usage,
+    cost_usd: costUsd,
+    content: choice.message.content || '',
+    // Thinking-mode models (the default for the current models) return their
+    // chain separately. Passed through instead of dropped so a caller can see
+    // it, but NOT concatenated into `content` - that would corrupt every
+    // consumer that treats content as the answer.
+    reasoning_content: choice.message.reasoning_content,
+    tool_calls: choice.message.tool_calls,
+    raw: response
+  };
+}
+
+/**
+ * Pure state-accumulation for one streamed chunk, factored out for the same
+ * reason as providers/openai.js's: usage/cost extraction is then unit-testable
+ * against canned chunks with no live API.
+ *
+ * DeepSeek only sends `usage` on the final chunk, and only when the request
+ * asked for it (`stream_options.include_usage`, set in chatStream below) -
+ * without that flag a streamed call carries no usage at all and the cost
+ * tracking this project is built around would sit at zero while looking fine.
+ */
+function applyStreamChunk(state, chunk, onDelta) {
+  const choice = chunk.choices && chunk.choices[0];
+  if (choice && choice.delta) {
+    if (choice.delta.content) {
+      state.content += choice.delta.content;
+      onDelta(choice.delta.content);
+    }
+    if (choice.delta.reasoning_content) state.reasoningContent = (state.reasoningContent || '') + choice.delta.reasoning_content;
+  }
+  if (chunk.usage) {
+    state.inputTokens = chunk.usage.prompt_tokens || 0;
+    state.outputTokens = chunk.usage.completion_tokens || 0;
+    state.cacheHitTokens = chunk.usage.prompt_cache_hit_tokens || 0;
+    state.cacheMissTokens = chunk.usage.prompt_cache_miss_tokens;
+  }
+}
+
+async function chatStream(client, payload, { onDelta, signal } = {}) {
+  const request = {
+    model: payload.model,
+    messages: payload.messages,
+    temperature: typeof payload.temperature === 'number' ? payload.temperature : 0.0,
+    max_tokens: payload.max_tokens || 1024,
+    stream: true,
+    stream_options: { include_usage: true }
+  };
+
+  const start = Date.now();
+  const stream = await client.chat.completions.create(request, signal ? { signal } : undefined);
+
+  const state = { content: '', inputTokens: 0, outputTokens: 0, cacheHitTokens: 0, reasoningContent: '' };
+  for await (const chunk of stream) {
+    applyStreamChunk(state, chunk, onDelta || (() => {}));
+  }
+
+  const latencyMs = Date.now() - start;
+  const usage = usageFrom({
+    prompt_tokens: state.inputTokens,
+    completion_tokens: state.outputTokens,
+    prompt_cache_hit_tokens: state.cacheHitTokens,
+    ...(typeof state.cacheMissTokens === 'number' && { prompt_cache_miss_tokens: state.cacheMissTokens })
+  });
+  const costUsd = estimateCost(payload.model, usage.input_tokens, usage.output_tokens, { cacheHitTokens: usage.cache_hit_tokens });
+
+  return {
+    provider: 'deepseek',
+    model: payload.model,
+    latency_ms: latencyMs,
+    usage,
+    cost_usd: costUsd,
+    content: state.content,
+    reasoning_content: state.reasoningContent || undefined,
+    tool_calls: undefined
+  };
+}
+
+module.exports = { buildClient, chat, chatStream, applyStreamChunk, estimateCost, isPeak, rateFor, BASE_URL };

package/providers/index.js

@@ -0,0 +1,62 @@
+// model-router/providers/index.js
+//
+// The provider registry. Before this file, "which provider serves this model?"
+// was answered by an if/else chain repeated in eight places across server.js
+// (detection, two dispatch forks, the streaming client + stream-function
+// forks, the cascade grader, a second tier branch, and a metrics label). Every
+// one of them knew exactly two providers, so adding a third meant finding and
+// patching all eight - and missing one produced a weird failure far from the
+// edit, not an error at the place that was forgotten.
+//
+// Adding a provider is now: write providers/<name>.js to the contract
+// (buildClient/chat/chatStream/estimateCost), require it here, give it a model
+// prefix and an env key. Everything else routes through this file.
+const anthropic = require('./anthropic');
+const openai = require('./openai');
+const deepseek = require('./deepseek');
+const openrouter = require('./openrouter');
+
+const PROVIDERS = { anthropic, openai, deepseek, openrouter };
+
+// The env var holding each provider's key. Used for both presence checks and
+// the "not configured" error message, so those two can never disagree about
+// which variable a provider actually needs.
+const ENV_KEYS = {
+  anthropic: 'ANTHROPIC_API_KEY',
+  openai: 'OPENAI_API_KEY',
+  deepseek: 'DEEPSEEK_API_KEY',
+  openrouter: 'OPENROUTER_API_KEY'
+};
+
+/**
+ * Which provider serves this model? Returns null when nothing claims it, which
+ * callers turn into a 400 "Unsupported model" - the same answer as before for
+ * anything unmatched.
+ *
+ * ORDER MATTERS: OpenRouter is checked first because its ids are
+ * `vendor/model`, and `deepseek/deepseek-chat` must not be captured by the
+ * direct DeepSeek prefix. (The direct DeepSeek ids are `deepseek-flash` /
+ * `deepseek-v4-pro` - no slash - so the two never actually collide, but the
+ * ordering is what guarantees that stays true if either vendor renames.)
+ */
+function detectProvider(model) {
+  if (typeof model !== 'string' || !model) return null;
+  if (openrouter.isOpenRouterModel(model)) return 'openrouter';
+  if (model.startsWith('claude-')) return 'anthropic';
+  if (model.startsWith('deepseek-')) return 'deepseek';
+  if (model.startsWith('gpt-') || model.startsWith('o1') || model.startsWith('o3')) return 'openai';
+  return null;
+}
+
+function get(name) { return PROVIDERS[name] || null; }
+
+function envKey(name) { return ENV_KEYS[name] || null; }
+
+function names() { return Object.keys(PROVIDERS); }
+
+/** Provider names that look usable given the current environment (presence only, never values). */
+function configured(env = process.env) {
+  return names().filter((n) => !!env[ENV_KEYS[n]]);
+}
+
+module.exports = { PROVIDERS, ENV_KEYS, detectProvider, get, envKey, names, configured };

package/providers/openai.js

@@ -15,7 +15,14 @@ function estimateCost(model, inputTokens, outputTokens) {
   return ((inputTokens * rate.input) + (outputTokens * rate.output)) / 1_000_000;
 }
 
-async function chat(client, payload) {
+// `options.requestLogprobs` (cascade routing, step 34): ask the API for
+// per-token logprobs so cascade.js can estimate confidence from the response.
+// ONLY set by the router for an OpenAI candidate when cascade is active for
+// that dispatch - a normal caller never sees this, and it's near-zero extra
+// cost on the request it's attached to. top_logprobs: 1 keeps the payload
+// small (one alternative per token) while still carrying the emitted token's
+// own logprob, which is all the confidence math needs.
+async function chat(client, payload, options = {}) {
   const request = {
     model: payload.model,
     messages: payload.messages,
@@ -23,7 +30,8 @@ async function chat(client, payload) {
     max_tokens: payload.max_tokens || 1024,
     ...(payload.tools && { tools: payload.tools }),
     ...(payload.tool_choice && { tool_choice: payload.tool_choice }),
-    ...(payload.response_format && { response_format: payload.response_format })
+    ...(payload.response_format && { response_format: payload.response_format }),
+    ...(options.requestLogprobs && { logprobs: true, top_logprobs: 1 })
   };
 
   const start = Date.now();