cachegate 1.3.1 → 1.4.1

package/.env.example

@@ -16,12 +16,37 @@ MODEL_ROUTER_INTERNAL_KEY=your-random-internal-key
 # from outside your own machine.
 # ALLOW_INSECURE_LOCAL_DEV=true
 
-# Required (at least one provider key - Anthropic, OpenAI, or both).
-# The model itself is named PER REQUEST in the API call's own "model"
-# field (see README's "Usage" section), not configured here - there is
-# no ANTHROPIC_MODEL/OPENAI_MODEL env var to set.
+# Required (at least one provider key - any of the four). Which provider serves
+# a request is decided by the model name in the API call's own "model" field
+# (see README's "Usage" section), not configured here - there is no
+# ANTHROPIC_MODEL/OPENAI_MODEL env var to set:
+#   claude-*        -> Anthropic
+#   gpt-*/o1*/o3*   -> OpenAI
+#   deepseek-*      -> DeepSeek    (deepseek-flash, deepseek-v4-pro)
+#   vendor/model    -> OpenRouter  (e.g. meta/llama-3-70b, deepseek/deepseek-chat)
 ANTHROPIC_API_KEY=sk-ant-api03-...
 
+# Optional: DeepSeek. Two things worth knowing before you set this:
+#  * pricing is TIME-AWARE here (DeepSeek bills peak and off-peak, off-peak
+#    exactly half), so cost_usd - and cost-based routing - follows the current
+#    billing window rather than one flat rate.
+#  * DeepSeek reports its own disk-cache tiers (prompt_cache_hit_tokens /
+#    prompt_cache_miss_tokens). Cached input is ~50x cheaper than a miss, and
+#    those fields are mapped through, so a cached request shows its real cost
+#    instead of being priced as if nothing was cached.
+# DEEPSEEK_API_KEY=sk-...
+# DEEPSEEK_BASE_URL=https://api.deepseek.com    # override for a proxy/gateway
+
+# Optional: OpenRouter - one key, many vendors, addressed as vendor/model.
+# Pricing is fetched from OpenRouter's own /models endpoint and cached, NOT
+# hardcoded: their catalog is large and changes without a release of this
+# project. When a model's price is unknown, cost_usd is null (never 0 - a zero
+# would read as "free" to cost-based routing and win every comparison).
+# OPENROUTER_API_KEY=sk-or-...
+# OPENROUTER_SITE_URL=https://your-app.example.com   # optional attribution headers:
+# OPENROUTER_SITE_NAME=Your App                      # sent only if you set them
+# OPENROUTER_BASE_URL=https://openrouter.ai/api/v1   # override for a proxy/gateway
+
 # Also required for the semantic cache's embeddings, regardless of
 # which provider actually serves chat requests - it's the only
 # embedding backend implemented (see semanticCache.js / README).
@@ -125,3 +150,57 @@ ANTHROPIC_API_KEY=sk-ant-api03-...
 # legitimate reason for a much larger payload. Raise only if you have
 # a specific reason to expect longer request bodies.
 # JSON_BODY_LIMIT=2mb
+
+# Optional: guardrails (roadmap step 25). Both default OFF, run
+# pre-dispatch (before either cache lookup, so cached content is
+# already redacted/policy-checked), zero behavior change unless you opt in.
+
+# PII redaction (pii.js): pattern-based (emails, phone numbers, SSNs,
+# credit card numbers, common API-key/secret shapes), synchronous and
+# local - no network call of its own. redact() never surfaces the
+# actual matched value anywhere, even internally - only a
+# {type, count} summary.
+# GUARDRAILS_PII_REDACTION=true
+
+# Prompt-injection detection (guardrails.js): heuristic patterns
+# (instruction-override, system-prompt-leak, role-play jailbreak,
+# "developer mode", DAN, "no restrictions" framing). GUARDRAILS_ENABLED
+# turns detection on at all; GUARDRAILS_INJECTION_ACTION picks what
+# happens on a hit - default `flag` (logged, request proceeds, since
+# heuristics false-positive) or `block` (403, request rejected).
+# GUARDRAILS_ENABLED=true
+# GUARDRAILS_INJECTION_ACTION=flag   # flag (default) | block | log - flag and log currently behave identically (console.warn + pass through); block is the only one that changes the response (403)
+
+# Optional: cascade routing (roadmap step 34). Default OFF - this is
+# genuinely more speculative than health scoring (an LLM grading another
+# LLM's confidence is inherently fuzzy), so it ships inert and only
+# activates when explicitly opted in. Applies to "router:" virtual models
+# only: dispatch to the top-ranked candidate, estimate its confidence, and
+# escalate to the next candidate when below the threshold instead of
+# accepting a cheap-but-unconfident answer.
+# CASCADE_ENABLED=true
+
+# Optional: the confidence floor (0-1) below which a successful response is
+# escalated rather than accepted. Only matters when CASCADE_ENABLED=true.
+# CASCADE_CONFIDENCE_THRESHOLD=0.5
+
+# Optional: the small/cheap grader model used to estimate confidence for
+# Anthropic candidates (Anthropic has no native logprobs; OpenAI confidence
+# comes from logprobs automatically when cascade is on). Unset (default)
+# means Anthropic candidates are never escalated - cascade fails open to
+# "accept" when there's no confidence signal. Set it to a full model name
+# the router has a key for (e.g. gpt-4o-mini or claude-haiku-4-5-20251001);
+# the provider is inferred from the name.
+# CASCADE_GRADER_MODEL=gpt-4o-mini
+
+# Optional: OpenTelemetry export (roadmap step 36.2). Default OFF - ships
+# fully inert; the SDK is never even loaded unless you opt in. Every request
+# gets a root span (trace_id carried as a span attribute so it can be joined
+# to the same request's metrics rows) plus child spans for cache lookup,
+# coalescing wait, and each provider dispatch attempt.
+# OTEL_ENABLED=true
+
+# Required for the SDK to actually initialize when OTEL_ENABLED=true (the
+# OTLP HTTP collector endpoint, e.g. http://localhost:4318/v1/traces). Unset
+# = the SDK never initializes at all, not "initializes and exports nowhere."
+# OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318/v1/traces

package/README.md

@@ -35,17 +35,16 @@ fills instead:
 
 - **Not a hosted service.** There's no cloud offering, no login, no
   billing, no multi-tenant key custody here — this is the engine you
-  run yourself. If you want that instead, that's a separate, closed
-  product built on top of this same engine — not a fork of this one,
-  and not something this repository will ever grow into. This project
-  intentionally doesn't ship the pieces (billing, multi-tenant key
-  custody, a login system) that a competing hosted offering would need,
-  and isn't looking for PRs that add them (see `CONTRIBUTING.md`'s
-  scope note) — not because the license forbids it (MIT permits
-  exactly that — see `LICENSE`), but because it's not what this project
-  is for.
-- **Not a 140-provider gateway.** Anthropic and OpenAI today (see
-  "Features" below for the honest current gap against a wider pitch).
+  run yourself. This project intentionally doesn't ship the pieces (billing,
+  multi-tenant key custody, a login system) that a competing hosted
+  offering would need, and isn't looking for PRs that add them (see
+  `CONTRIBUTING.md`'s scope note) — not because the license forbids it
+  (MIT permits exactly that — see `LICENSE`), but because it's not what
+  this project is for.
+- **Not a 140-provider gateway.** Four providers today — Anthropic, OpenAI,
+  DeepSeek and OpenRouter (the last one fronting many vendors behind a single
+  key). No Gemini, no Groq, no local models yet; see "Features" below for the
+  honest current gap against a wider pitch.
 - **Not a vector-indexed semantic cache** (yet) — see "Two kinds of
   cache hit" for the real, disclosed scale limit.
 
@@ -105,8 +104,10 @@ Create or edit your local `.env` file (do **not** overwrite an existing one):
 PORT=4000
 MODEL_ROUTER_INTERNAL_KEY=your-random-internal-key
 ANTHROPIC_API_KEY=your-real-key-here
-# Optional:
+# Optional - any one of these unlocks that provider's models:
 # OPENAI_API_KEY=your-openai-key-here
+# DEEPSEEK_API_KEY=your-deepseek-key-here       # models: deepseek-flash, deepseek-v4-pro
+# OPENROUTER_API_KEY=your-openrouter-key-here   # models: any vendor/model id, e.g. meta/llama-3-70b
 # REDIS_URL=redis://localhost:6379
 ```
 
@@ -198,7 +199,9 @@ monorepo today).
 ## Usage
 
 Direct dispatch - name a specific provider's model, same as calling that
-provider yourself:
+provider yourself. The provider is chosen from the model name: `claude-*` ->
+Anthropic, `gpt-*`/`o1*`/`o3*` -> OpenAI, `deepseek-*` -> DeepSeek, and
+`vendor/model` -> OpenRouter.
 
 ```bash
 curl http://localhost:4000/v1/chat/completions \
@@ -211,6 +214,13 @@ curl http://localhost:4000/v1/chat/completions \
   }'
 ```
 
+Same call against DeepSeek or OpenRouter - only the `model` changes:
+
+```bash
+  -d '{"model": "deepseek-flash", "messages": [{"role": "user", "content": "Say hello"}]}'
+  -d '{"model": "meta/llama-3-70b", "messages": [{"role": "user", "content": "Say hello"}]}'
+```
+
 Routed dispatch - name a capability tier instead, and the router picks
 the cheapest currently-healthy provider for it:
 
@@ -278,8 +288,18 @@ auth, for local development only.
   streaming caller still gets the caching benefit. See "Streaming"
   below for the real scope boundary (tool-call streaming isn't
   included) and the cost-tracking detail it depends on.
-- Anthropic and OpenAI providers. (Not yet: Gemini, Groq, local models -
-  a real gap against the two-provider skeleton's original pitch.)
+- Four providers: Anthropic, OpenAI, DeepSeek and OpenRouter. (Not yet:
+  Gemini, Groq, local models - still a real gap against a wide-gateway pitch,
+  just a smaller one.) Adding another is deliberately cheap now: one module
+  under `providers/` plus a line in `providers/index.js` - every dispatch
+  path, the model-name detection and the "which key is missing" error all read
+  from that registry rather than hardcoding a pair.
+- **DeepSeek pricing is time-aware.** DeepSeek bills input in two tiers (cache
+  hit vs miss) and every rate has a peak and an off-peak value, so its cost
+  estimates - and therefore cost-based routing - reflect the current billing
+  window instead of a single flat rate. Opting into DeepSeek is also the one
+  place where prompt caching is billed this aggressively, so `cost_usd` on
+  those requests can be dramatically lower than the token count suggests.
 - Redis-backed exact-match response cache by content hash - the first,
   free, zero-risk check on every request.
 - A semantic cache on top of it, for near-duplicate prompts the exact
@@ -318,6 +338,23 @@ auth, for local development only.
   provider fallback), and the metrics store. They don't call a real
   provider API - that needs live keys and real spend, out of scope for
   this suite.
+- **Guardrails - PII redaction and prompt-injection detection**, both
+  off by default (opt-in, zero behavior change until you set an env
+  var). Runs pre-dispatch, before either cache lookup, so redacted
+  content is what gets cached and a blocked request never reaches a
+  provider:
+  - `GUARDRAILS_PII_REDACTION=true` - pattern-based detection +
+    redaction for emails, phone numbers, SSNs, credit card numbers
+    (shape + Luhn checksum), and common vendor API-key/secret shapes
+    (`pii.js`). Never surfaces the actual matched value anywhere, even
+    internally - only a `{type, count}` summary.
+  - `GUARDRAILS_ENABLED=true` - heuristic prompt-injection detection
+    (instruction-override, system-prompt-leak, role-play jailbreak,
+    "developer mode", DAN, "no restrictions" framing - `guardrails.js`).
+    Default action on a hit is `flag` (logged, request still proceeds) -
+    heuristics false-positive, so auto-blocking real traffic isn't the
+    default. Set `GUARDRAILS_INJECTION_ACTION=block` to reject a
+    detected attempt with `403` instead.
 
 ## Two kinds of cache hit - why they're reported separately
 
@@ -360,6 +397,99 @@ doesn't have one). That's fine at single-instance, self-hosted volume;
 it is not built to scale past that cap. See `semanticCache.js` for the
 full reasoning.
 
+## Provider prompt caching + Cachegate's own cache - compounding, not fighting
+
+Anthropic and OpenAI both have their own prompt-caching feature,
+separate from anything in this project - a KV-cache the provider
+itself keeps for a repeated, unchanging prefix (a long system prompt,
+a set of few-shot examples, a big shared document) so it doesn't get
+reprocessed on every call. It's easy to assume that overlaps with
+Cachegate's own exact/semantic cache and picking one means giving up
+the other. It doesn't - they solve different problems, and used
+together they compound:
+
+- **Cachegate's cache is checked FIRST, before any provider is ever
+  called.** An exact or semantic hit costs `$0` and involves the
+  provider not at all - strictly better than even a heavily-discounted
+  cached-prefix rate, because there's no completion call at all.
+- **Provider-level prompt caching only ever matters on a genuine
+  Cachegate miss** - a request different enough (in its varying tail)
+  that it doesn't match anything cached, but sharing a long, unchanging
+  prefix (system prompt, few-shot examples) with other misses that came
+  before it. That's the case Cachegate's own cache structurally can't
+  help with - the *tail* differs, so the request as a whole is a miss -
+  but the provider's own cache can still skip reprocessing the shared
+  *prefix*, cutting cost and latency on every one of those misses.
+
+**This already works transparently - no cachegate code change
+needed.** `providers/anthropic.js` and `providers/openai.js` both
+forward your `messages`/`system`/`tools` fields through to the
+provider's SDK as-is; neither reshapes message content or strips
+unrecognized properties from it. Concretely:
+
+- **Anthropic**: mark the unchanging part with
+  `cache_control: {"type": "ephemeral"}`, same as you would calling
+  Anthropic directly - on the system prompt, on a tool definition, or
+  on a specific content block within `messages`. Whatever object you
+  put there reaches `client.messages.create()` unchanged.
+
+  ```bash
+  curl http://localhost:4000/v1/chat/completions \
+    -H "Content-Type: application/json" \
+    -H "Authorization: Bearer your-random-internal-key" \
+    -d '{
+      "model": "claude-sonnet-4-5-20250929",
+      "max_tokens": 1024,
+      "messages": [
+        {
+          "role": "system",
+          "content": [
+            {
+              "type": "text",
+              "text": "<...your long, unchanging system prompt / few-shot examples...>",
+              "cache_control": {"type": "ephemeral"}
+            }
+          ]
+        },
+        {"role": "user", "content": "This part changes on every call."}
+      ]
+    }'
+  ```
+
+  Anthropic's own docs are the source of truth for the minimum
+  cacheable prompt length (model-dependent) and the 5-minute default
+  TTL - this project doesn't set or override either.
+
+- **OpenAI**: fully automatic, no request changes at all. Once a
+  prompt's shared prefix is long enough (OpenAI's own current
+  threshold; see their docs, not repeated here since it's a number
+  they control and could change), OpenAI caches it on their side by
+  default - the exact same `messages` array you're already sending
+  through `providers/openai.js` unmodified is what makes this work, or
+  not, entirely on their end.
+
+**One thing worth watching, specific to Cachegate's own exact cache:**
+`cache.buildCacheKey()` hashes your `messages` (and `tools`) as given -
+a `cache_control` block is just another property on a content object,
+and it participates in that hash like anything else. Two requests that
+are otherwise identical but differ only in whether `cache_control` is
+present will land in **different** Cachegate exact-cache entries -
+Cachegate doesn't know the extra field is caching metadata, it's just
+part of the request shape. Not a bug, just a consequence of exact
+meaning exact: keep `cache_control` usage *consistent* across calls
+for the same logical prompt (always include it, or never) rather than
+sometimes adding it, or you'll needlessly fragment Cachegate's own
+cache into two variants of what should be one entry.
+
+**If `GUARDRAILS_PII_REDACTION` is on** (see "Features" above): PII
+redaction only ever rewrites a content block's `text` field in place -
+`cache_control` and every other property on that block pass through
+untouched. Redaction changing the *text* of a normally-stable shared
+prefix would still be an unusual thing to have happen (a system prompt
+containing PII isn't a common shape), but if it ever does, the
+redacted version is what both Cachegate's cache key and the provider's
+own cache lookup see - consistently, on every call, not just some.
+
 ## Streaming
 
 `stream: true` forwards a real, incremental, token-by-token response

package/cache.js

@@ -22,21 +22,141 @@ const redis = require('./redisClient');
 // isolated even if the caller's own scope-naming convention were ever
 // guessed or leaked; a compromised/guessed prefix alone can't be walked
 // into another scope's cached content.
+
+// Prompt canonicalization (Phase 2, step 21) - normalizes ONLY what gets
+// HASHED, never what is sent to the provider (buildCacheKey receives the
+// payload, but the provider call in server.js uses payload.messages raw).
+// Two prompts that differ only by whitespace/case-of-structure/literal
+// values must hash to the SAME key. Conservative by design: a false-
+// positive collapse (two prompts that actually want different answers
+// sharing one key) is worse than a miss, so this folds only surface-level
+// variation, never meaning.
+function normalizeMessages(messages) {
+  if (!Array.isArray(messages)) return messages;
+  return messages.map((m) => {
+    if (!m || typeof m !== 'object') return m;
+    // Canonical field order (role, content first) so {role,content} and
+    // {content,role} hash identically; any other fields (name,
+    // tool_call_id, ...) keep their original relative order after that.
+    const out = { role: m.role, content: normalizeContent(m.content) };
+    for (const key of Object.keys(m)) {
+      if (key !== 'role' && key !== 'content') out[key] = m[key];
+    }
+    return out;
+  });
+}
+
+function normalizeContent(content) {
+  if (typeof content === 'string') return normalizeText(content);
+  if (Array.isArray(content)) return content.map(normalizeContent);
+  return content;
+}
+
+// Conservative canonical form of one text block: NFC unicode, structural
+// The literal-slotting steps, defined ONCE: normalizeText applies them, and slottingFlags() reports
+// which ones actually fire. Two definitions would be the same mistake R2 fixed one file over - a rule
+// enforced on one path and quietly stale on the other - and here it would be worse, because the
+// measurement would drift away from the behaviour it is supposed to be measuring.
+//
+// Order is most-specific-first (URL -> email -> date -> number) so a longer token is not half-consumed
+// by a shorter pattern. URLs and emails are unconditional: an incidental identifier really does want
+// the same answer. Numbers and dates are GATED, default OFF (CACHE_KEY_SLOT_NUMBERS) - a number or a
+// date is very often THE substance of the answer, and on an EXACT cache (no similarity threshold, a
+// guaranteed match) that is a correctness bug rather than a tuning knob.
+const SLOTTERS = [
+  { name: 'url', re: /(?:https?:\/\/|www\.)\S+/gi, gated: false },
+  { name: 'email', re: /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g, gated: false },
+  { name: 'date', re: /\b\d{4}-\d{2}-\d{2}\b/g, gated: true },
+  { name: 'date', re: /\b\d{1,2}\/\d{1,2}\/\d{2,4}\b/g, gated: true },
+  { name: 'number', re: /\b\d+(?:\.\d+)?\b/g, gated: true }
+];
+
+function numbersSlottingEnabled() {
+  return process.env.CACHE_KEY_SLOT_NUMBERS === 'true';
+}
+
+// punctuation folding, whitespace collapse - everything that happens BEFORE slotting.
+function foldText(text) {
+  return String(text)
+    .normalize('NFC')
+    .replace(/\u00A0/g, ' ') // non-breaking space
+    .replace(/[\u2018\u2019\u201A\u201B]/g, "'") // curly single quotes
+    .replace(/[\u201C\u201D\u201E\u201F]/g, '"') // curly double quotes
+    .replace(/[\u2013\u2014]/g, '-') // en/em dash
+    .replace(/\u2026/g, '...') // ellipsis
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function normalizeText(text) {
+  let out = foldText(text);
+  for (const slotter of SLOTTERS) {
+    if (slotter.gated && !numbersSlottingEnabled()) continue;
+    // String.prototype.replace with a /g regex resets lastIndex, so sharing these regex objects
+    // between calls is safe (unlike .test()/.exec(), which are stateful - hence match() below).
+    out = out.replace(slotter.re, '<var>');
+  }
+  return out;
+}
+
+// Which slotting steps WOULD fire for this request, for the R4 measurement.
+//
+// Reports gated steps even when the gate is off, on purpose: that is the only way to price turning
+// number/date slotting ON without turning it on, which is exactly the shape of the decision waiting
+// for data. It reports what the rules MATCH, not what changed the key - a prompt with no URL and one
+// with three URLs are both "url fired", and the hit-rate split is what is being measured.
+function slottingFlags(messages) {
+  const flags = { url: 0, email: 0, date: 0, number: 0, numbers_gate: numbersSlottingEnabled() ? 1 : 0 };
+  if (!Array.isArray(messages)) return flags;
+  for (const message of messages) {
+    if (!message || typeof message.content !== 'string') continue;
+    const folded = foldText(message.content);
+    for (const slotter of SLOTTERS) {
+      if (flags[slotter.name]) continue; // already known to fire; no need to scan again
+      if (folded.match(slotter.re)) flags[slotter.name] = 1;
+    }
+  }
+  return flags;
+}
+
+// The fields that change the SHAPE of an answer rather than its content, defined ONCE because two
+// paths consume them: buildCacheKey folds them into the exact-cache key, and semanticCache.js stores
+// them beside an entry and refuses a hit across a mismatch. Adding them one at a time is precisely how
+// the two paths drifted apart - this key learned about response_format and the semantic path never did
+// - so the list lives in one place and both call it.
+//
+// temperature is deliberately ABSENT: it already participates in the exact key with a real numeric
+// default (`?? 0.0`), and semantic matching is approximate by design, so gating on it there would
+// mostly lower the hit rate for callers who leave it at the default. seed IS here: it is a determinism
+// contract, not a sampling hint.
+//
+// Key ORDER matters - it is part of the serialization the hash is taken over - and it is kept
+// identical to the inline field list this replaced. JSON.stringify drops undefined values, so a payload
+// with no seed produces a byte-identical key and the change does not flush the cache.
+function shapeFields(payload) {
+  return {
+    tools: payload.tools,
+    tool_choice: payload.tool_choice,
+    response_format: payload.response_format,
+    seed: payload.seed
+  };
+}
+
 function buildCacheKey(scope, payload) {
+  const canonicalMessages = normalizeMessages(payload.messages);
+  if (process.env.CACHE_KEY_DEBUG) {
+    // Traceable, not silent (step 21.2): a false-positive collision can be
+    // reconstructed by re-running normalizeMessages on the original - this
+    // opt-in log line just makes it visible without that extra step.
+    console.log('[cache] canonical messages:', JSON.stringify(canonicalMessages));
+  }
   const normalized = JSON.stringify({
     ...(scope != null ? { scope } : {}),
     model: payload.model,
-    messages: payload.messages,
+    messages: canonicalMessages,
     temperature: payload.temperature ?? 0.0,
     max_tokens: payload.max_tokens,
-    tools: payload.tools,
-    tool_choice: payload.tool_choice,
-    // response_format changes the SHAPE of the answer (json_object vs
-    // plain text), so it must participate in the key too - otherwise a
-    // cached plain-text response could be served to a json_object caller
-    // (or vice versa). openai.js forwards it (see its own chat()); this
-    // file used to omit it, making an "exact" hit not always exact.
-    response_format: payload.response_format
+    ...shapeFields(payload)
   });
   const hash = crypto.createHash('sha256').update(normalized).digest('hex');
   const prefix = scope != null ? `ROUTER:${scope}:` : 'ROUTER:';
@@ -45,6 +165,10 @@ function buildCacheKey(scope, payload) {
 
 module.exports = {
   buildCacheKey,
+  shapeFields,
+  slottingFlags,
+  normalizeMessages,
+  normalizeText,
 
   isConnected() {
     return redis.isConnected();