cachegate 1.3.0 → 1.4.0

package/router.js

@@ -147,6 +147,16 @@ const COMPARISON_OUTPUT_TOKENS = 500;
 // (in which case we still have to pick one - see pickCandidate).
 const UNHEALTHY_ERROR_RATE = 0.5;
 
+// Step 33 (health scoring + circuit breakers): a second, STRICTER tier
+// above "unhealthy" - "shed". A candidate at or above SHED_ERROR_RATE is
+// clearly down (not just having a rough patch), so it's fully excluded
+// this request rather than merely deprioritized. avgQualityScore (step 32)
+// is a second, independent confirming signal: a candidate that keeps
+// needing failover to nominally "succeed" sheds even before its raw
+// error-rate count crosses the threshold. Both thresholds are env-tunable.
+const SHED_ERROR_RATE = Number(process.env.ROUTER_SHED_ERROR_RATE) || 0.9;
+const SHED_QUALITY_SCORE = Number(process.env.ROUTER_SHED_QUALITY_SCORE) || 0.2;
+
 // Minimum number of recent requests before a provider's error rate is
 // treated as meaningful. Without this, a brand-new provider (or one whose
 // traffic just resumed) with a SINGLE request that happened to error has
@@ -205,18 +215,36 @@ async function pickCandidate(virtualModel, scope) {
     const estimatedCostUsd = estimate
       ? estimate(candidate.model, COMPARISON_INPUT_TOKENS, COMPARISON_OUTPUT_TOKENS)
       : Infinity;
-    const providerStat = stats[candidate.provider] || { errorRate: 0, avgLatencyMs: null, sampleSize: 0 };
+    const providerStat = stats[candidate.provider] || { errorRate: 0, avgLatencyMs: null, avgQualityScore: null, sampleSize: 0 };
+    const hasEnoughSamples = providerStat.sampleSize >= MIN_HEALTH_SAMPLES;
+    // shed = clearly down (33.1): high error rate, OR low avgQualityScore
+    // (keeps failing over to "succeed"). avgQualityScore is null when there
+    // is no quality data yet - insufficient data must never shed.
+    const shed =
+      hasEnoughSamples &&
+      (providerStat.errorRate >= SHED_ERROR_RATE ||
+        (typeof providerStat.avgQualityScore === 'number' && providerStat.avgQualityScore < SHED_QUALITY_SCORE));
     return {
       ...candidate,
       estimatedCostUsd,
       errorRate: providerStat.errorRate,
       avgLatencyMs: providerStat.avgLatencyMs,
-      healthy: providerStat.sampleSize < MIN_HEALTH_SAMPLES || providerStat.errorRate < UNHEALTHY_ERROR_RATE
+      avgQualityScore: providerStat.avgQualityScore,
+      shed,
+      healthy: !hasEnoughSamples || providerStat.errorRate < UNHEALTHY_ERROR_RATE
     };
   });
 
-  const healthy = scored.filter((c) => c.healthy);
-  const pool = healthy.length > 0 ? healthy : scored; // all unhealthy: pick the least-bad rather than fail outright
+  // 33.1: shed the clearly-down candidates first (stricter than unhealthy).
+  const notShed = scored.filter((c) => !c.shed);
+  // 33.2: all-down fallback - if every candidate is shed, fall back to the
+  // full list and try anyway (least-bad rather than fail outright), the
+  // same philosophy as the all-unhealthy fallback below.
+  const afterShed = notShed.length > 0 ? notShed : scored;
+  // Existing unhealthy filter (a rough patch, 0.5 <= errorRate < 0.9) on
+  // what remains after shedding.
+  const healthy = afterShed.filter((c) => c.healthy);
+  const pool = healthy.length > 0 ? healthy : afterShed; // all unhealthy: pick the least-bad rather than fail outright
 
   const strategy = loadStrategy();
   let ranked;
@@ -245,6 +273,7 @@ async function pickCandidate(virtualModel, scope) {
       consideredTier: virtualModel,
       strategy,
       candidates: scored,
+      shedExcludedACandidate: notShed.length < scored.length,
       allUnhealthy: healthy.length === 0,
       latencyGuardExcludedACandidate: guardApplied
     }

package/semanticCache.js

@@ -20,6 +20,19 @@
 // meant to scale past that cap. A real vector index is the honest next
 // step if traffic outgrows it.
 //
+// What that brute force actually COSTS was worth measuring rather than
+// assuming, and it was not the arithmetic: each entry used to carry its
+// embedding as JSON text, so a 1536-dim vector arrived as ~30 KB of
+// digits and a lookup parsed up to 200 of them - 5.89 MB of JSON
+// number-parsing per semantic miss, against 200 x 1536 multiply-adds to
+// score them (48.7 ms, measured). The vector is now stored as base64
+// Float32 with its L2 norm beside it: 1.58 MB and 3.6 ms for the same
+// work, 13x faster. Entries written before this change are still read
+// correctly - both formats live in the same list until the old ones age
+// out - and the cap, the threshold and the honesty framing above are
+// unchanged. This is an encoding improvement, NOT the vector index the
+// last paragraph is still waiting for.
+//
 // The threshold is a probabilistic judgment call, not a guarantee: a
 // "hit" above the threshold is the router's best guess that two
 // prompts want the same answer, not proof they do. Set it too low and
@@ -32,6 +45,15 @@
 
 const redis = require('./redisClient');
 const embeddingsDefault = require('./embeddings');
+// The answer-shape definition is shared with the exact cache ON PURPOSE: the two paths diverged once -
+// cache.js keyed response_format and this file never did - and a field added to one of them silently
+// stops being enforced by the other. No cycle: cache.js does not require this file.
+const { shapeFields } = require('./cache');
+
+// The shape of a request that demands nothing about the answer's form: no response_format, no tools,
+// no tool_choice, no seed. The semantic shape gate uses it to decide whether an entry carrying no
+// recorded shape may still be served - see the gate in findMatch.
+const SHAPE_OF_NEUTRAL = JSON.stringify(shapeFields({}));
 
 const MAX_CANDIDATES_PER_MODEL = Number(process.env.SEMANTIC_CACHE_MAX_CANDIDATES) || 200;
 const DEFAULT_TTL_SECONDS = Number(process.env.SEMANTIC_CACHE_TTL_SECONDS) || 3600;
@@ -45,6 +67,44 @@ function listKey(scope, model) {
   return scope != null ? `SEMANTIC_LIST:${scope}:${model}` : `SEMANTIC_LIST:${model}`;
 }
 
+// Embeddings are stored as base64 Float32 rather than JSON numbers. Float32 loses ~7 significant digits
+// versus the JSON doubles the old format held, and cosine similarity over 1536 dimensions cannot see the
+// difference - test/semanticCache.test.js asserts both paths agree, and the legacy branch below is what
+// keeps entries written before this change matching correctly.
+function encodeEmbedding(vec) {
+  const f = Float32Array.from(vec);
+  return Buffer.from(f.buffer, f.byteOffset, f.byteLength).toString('base64');
+}
+
+function decodeEmbedding(record) {
+  const e = record && record.embedding;
+  if (Array.isArray(e)) return e;                       // legacy: JSON doubles
+  if (typeof e !== 'string') return null;
+  const buf = Buffer.from(e, 'base64');
+  // A Float32 vector is 4 bytes per element; anything else is not one. Checked rather than assumed so a
+  // corrupt or truncated entry is skipped (as a malformed JSON entry already was) instead of producing a
+  // wrong-length array that silently scores 0 against everything.
+  if (buf.length === 0 || buf.length % 4 !== 0) return null;
+  return new Float32Array(buf.buffer, buf.byteOffset, buf.length / 4);
+}
+
+function normOf(v) {
+  let n = 0;
+  for (let i = 0; i < v.length; i++) n += v[i] * v[i];
+  return Math.sqrt(n);
+}
+
+// Same result as cosineSimilarity(), given the two norms the caller already has or has stored. The stored
+// norm is the point: without it, every lookup recomputes the norm of every candidate - a third of the float
+// work per candidate, for a value that never changes.
+function similarityWithNorms(a, aNorm, b, bNorm) {
+  if (!a || !b || a.length !== b.length) return 0;
+  if (aNorm === 0 || bNorm === 0) return 0;
+  let dot = 0;
+  for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
+  return dot / (aNorm * bNorm);
+}
+
 function cosineSimilarity(a, b) {
   if (!a || !b || a.length !== b.length) return 0;
   let dot = 0;
@@ -79,6 +139,11 @@ function isEnabled(embeddings = embeddingsDefault) {
 // approximate text match can't guarantee the exact argument values a
 // tool call needs, and returning a plausible-but-wrong tool call is a
 // worse failure than a cache miss.
+//
+// response_format is NOT excluded, it is FILTERED (see findMatch): the
+// answer is still worth caching, it just may not be served to a caller
+// who asked for a different shape. Excluding it outright would throw away
+// hit rate to solve a matching problem.
 function isCacheable(payload) {
   return !payload.tools;
 }
@@ -86,6 +151,11 @@ function isCacheable(payload) {
 async function findMatch(scope, payload, { threshold = DEFAULT_THRESHOLD, embeddings = embeddingsDefault } = {}) {
   if (!isEnabled(embeddings) || !isCacheable(payload)) return null;
 
+  // The shape this request needs. A semantic match is approximate about the PROMPT, never about the
+  // answer's shape: a json_object caller served a cached plain-text answer fails to parse it, which
+  // surfaces as an upstream outage rather than a cache miss. Same reasoning that excludes tool calls.
+  const shape = JSON.stringify(shapeFields(payload));
+
   let queryEmbedding;
   try {
     queryEmbedding = await embeddings.embed(extractPromptText(payload));
@@ -102,7 +172,9 @@ async function findMatch(scope, payload, { threshold = DEFAULT_THRESHOLD, embedd
     return null;
   }
 
+  const queryNorm = normOf(queryEmbedding);
   let best = null;
+  let shapeSkipped = 0;
   for (const line of raw) {
     let record;
     try {
@@ -110,11 +182,36 @@ async function findMatch(scope, payload, { threshold = DEFAULT_THRESHOLD, embedd
     } catch {
       continue; // a malformed entry is skipped, not fatal
     }
-    const similarity = cosineSimilarity(queryEmbedding, record.embedding);
+    // Shape gate. Refuse a candidate that cannot PROVE the shape matches.
+    //
+    // The asymmetry is deliberate, and it reconciles two requirements that both have to hold:
+    //   * A request that demands nothing about the answer's shape can consume either form, so an entry
+    //     with no recorded shape - stored before this gate existed - is still served. That is what
+    //     keeps "an upgrade is not a cache flush" true for the plain-prose case, which is the vast
+    //     majority of traffic.
+    //   * A request that DOES demand a shape may only be served by an entry that proves it matches.
+    //     Prose handed to a json_object caller fails to parse in the caller and surfaces as an upstream
+    //     outage - worse than a miss - and that direction is the entire reason this gate exists.
+    // A recorded shape that differs is refused in both directions: the reverse (json handed to a prose
+    // caller) does not crash anything, but it is still the wrong answer to the question asked.
+    const shapeNeutral = shape === SHAPE_OF_NEUTRAL;
+    if (!(record.shape === shape || (record.shape === undefined && shapeNeutral))) {
+      shapeSkipped += 1;
+      continue;
+    }
+    const vector = decodeEmbedding(record);
+    if (!vector) continue; // ditto an entry whose vector cannot be read
+    const candidateNorm = typeof record.norm === 'number' ? record.norm : normOf(vector);
+    const similarity = similarityWithNorms(queryEmbedding, queryNorm, vector, candidateNorm);
     if (similarity >= threshold && (!best || similarity > best.similarity)) {
       best = { entry: record.entry, similarity };
     }
   }
+  if (!best && shapeSkipped > 0) {
+    // Visible, because "the semantic cache stopped hitting" otherwise looks like a tuning problem
+    // rather than the shape gate doing its job.
+    console.log(`[semantic] ${shapeSkipped} candidate(s) skipped: answer shape differs from this request`);
+  }
   return best;
 }
 
@@ -130,8 +227,17 @@ async function store(scope, payload, entry, { ttlSeconds = DEFAULT_TTL_SECONDS,
   }
 
   const key = listKey(scope, payload.model);
+  // The shape is stored BESIDE the entry rather than folded into the key: matching here is by
+  // embedding, so there is no key to fold it into. findMatch refuses a candidate whose shape differs.
+  const shape = JSON.stringify(shapeFields(payload));
   try {
-    await redis.client.lPush(key, JSON.stringify({ embedding, entry, storedAt: Date.now() }));
+    await redis.client.lPush(key, JSON.stringify({
+      embedding: encodeEmbedding(embedding),
+      norm: normOf(embedding),
+      shape,
+      entry,
+      storedAt: Date.now()
+    }));
     await redis.client.lTrim(key, 0, MAX_CANDIDATES_PER_MODEL - 1);
     // A rolling TTL on the whole per-model bucket, reset on every
     // store - simple and predictable (as long as there's traffic to
@@ -155,5 +261,11 @@ module.exports = {
   listKey,
   cosineSimilarity,
   extractPromptText,
-  DEFAULT_THRESHOLD
+  DEFAULT_THRESHOLD,
+  // exported for test/semanticCache.test.js: the storage format is a compatibility surface (both formats
+  // must keep matching), so its round-trip is asserted rather than assumed
+  encodeEmbedding,
+  decodeEmbedding,
+  normOf,
+  similarityWithNorms
 };