cachegate 1.0.0

package/redisClient.js

@@ -0,0 +1,45 @@
+// model-router/redisClient.js
+//
+// One shared Redis connection, used by both the exact-match cache
+// (cache.js) and the semantic cache (semanticCache.js). Previously
+// cache.js opened and owned this connection privately; pulled out here
+// so the semantic cache doesn't open a second connection to the same
+// Redis instance for the same purpose.
+
+const { createClient } = require('redis');
+
+const client = createClient({
+  url: process.env.REDIS_URL || 'redis://localhost:6379'
+});
+
+client.on('error', (err) => {
+  console.warn('⚠️ Redis client error:', err?.message || err?.code || String(err));
+});
+
+let readyResolve;
+// Resolves once the initial connection attempt finishes, success or
+// failure - tests await this instead of polling isConnected() in a
+// loop. Normal request handling doesn't need it: get/set/find/store all
+// already check isConnected() and degrade gracefully.
+const ready = new Promise((resolve) => { readyResolve = resolve; });
+
+(async () => {
+  try {
+    if (process.env.REDIS_URL && !client.isOpen) {
+      await client.connect();
+      console.log('✅ Model Router connected to Redis');
+    } else if (!process.env.REDIS_URL) {
+      console.warn('⚠️ REDIS_URL not set. Caching disabled.');
+    }
+  } catch (err) {
+    console.warn('⚠️ Failed to connect to Redis:', err.message);
+  } finally {
+    readyResolve();
+  }
+})();
+
+function isConnected() {
+  return client.isOpen;
+}
+
+module.exports = { client, isConnected, ready };

package/router.js

@@ -0,0 +1,218 @@
+// model-router/router.js
+//
+// The actual routing decision. Before this file existed, "the router"
+// only dispatched: it read the model name the caller already sent
+// (e.g. "claude-sonnet-4-5-20250929") and forwarded to whichever
+// provider owns that name. That's not routing - the caller had already
+// made the choice. This file adds the thing the product is named for:
+// given a REQUEST FOR A CAPABILITY (not a specific vendor's model), pick
+// a currently-healthy provider that can serve it - by cost (default),
+// by latency, or by cost with a latency guard rail; see ROUTER_STRATEGY
+// below for what each one actually does and doesn't guarantee.
+//
+// Backward compatibility is deliberate: a caller that already names a
+// concrete model (any name not starting with "router:") is dispatched
+// exactly as before, unchanged, in server.js. Nothing here overrides an
+// explicit choice - virtual models are opt-in.
+
+const anthropicProvider = require('./providers/anthropic');
+const openaiProvider = require('./providers/openai');
+const metrics = require('./metrics');
+
+// A tier groups equivalent-capability models across providers - the
+// deployer's judgment call about which models belong in the same
+// bucket, not a claim this router can verify. Within a tier, selection
+// is ALWAYS by estimated cost, full stop - never by some notion of
+// "quality." That used to be fuzzy: a tier named "router:best" implied
+// picking the best model, but pickCandidate() only ever compared cost,
+// so it silently picked whichever candidate was cheaper - cheaper, not
+// better. Naming it "frontier" instead of "best" says what's actually
+// true: this is the pool of frontier-capability models the deployer
+// trusts, and the router's only job is finding the cheapest healthy one
+// in that pool. If a deployment genuinely needs "always this specific
+// model regardless of price," that's what naming a concrete model
+// directly (skipping "router:" tiers entirely) is for.
+const DEFAULT_TIERS = {
+  'router:fast-cheap': [
+    { provider: 'openai', model: 'gpt-4o-mini' },
+    { provider: 'anthropic', model: 'claude-haiku-4-5-20251001' }
+  ],
+  'router:frontier': [
+    { provider: 'anthropic', model: 'claude-sonnet-4-5-20250929' },
+    { provider: 'openai', model: 'gpt-4o' }
+  ]
+};
+
+function loadTiers() {
+  if (!process.env.ROUTER_TIERS_JSON) return DEFAULT_TIERS;
+  try {
+    return JSON.parse(process.env.ROUTER_TIERS_JSON);
+  } catch (err) {
+    console.warn('⚠️ ROUTER_TIERS_JSON is not valid JSON, using defaults:', err.message);
+    return DEFAULT_TIERS;
+  }
+}
+
+// Three strategies, not one blended score. A weighted cost/latency
+// formula LOOKS more sophisticated but is really just a made-up
+// tradeoff dressed up as intelligence - whatever weights it used would
+// be a guess this router has no basis for making on the deployer's
+// behalf. These three are each simple enough to state exactly what they
+// do:
+//
+//   cost                 - (default, unchanged from before) cheapest
+//                          healthy candidate, full stop.
+//   latency              - fastest healthy candidate by recent average
+//                          latency, full stop. Cost isn't considered at
+//                          all except as a tiebreaker.
+//   latency-guarded-cost - cheapest healthy candidate, EXCLUDING any
+//                          candidate whose recent average latency is
+//                          more than ROUTER_LATENCY_GUARD_MULTIPLIER
+//                          (default 3x) slower than the fastest known
+//                          healthy candidate. A candidate with no
+//                          latency history yet is never excluded by the
+//                          guard - it hasn't had a chance to be slow.
+//                          This is the one genuinely "latency-aware"
+//                          option that still keeps cost as the primary
+//                          signal: it's a guard rail against picking
+//                          something dramatically slower to save a
+//                          fraction of a cent, not a full re-ranking.
+const VALID_STRATEGIES = ['cost', 'latency', 'latency-guarded-cost'];
+const DEFAULT_STRATEGY = 'cost';
+const LATENCY_GUARD_MULTIPLIER = Number(process.env.ROUTER_LATENCY_GUARD_MULTIPLIER) || 3;
+
+function loadStrategy() {
+  const raw = (process.env.ROUTER_STRATEGY || DEFAULT_STRATEGY).trim();
+  if (!VALID_STRATEGIES.includes(raw)) {
+    console.warn(`⚠️ Unknown ROUTER_STRATEGY "${raw}", falling back to "${DEFAULT_STRATEGY}". Valid values: ${VALID_STRATEGIES.join(', ')}`);
+    return DEFAULT_STRATEGY;
+  }
+  return raw;
+}
+
+function byCostAscending(a, b) {
+  return a.estimatedCostUsd - b.estimatedCostUsd;
+}
+
+function byLatencyThenCost(a, b) {
+  const aLatency = typeof a.avgLatencyMs === 'number' ? a.avgLatencyMs : Infinity;
+  const bLatency = typeof b.avgLatencyMs === 'number' ? b.avgLatencyMs : Infinity;
+  if (aLatency !== bLatency) return aLatency - bLatency;
+  return byCostAscending(a, b); // tiebreak: equal latency (often both unknown) falls back to cost
+}
+
+/**
+ * Applies the `latency-guarded-cost` guard: drops any candidate whose
+ * avgLatencyMs is more than LATENCY_GUARD_MULTIPLIER times the fastest
+ * KNOWN healthy candidate's latency. If there's no latency data to
+ * compare at all (a fresh deployment with no history yet), the guard
+ * has nothing to guard against and every candidate passes through
+ * unchanged - this strategy degrades to plain cost-only until real
+ * latency data exists.
+ */
+function applyLatencyGuard(pool) {
+  const knownLatencies = pool
+    .map((c) => c.avgLatencyMs)
+    .filter((v) => typeof v === 'number');
+  if (knownLatencies.length === 0) return pool;
+
+  const fastest = Math.min(...knownLatencies);
+  const guarded = pool.filter(
+    (c) => typeof c.avgLatencyMs !== 'number' || c.avgLatencyMs <= fastest * LATENCY_GUARD_MULTIPLIER
+  );
+  // The guard is a filter, not a veto - never let it eliminate every
+  // candidate (a tier with one badly-behaved provider should still
+  // route somewhere rather than error out).
+  return guarded.length > 0 ? guarded : pool;
+}
+
+function isVirtualModel(model) {
+  return typeof model === 'string' && model.startsWith('router:');
+}
+
+function estimatorFor(provider) {
+  if (provider === 'anthropic') return anthropicProvider.estimateCost;
+  if (provider === 'openai') return openaiProvider.estimateCost;
+  return null;
+}
+
+// A fixed token assumption used ONLY to compare candidates against each
+// other on a like-for-like basis (same assumed size for every
+// candidate) - it is not a prediction of this request's real size.
+const COMPARISON_INPUT_TOKENS = 1000;
+const COMPARISON_OUTPUT_TOKENS = 500;
+
+// A provider whose recent error rate is at or above this is treated as
+// unhealthy and skipped unless every candidate in the tier is unhealthy
+// (in which case we still have to pick one - see pickCandidate).
+const UNHEALTHY_ERROR_RATE = 0.5;
+
+/**
+ * Choose a {provider, model} pair for a virtual model name. Always
+ * excludes unhealthy candidates first (recent error rate too high,
+ * unless every candidate is unhealthy - see below); the strategy (env
+ * ROUTER_STRATEGY, default "cost") decides how what's left gets ranked.
+ * See the strategy comment above loadStrategy() for what each one
+ * actually does.
+ */
+async function pickCandidate(virtualModel) {
+  const tiers = loadTiers();
+  const candidates = tiers[virtualModel];
+  if (!candidates || candidates.length === 0) {
+    return { error: `Unknown routing tier: ${virtualModel}` };
+  }
+
+  const stats = await metrics.providerStats();
+
+  const scored = candidates.map((candidate) => {
+    const estimate = estimatorFor(candidate.provider);
+    const estimatedCostUsd = estimate
+      ? estimate(candidate.model, COMPARISON_INPUT_TOKENS, COMPARISON_OUTPUT_TOKENS)
+      : Infinity;
+    const providerStat = stats[candidate.provider] || { errorRate: 0, avgLatencyMs: null, sampleSize: 0 };
+    return {
+      ...candidate,
+      estimatedCostUsd,
+      errorRate: providerStat.errorRate,
+      avgLatencyMs: providerStat.avgLatencyMs,
+      healthy: providerStat.errorRate < UNHEALTHY_ERROR_RATE
+    };
+  });
+
+  const healthy = scored.filter((c) => c.healthy);
+  const pool = healthy.length > 0 ? healthy : scored; // all unhealthy: pick the least-bad rather than fail outright
+
+  const strategy = loadStrategy();
+  let ranked;
+  let guardApplied = false;
+  if (strategy === 'latency') {
+    ranked = [...pool].sort(byLatencyThenCost);
+  } else if (strategy === 'latency-guarded-cost') {
+    const guarded = applyLatencyGuard(pool);
+    guardApplied = guarded.length < pool.length;
+    ranked = [...guarded].sort(byCostAscending);
+  } else {
+    ranked = [...pool].sort(byCostAscending);
+  }
+
+  const chosen = ranked[0];
+  return {
+    provider: chosen.provider,
+    model: chosen.model,
+    // Same order `chosen` was drawn from, stripped down to just
+    // {provider, model} - lets a caller (server.js's failover loop)
+    // retry the next-best candidate if the top choice's live call
+    // fails, without re-running this scoring/health/strategy pass a
+    // second time. Always has at least one entry when `chosen` does.
+    rankedCandidates: ranked.map((c) => ({ provider: c.provider, model: c.model })),
+    reason: {
+      consideredTier: virtualModel,
+      strategy,
+      candidates: scored,
+      allUnhealthy: healthy.length === 0,
+      latencyGuardExcludedACandidate: guardApplied
+    }
+  };
+}
+
+module.exports = { isVirtualModel, pickCandidate, loadTiers, loadStrategy };

package/semanticCache.js

@@ -0,0 +1,154 @@
+// model-router/semanticCache.js
+//
+// Catches NEAR-duplicate prompts that cache.js's exact hash match
+// can't: a paraphrase of the same question, reordered context,
+// different whitespace. cache.js stays the first, free, zero-risk
+// check; this one only runs when that misses, and it costs something
+// real every time it runs - one embedding call - whether or not it
+// finds a match. That's a genuine tradeoff, not free money: it's worth
+// it only when near-duplicate traffic is common enough that avoiding
+// the occasional full completion call outweighs the embedding calls
+// spent looking. See the README for the honest framing of what this
+// can and can't claim.
+//
+// Storage: a plain Redis LIST per model, no RediSearch/vector-search
+// module assumed - most self-hosted Redis instances (including
+// Render's managed Redis) don't have that module. A lookup pulls up to
+// MAX_CANDIDATES_PER_MODEL recent entries for that model and computes
+// cosine similarity IN NODE, not in Redis. This is brute-force, not
+// indexed - fine at the volume a self-hosted single instance sees, not
+// meant to scale past that cap. A real vector index is the honest next
+// step if traffic outgrows it.
+//
+// The threshold is a probabilistic judgment call, not a guarantee: a
+// "hit" above the threshold is the router's best guess that two
+// prompts want the same answer, not proof they do. Set it too low and
+// it returns confidently wrong answers - the same failure mode that
+// makes vendor-claimed 90%+ cache hit rates suspect (see this project's
+// own market research on real vs. advertised hit rates). Every
+// semantic hit is tracked separately from an exact hit in metrics.js /
+// GET /stats for exactly this reason - the two are not equally
+// trustworthy and shouldn't be blended into one inflated number.
+
+const redis = require('./redisClient');
+const embeddingsDefault = require('./embeddings');
+
+const MAX_CANDIDATES_PER_MODEL = Number(process.env.SEMANTIC_CACHE_MAX_CANDIDATES) || 200;
+const DEFAULT_TTL_SECONDS = Number(process.env.SEMANTIC_CACHE_TTL_SECONDS) || 3600;
+const DEFAULT_THRESHOLD = Number(process.env.SEMANTIC_CACHE_THRESHOLD) || 0.93;
+
+function listKey(model) {
+  return `SEMANTIC_LIST:${model}`;
+}
+
+function cosineSimilarity(a, b) {
+  if (!a || !b || a.length !== b.length) return 0;
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  if (normA === 0 || normB === 0) return 0;
+  return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+
+/**
+ * The text a semantic match is based on: just the conversation content,
+ * not incidental request parameters (temperature, max_tokens) that
+ * don't change what's actually being asked.
+ */
+function extractPromptText(payload) {
+  return payload.messages
+    .map((m) => `${m.role}: ${typeof m.content === 'string' ? m.content : JSON.stringify(m.content)}`)
+    .join('\n');
+}
+
+function isEnabled(embeddings = embeddingsDefault) {
+  if (process.env.SEMANTIC_CACHE_ENABLED === 'false') return false;
+  return redis.isConnected() && embeddings.isEnabled();
+}
+
+// Tool-calling requests are excluded from semantic caching: an
+// approximate text match can't guarantee the exact argument values a
+// tool call needs, and returning a plausible-but-wrong tool call is a
+// worse failure than a cache miss.
+function isCacheable(payload) {
+  return !payload.tools;
+}
+
+async function findMatch(payload, { threshold = DEFAULT_THRESHOLD, embeddings = embeddingsDefault } = {}) {
+  if (!isEnabled(embeddings) || !isCacheable(payload)) return null;
+
+  let queryEmbedding;
+  try {
+    queryEmbedding = await embeddings.embed(extractPromptText(payload));
+  } catch (err) {
+    console.warn('⚠️ Semantic cache lookup failed to embed, skipping:', err.message);
+    return null;
+  }
+
+  let raw;
+  try {
+    raw = await redis.client.lRange(listKey(payload.model), 0, MAX_CANDIDATES_PER_MODEL - 1);
+  } catch (err) {
+    console.warn('⚠️ Semantic cache lookup failed:', err.message);
+    return null;
+  }
+
+  let best = null;
+  for (const line of raw) {
+    let record;
+    try {
+      record = JSON.parse(line);
+    } catch {
+      continue; // a malformed entry is skipped, not fatal
+    }
+    const similarity = cosineSimilarity(queryEmbedding, record.embedding);
+    if (similarity >= threshold && (!best || similarity > best.similarity)) {
+      best = { entry: record.entry, similarity };
+    }
+  }
+  return best;
+}
+
+async function store(payload, entry, { ttlSeconds = DEFAULT_TTL_SECONDS, embeddings = embeddingsDefault } = {}) {
+  if (!isEnabled(embeddings) || !isCacheable(payload)) return false;
+
+  let embedding;
+  try {
+    embedding = await embeddings.embed(extractPromptText(payload));
+  } catch (err) {
+    console.warn('⚠️ Semantic cache store failed to embed, skipping:', err.message);
+    return false;
+  }
+
+  const key = listKey(payload.model);
+  try {
+    await redis.client.lPush(key, JSON.stringify({ embedding, entry, storedAt: Date.now() }));
+    await redis.client.lTrim(key, 0, MAX_CANDIDATES_PER_MODEL - 1);
+    // A rolling TTL on the whole per-model bucket, reset on every
+    // store - simple and predictable (as long as there's traffic to
+    // that model, the bucket stays warm; if it goes quiet for
+    // ttlSeconds, the whole bucket - old and new entries alike -
+    // expires together), not a precise per-entry TTL. Documented
+    // tradeoff, not an oversight.
+    await redis.client.expire(key, ttlSeconds);
+    return true;
+  } catch (err) {
+    console.warn('⚠️ Semantic cache store failed:', err.message);
+    return false;
+  }
+}
+
+module.exports = {
+  isEnabled,
+  isCacheable,
+  findMatch,
+  store,
+  cosineSimilarity,
+  extractPromptText,
+  DEFAULT_THRESHOLD
+};