cachegate 1.3.0 → 1.3.1

package/LICENSE

@@ -1,21 +1,21 @@
-MIT License
-
-Copyright (c) 2026 MemoCode
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+MIT License
+
+Copyright (c) 2026 MemoCode
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/failover.js

@@ -1,76 +1,76 @@
-// model-router/failover.js
-//
-// Pure control flow for trying a virtual model's ranked candidates in
-// order until one succeeds - isolated from the actual provider-calling
-// code (server.js's dispatchToProvider) so it's directly unit-testable
-// with a fake dispatch function, no live API keys or network calls
-// needed. Same reasoning as providers/*.js's own
-// applyStreamEvent/applyStreamChunk factoring: the trickiest logic
-// shouldn't require a real provider to test.
-//
-// Addresses ROADMAP.md's gap #2 ("no provider failover on a
-// 5xx/rate-limit"): router.js's pickCandidate() already ranks every
-// candidate in a tier by the configured strategy, but server.js used
-// to dispatch to the top-ranked one only - if it failed, the whole
-// request failed, even when a second healthy candidate existed in the
-// same tier. This module is what actually walks that ranked list.
-
-// Whether a failed dispatch attempt is worth retrying against the NEXT
-// candidate, vs failing the request outright. The distinction: is the
-// REQUEST itself broken (retrying elsewhere would fail identically),
-// or did THIS provider fail in a way another provider might not (rate
-// limit, an outage, a bad or expired key)? Bad request (400) and
-// unknown model (404) are the request's own fault, not retried - the
-// Anthropic and OpenAI SDKs both set `.status` on a thrown APIError.
-// A network-level failure with no HTTP response at all (no `.status`)
-// is treated as the provider's fault too, since it's not the
-// request's content that's the problem. An auth failure (401) or
-// missing-key misconfiguration is ALSO treated as retryable on
-// purpose: a different candidate in the tier may use a different
-// provider whose key is fine, so the request can still succeed - the
-// broken key itself still surfaces on the dashboard's Provider alerts
-// table via the metrics.record() call made before moving on (see
-// server.js), so failover keeps requests succeeding without hiding
-// the underlying problem from whoever needs to go fix that key.
-function isRetryableError(err) {
-  const status = err && (err.status || err.statusCode);
-  return status !== 400 && status !== 404;
-}
-
-/**
- * Calls `dispatch(candidate)` for each candidate in order until one
- * resolves. On a rejection, calls `onAttemptFailed(candidate, err,
- * isLastCandidate)` (for logging/metrics only - it has no bearing on
- * control flow) and, unless the error is non-retryable or this was
- * the last candidate, moves on to the next one. Rethrows the error
- * from the LAST attempt if every candidate fails - the caller decides
- * what HTTP status/response that becomes.
- *
- * Resolves to `{ result, candidate, attempts }` on success -
- * `attempts` is 1 when the first candidate just worked, >1 when
- * failover actually happened (worth logging distinctly - see
- * server.js's caller).
- *
- * @param {Array<{provider: string, model: string}>} candidates ranked
- *   order, e.g. router.js's pickCandidate().rankedCandidates
- * @param {(candidate) => Promise<any>} dispatch
- * @param {(candidate, err, isLastCandidate) => void} [onAttemptFailed]
- */
-async function dispatchWithFailover(candidates, dispatch, onAttemptFailed) {
-  let lastErr;
-  for (let i = 0; i < candidates.length; i++) {
-    const candidate = candidates[i];
-    const isLastCandidate = i === candidates.length - 1;
-    try {
-      const result = await dispatch(candidate);
-      return { result, candidate, attempts: i + 1 };
-    } catch (err) {
-      lastErr = err;
-      if (onAttemptFailed) onAttemptFailed(candidate, err, isLastCandidate);
-      if (!isRetryableError(err) || isLastCandidate) throw err;
-    }
-  }
-  throw lastErr; // unreachable when candidates.length > 0; kept honest for an empty list
-}
-
-module.exports = { isRetryableError, dispatchWithFailover };
+// model-router/failover.js
+//
+// Pure control flow for trying a virtual model's ranked candidates in
+// order until one succeeds - isolated from the actual provider-calling
+// code (server.js's dispatchToProvider) so it's directly unit-testable
+// with a fake dispatch function, no live API keys or network calls
+// needed. Same reasoning as providers/*.js's own
+// applyStreamEvent/applyStreamChunk factoring: the trickiest logic
+// shouldn't require a real provider to test.
+//
+// Addresses ROADMAP.md's gap #2 ("no provider failover on a
+// 5xx/rate-limit"): router.js's pickCandidate() already ranks every
+// candidate in a tier by the configured strategy, but server.js used
+// to dispatch to the top-ranked one only - if it failed, the whole
+// request failed, even when a second healthy candidate existed in the
+// same tier. This module is what actually walks that ranked list.
+
+// Whether a failed dispatch attempt is worth retrying against the NEXT
+// candidate, vs failing the request outright. The distinction: is the
+// REQUEST itself broken (retrying elsewhere would fail identically),
+// or did THIS provider fail in a way another provider might not (rate
+// limit, an outage, a bad or expired key)? Bad request (400) and
+// unknown model (404) are the request's own fault, not retried - the
+// Anthropic and OpenAI SDKs both set `.status` on a thrown APIError.
+// A network-level failure with no HTTP response at all (no `.status`)
+// is treated as the provider's fault too, since it's not the
+// request's content that's the problem. An auth failure (401) or
+// missing-key misconfiguration is ALSO treated as retryable on
+// purpose: a different candidate in the tier may use a different
+// provider whose key is fine, so the request can still succeed - the
+// broken key itself still surfaces on the dashboard's Provider alerts
+// table via the metrics.record() call made before moving on (see
+// server.js), so failover keeps requests succeeding without hiding
+// the underlying problem from whoever needs to go fix that key.
+function isRetryableError(err) {
+  const status = err && (err.status || err.statusCode);
+  return status !== 400 && status !== 404;
+}
+
+/**
+ * Calls `dispatch(candidate)` for each candidate in order until one
+ * resolves. On a rejection, calls `onAttemptFailed(candidate, err,
+ * isLastCandidate)` (for logging/metrics only - it has no bearing on
+ * control flow) and, unless the error is non-retryable or this was
+ * the last candidate, moves on to the next one. Rethrows the error
+ * from the LAST attempt if every candidate fails - the caller decides
+ * what HTTP status/response that becomes.
+ *
+ * Resolves to `{ result, candidate, attempts }` on success -
+ * `attempts` is 1 when the first candidate just worked, >1 when
+ * failover actually happened (worth logging distinctly - see
+ * server.js's caller).
+ *
+ * @param {Array<{provider: string, model: string}>} candidates ranked
+ *   order, e.g. router.js's pickCandidate().rankedCandidates
+ * @param {(candidate) => Promise<any>} dispatch
+ * @param {(candidate, err, isLastCandidate) => void} [onAttemptFailed]
+ */
+async function dispatchWithFailover(candidates, dispatch, onAttemptFailed) {
+  let lastErr;
+  for (let i = 0; i < candidates.length; i++) {
+    const candidate = candidates[i];
+    const isLastCandidate = i === candidates.length - 1;
+    try {
+      const result = await dispatch(candidate);
+      return { result, candidate, attempts: i + 1 };
+    } catch (err) {
+      lastErr = err;
+      if (onAttemptFailed) onAttemptFailed(candidate, err, isLastCandidate);
+      if (!isRetryableError(err) || isLastCandidate) throw err;
+    }
+  }
+  throw lastErr; // unreachable when candidates.length > 0; kept honest for an empty list
+}
+
+module.exports = { isRetryableError, dispatchWithFailover };

package/metrics.js

@@ -358,8 +358,10 @@ function ensureWriteStream() {
  */
 function record(scope, entry) {
   if (usingPostgres()) {
-    recordToPostgres(scope, entry); // fire-and-forget - see its own comment
-    return;
+    // Returns the INSERT promise so tests/scripts can await it; request
+    // paths ignore it, so it stays fire-and-forget for them (a metrics
+    // write must never be the reason a real request fails or slows down).
+    return recordToPostgres(scope, entry);
   }
   try {
     const line = JSON.stringify({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cachegate",
-  "version": "1.3.0",
+  "version": "1.3.1",
   "description": "Self-hostable, OpenAI-compatible LLM proxy: routes to the cheapest healthy provider, caches responses exactly and semantically, tracks cost and latency per call.",
   "license": "MIT",
   "main": "server.js",

package/providers/anthropic.js

@@ -1,122 +1,122 @@
-// model-router/providers/anthropic.js
-const { Anthropic } = require('@anthropic-ai/sdk');
-
-function buildClient(apiKey) {
-  return new Anthropic({ apiKey });
-}
-
-function estimateCost(model, inputTokens, outputTokens) {
-  // Approximate pricing per 1M tokens — update as Anthropic changes rates
-  const rates = {
-    'claude-sonnet-4-5-20250929': { input: 3.0, output: 15.0 },
-    'claude-haiku-4-5-20251001': { input: 0.8, output: 4.0 },
-    'claude-3-5-sonnet-20241022': { input: 3.0, output: 15.0 },
-    'claude-3-5-sonnet-20240620': { input: 3.0, output: 15.0 }
-  };
-  const rate = rates[model] || { input: 3.0, output: 15.0 };
-  return ((inputTokens * rate.input) + (outputTokens * rate.output)) / 1_000_000;
-}
-
-async function chat(client, payload) {
-  const systemMessage = payload.messages.find(m => m.role === 'system');
-  const userMessages = payload.messages.filter(m => m.role !== 'system');
-
-  const request = {
-    model: payload.model,
-    max_tokens: payload.max_tokens || 1024,
-    temperature: typeof payload.temperature === 'number' ? payload.temperature : 0.0,
-    messages: userMessages,
-    ...(systemMessage && { system: systemMessage.content }),
-    ...(payload.tools && { tools: payload.tools }),
-    ...(payload.tool_choice && { tool_choice: payload.tool_choice })
-  };
-
-  const start = Date.now();
-  const response = await client.messages.create(request);
-  const latencyMs = Date.now() - start;
-
-  const inputTokens = response.usage.input_tokens;
-  const outputTokens = response.usage.output_tokens;
-  const costUsd = estimateCost(payload.model, inputTokens, outputTokens);
-
-  const toolCall = response.content.find(c => c.type === 'tool_use');
-  const textContent = response.content
-    .filter(c => c.type === 'text')
-    .map(c => c.text)
-    .join('');
-
-  return {
-    provider: 'anthropic',
-    model: payload.model,
-    latency_ms: latencyMs,
-    usage: { input_tokens: inputTokens, output_tokens: outputTokens },
-    cost_usd: costUsd,
-    content: textContent,
-    tool_calls: toolCall ? [toolCall] : undefined,
-    raw: response
-  };
-}
-
-/**
- * Pure state-accumulation for one Anthropic streaming event - factored
- * out from chatStream() so the trickiest part (pulling usage/cost data
- * out of a stream instead of one final response object) is directly
- * unit-testable with canned events, no live API needed. Mutates
- * `state` ({content, inputTokens, outputTokens}) and calls onDelta()
- * with each new piece of assistant text.
- */
-function applyStreamEvent(state, event, onDelta) {
-  if (event.type === 'message_start') {
-    state.inputTokens = event.message.usage.input_tokens;
-    state.outputTokens = event.message.usage.output_tokens || 0;
-  } else if (event.type === 'content_block_delta' && event.delta && event.delta.type === 'text_delta') {
-    state.content += event.delta.text;
-    onDelta(event.delta.text);
-  } else if (event.type === 'message_delta' && event.usage) {
-    // Anthropic reports output_tokens progressively here; the last one
-    // received before message_stop is the final total.
-    state.outputTokens = event.usage.output_tokens;
-  }
-}
-
-/**
- * Streaming counterpart to chat(). Scope: plain text content only - no
- * tools/tool_choice forwarded (server.js rejects stream:true + tools
- * before this is ever called; see streaming.js for why).
- */
-async function chatStream(client, payload, { onDelta, signal } = {}) {
-  const systemMessage = payload.messages.find(m => m.role === 'system');
-  const userMessages = payload.messages.filter(m => m.role !== 'system');
-
-  const request = {
-    model: payload.model,
-    max_tokens: payload.max_tokens || 1024,
-    temperature: typeof payload.temperature === 'number' ? payload.temperature : 0.0,
-    messages: userMessages,
-    ...(systemMessage && { system: systemMessage.content }),
-    stream: true
-  };
-
-  const start = Date.now();
-  const stream = await client.messages.create(request, signal ? { signal } : undefined);
-
-  const state = { content: '', inputTokens: 0, outputTokens: 0 };
-  for await (const event of stream) {
-    applyStreamEvent(state, event, onDelta || (() => {}));
-  }
-
-  const latencyMs = Date.now() - start;
-  const costUsd = estimateCost(payload.model, state.inputTokens, state.outputTokens);
-
-  return {
-    provider: 'anthropic',
-    model: payload.model,
-    latency_ms: latencyMs,
-    usage: { input_tokens: state.inputTokens, output_tokens: state.outputTokens },
-    cost_usd: costUsd,
-    content: state.content,
-    tool_calls: undefined
-  };
-}
-
-module.exports = { buildClient, chat, chatStream, applyStreamEvent, estimateCost };
+// model-router/providers/anthropic.js
+const { Anthropic } = require('@anthropic-ai/sdk');
+
+function buildClient(apiKey) {
+  return new Anthropic({ apiKey });
+}
+
+function estimateCost(model, inputTokens, outputTokens) {
+  // Approximate pricing per 1M tokens — update as Anthropic changes rates
+  const rates = {
+    'claude-sonnet-4-5-20250929': { input: 3.0, output: 15.0 },
+    'claude-haiku-4-5-20251001': { input: 0.8, output: 4.0 },
+    'claude-3-5-sonnet-20241022': { input: 3.0, output: 15.0 },
+    'claude-3-5-sonnet-20240620': { input: 3.0, output: 15.0 }
+  };
+  const rate = rates[model] || { input: 3.0, output: 15.0 };
+  return ((inputTokens * rate.input) + (outputTokens * rate.output)) / 1_000_000;
+}
+
+async function chat(client, payload) {
+  const systemMessage = payload.messages.find(m => m.role === 'system');
+  const userMessages = payload.messages.filter(m => m.role !== 'system');
+
+  const request = {
+    model: payload.model,
+    max_tokens: payload.max_tokens || 1024,
+    temperature: typeof payload.temperature === 'number' ? payload.temperature : 0.0,
+    messages: userMessages,
+    ...(systemMessage && { system: systemMessage.content }),
+    ...(payload.tools && { tools: payload.tools }),
+    ...(payload.tool_choice && { tool_choice: payload.tool_choice })
+  };
+
+  const start = Date.now();
+  const response = await client.messages.create(request);
+  const latencyMs = Date.now() - start;
+
+  const inputTokens = response.usage.input_tokens;
+  const outputTokens = response.usage.output_tokens;
+  const costUsd = estimateCost(payload.model, inputTokens, outputTokens);
+
+  const toolCall = response.content.find(c => c.type === 'tool_use');
+  const textContent = response.content
+    .filter(c => c.type === 'text')
+    .map(c => c.text)
+    .join('');
+
+  return {
+    provider: 'anthropic',
+    model: payload.model,
+    latency_ms: latencyMs,
+    usage: { input_tokens: inputTokens, output_tokens: outputTokens },
+    cost_usd: costUsd,
+    content: textContent,
+    tool_calls: toolCall ? [toolCall] : undefined,
+    raw: response
+  };
+}
+
+/**
+ * Pure state-accumulation for one Anthropic streaming event - factored
+ * out from chatStream() so the trickiest part (pulling usage/cost data
+ * out of a stream instead of one final response object) is directly
+ * unit-testable with canned events, no live API needed. Mutates
+ * `state` ({content, inputTokens, outputTokens}) and calls onDelta()
+ * with each new piece of assistant text.
+ */
+function applyStreamEvent(state, event, onDelta) {
+  if (event.type === 'message_start') {
+    state.inputTokens = event.message.usage.input_tokens;
+    state.outputTokens = event.message.usage.output_tokens || 0;
+  } else if (event.type === 'content_block_delta' && event.delta && event.delta.type === 'text_delta') {
+    state.content += event.delta.text;
+    onDelta(event.delta.text);
+  } else if (event.type === 'message_delta' && event.usage) {
+    // Anthropic reports output_tokens progressively here; the last one
+    // received before message_stop is the final total.
+    state.outputTokens = event.usage.output_tokens;
+  }
+}
+
+/**
+ * Streaming counterpart to chat(). Scope: plain text content only - no
+ * tools/tool_choice forwarded (server.js rejects stream:true + tools
+ * before this is ever called; see streaming.js for why).
+ */
+async function chatStream(client, payload, { onDelta, signal } = {}) {
+  const systemMessage = payload.messages.find(m => m.role === 'system');
+  const userMessages = payload.messages.filter(m => m.role !== 'system');
+
+  const request = {
+    model: payload.model,
+    max_tokens: payload.max_tokens || 1024,
+    temperature: typeof payload.temperature === 'number' ? payload.temperature : 0.0,
+    messages: userMessages,
+    ...(systemMessage && { system: systemMessage.content }),
+    stream: true
+  };
+
+  const start = Date.now();
+  const stream = await client.messages.create(request, signal ? { signal } : undefined);
+
+  const state = { content: '', inputTokens: 0, outputTokens: 0 };
+  for await (const event of stream) {
+    applyStreamEvent(state, event, onDelta || (() => {}));
+  }
+
+  const latencyMs = Date.now() - start;
+  const costUsd = estimateCost(payload.model, state.inputTokens, state.outputTokens);
+
+  return {
+    provider: 'anthropic',
+    model: payload.model,
+    latency_ms: latencyMs,
+    usage: { input_tokens: state.inputTokens, output_tokens: state.outputTokens },
+    cost_usd: costUsd,
+    content: state.content,
+    tool_calls: undefined
+  };
+}
+
+module.exports = { buildClient, chat, chatStream, applyStreamEvent, estimateCost };