cachegate 1.3.0 → 1.4.0

package/providers/deepseek.js

@@ -0,0 +1,194 @@
+// model-router/providers/deepseek.js
+//
+// DeepSeek speaks the OpenAI wire format, so this is the `openai` SDK pointed
+// at DeepSeek's own base URL - no second HTTP client, no hand-rolled fetch.
+// What is NOT identical is the money: DeepSeek bills input in two tiers
+// (cache hit vs cache miss) and every rate has a PEAK and an OFF-PEAK value,
+// so a single flat rate table - the shape providers/openai.js can get away
+// with - would misprice most requests. Both facts come from the vendor's own
+// pages, read 2026-09-10:
+//   base URL + models + rates: api-docs.deepseek.com/quick_start/pricing
+//   usage fields:              api-docs.deepseek.com/guides/kv_cache
+//
+// The review that prompted this file said "DeepSeek features incredibly cheap
+// API calls and aggressive server-side prompt caching" and told the reader to
+// "parse cached_tokens from DeepSeek's usage response blocks". The second half
+// is wrong in a way that would have silently zeroed the savings: DeepSeek does
+// not report OpenAI's nested prompt_tokens_details.cached_tokens, it reports a
+// flat prompt_cache_hit_tokens / prompt_cache_miss_tokens pair. Cached input
+// here is ~50x cheaper than a miss, so getting that mapping wrong is the
+// difference between an accurate cost dashboard and a decorative one.
+const { OpenAI } = require('openai');
+
+const BASE_URL = process.env.DEEPSEEK_BASE_URL || 'https://api.deepseek.com';
+
+// USD per 1M tokens, PEAK rates. Off-peak is exactly half (vendor's own note:
+// "Off-peak rates are half of the peak rates"). Peak = 01:00-04:00 and
+// 06:00-10:00 UTC, Monday through Friday; everything else is off-peak.
+const PEAK_RATES = {
+  'deepseek-flash':                 { hit: 0.006, miss: 0.30, output: 1.20 },
+  'deepseek-v4-pro':                { hit: 0.044, miss: 1.32, output: 3.96 },
+  // Legacy names the API still accepts. The vendor states these are served by
+  // V4.1-Flash and BILLED AT THE FLASH PRICE, so they must not fall through to
+  // a "unknown model" default with different numbers.
+  'deepseek-v4-flash':              { hit: 0.006, miss: 0.30, output: 1.20 },
+  'deepseek-v4-flash-vision-exp':   { hit: 0.006, miss: 0.30, output: 1.20 }
+};
+// `deepseek-v4-pro` is being retired: from 2026-09-14 requests to it are routed
+// to V4.1-Flash and billed as Flash. Until that date the pro rate is real, so
+// both are priced and the switch happens on the vendor's side, not ours.
+const DEFAULT_RATE = PEAK_RATES['deepseek-flash'];
+
+function isPeak(now = new Date()) {
+  const day = now.getUTCDay();            // 0 = Sunday, 6 = Saturday
+  if (day === 0 || day === 6) return false;
+  const hour = now.getUTCHours();
+  return (hour >= 1 && hour < 4) || (hour >= 6 && hour < 10);
+}
+
+function rateFor(model, now = new Date()) {
+  const base = PEAK_RATES[model] || DEFAULT_RATE;
+  return isPeak(now) ? base : { hit: base.hit / 2, miss: base.miss / 2, output: base.output / 2 };
+}
+
+/**
+ * Cost for one call. Input tokens are SPLIT, not uniform: `cacheHitTokens` of
+ * them were served from DeepSeek's disk cache at the hit rate, the remainder at
+ * the miss rate. Getting this wrong is not a rounding error - at Flash's peak
+ * rates a fully-cached 1M-token input costs $0.006 instead of $0.30.
+ *
+ * The 3-argument form (no options) is kept working for callers that have no
+ * cache data: it prices everything as a miss, which is the conservative
+ * (never-understate) direction for a cost dashboard.
+ *
+ * `now` is injectable so tests can pin the billing window instead of passing
+ * or failing depending on what time of day the suite runs.
+ */
+function estimateCost(model, inputTokens, outputTokens, { cacheHitTokens = 0, now = new Date() } = {}) {
+  const rate = rateFor(model, now);
+  const hit = Math.max(0, Math.min(cacheHitTokens || 0, inputTokens || 0));
+  const miss = Math.max(0, (inputTokens || 0) - hit);
+  return ((hit * rate.hit) + (miss * rate.miss) + ((outputTokens || 0) * rate.output)) / 1_000_000;
+}
+
+function buildClient(apiKey) {
+  return new OpenAI({
+    apiKey,
+    baseURL: BASE_URL,
+    // DeepSeek requires no extra headers; the SDK's defaults are fine.
+    timeout: Number(process.env.DEEPSEEK_TIMEOUT_MS) || 60000
+  });
+}
+
+function usageFrom(usage = {}) {
+  const input = usage.prompt_tokens || 0;
+  const hit = usage.prompt_cache_hit_tokens || 0;
+  const miss = typeof usage.prompt_cache_miss_tokens === 'number'
+    ? usage.prompt_cache_miss_tokens
+    : Math.max(0, input - hit);
+  return { input_tokens: input, output_tokens: usage.completion_tokens || 0, cache_hit_tokens: hit, cache_miss_tokens: miss };
+}
+
+async function chat(client, payload, options = {}) {
+  const request = {
+    model: payload.model,
+    messages: payload.messages,
+    temperature: typeof payload.temperature === 'number' ? payload.temperature : 0.0,
+    max_tokens: payload.max_tokens || 1024,
+    ...(payload.tools && { tools: payload.tools }),
+    ...(payload.tool_choice && { tool_choice: payload.tool_choice }),
+    ...(payload.response_format && { response_format: payload.response_format })
+  };
+
+  const start = Date.now();
+  const response = await client.chat.completions.create(request);
+  const latencyMs = Date.now() - start;
+
+  const choice = response.choices[0];
+  const usage = usageFrom(response.usage);
+  const costUsd = estimateCost(payload.model, usage.input_tokens, usage.output_tokens, { cacheHitTokens: usage.cache_hit_tokens });
+
+  return {
+    provider: 'deepseek',
+    model: payload.model,
+    latency_ms: latencyMs,
+    usage,
+    cost_usd: costUsd,
+    content: choice.message.content || '',
+    // Thinking-mode models (the default for the current models) return their
+    // chain separately. Passed through instead of dropped so a caller can see
+    // it, but NOT concatenated into `content` - that would corrupt every
+    // consumer that treats content as the answer.
+    reasoning_content: choice.message.reasoning_content,
+    tool_calls: choice.message.tool_calls,
+    raw: response
+  };
+}
+
+/**
+ * Pure state-accumulation for one streamed chunk, factored out for the same
+ * reason as providers/openai.js's: usage/cost extraction is then unit-testable
+ * against canned chunks with no live API.
+ *
+ * DeepSeek only sends `usage` on the final chunk, and only when the request
+ * asked for it (`stream_options.include_usage`, set in chatStream below) -
+ * without that flag a streamed call carries no usage at all and the cost
+ * tracking this project is built around would sit at zero while looking fine.
+ */
+function applyStreamChunk(state, chunk, onDelta) {
+  const choice = chunk.choices && chunk.choices[0];
+  if (choice && choice.delta) {
+    if (choice.delta.content) {
+      state.content += choice.delta.content;
+      onDelta(choice.delta.content);
+    }
+    if (choice.delta.reasoning_content) state.reasoningContent = (state.reasoningContent || '') + choice.delta.reasoning_content;
+  }
+  if (chunk.usage) {
+    state.inputTokens = chunk.usage.prompt_tokens || 0;
+    state.outputTokens = chunk.usage.completion_tokens || 0;
+    state.cacheHitTokens = chunk.usage.prompt_cache_hit_tokens || 0;
+    state.cacheMissTokens = chunk.usage.prompt_cache_miss_tokens;
+  }
+}
+
+async function chatStream(client, payload, { onDelta, signal } = {}) {
+  const request = {
+    model: payload.model,
+    messages: payload.messages,
+    temperature: typeof payload.temperature === 'number' ? payload.temperature : 0.0,
+    max_tokens: payload.max_tokens || 1024,
+    stream: true,
+    stream_options: { include_usage: true }
+  };
+
+  const start = Date.now();
+  const stream = await client.chat.completions.create(request, signal ? { signal } : undefined);
+
+  const state = { content: '', inputTokens: 0, outputTokens: 0, cacheHitTokens: 0, reasoningContent: '' };
+  for await (const chunk of stream) {
+    applyStreamChunk(state, chunk, onDelta || (() => {}));
+  }
+
+  const latencyMs = Date.now() - start;
+  const usage = usageFrom({
+    prompt_tokens: state.inputTokens,
+    completion_tokens: state.outputTokens,
+    prompt_cache_hit_tokens: state.cacheHitTokens,
+    ...(typeof state.cacheMissTokens === 'number' && { prompt_cache_miss_tokens: state.cacheMissTokens })
+  });
+  const costUsd = estimateCost(payload.model, usage.input_tokens, usage.output_tokens, { cacheHitTokens: usage.cache_hit_tokens });
+
+  return {
+    provider: 'deepseek',
+    model: payload.model,
+    latency_ms: latencyMs,
+    usage,
+    cost_usd: costUsd,
+    content: state.content,
+    reasoning_content: state.reasoningContent || undefined,
+    tool_calls: undefined
+  };
+}
+
+module.exports = { buildClient, chat, chatStream, applyStreamChunk, estimateCost, isPeak, rateFor, BASE_URL };

package/providers/index.js

@@ -0,0 +1,62 @@
+// model-router/providers/index.js
+//
+// The provider registry. Before this file, "which provider serves this model?"
+// was answered by an if/else chain repeated in eight places across server.js
+// (detection, two dispatch forks, the streaming client + stream-function
+// forks, the cascade grader, a second tier branch, and a metrics label). Every
+// one of them knew exactly two providers, so adding a third meant finding and
+// patching all eight - and missing one produced a weird failure far from the
+// edit, not an error at the place that was forgotten.
+//
+// Adding a provider is now: write providers/<name>.js to the contract
+// (buildClient/chat/chatStream/estimateCost), require it here, give it a model
+// prefix and an env key. Everything else routes through this file.
+const anthropic = require('./anthropic');
+const openai = require('./openai');
+const deepseek = require('./deepseek');
+const openrouter = require('./openrouter');
+
+const PROVIDERS = { anthropic, openai, deepseek, openrouter };
+
+// The env var holding each provider's key. Used for both presence checks and
+// the "not configured" error message, so those two can never disagree about
+// which variable a provider actually needs.
+const ENV_KEYS = {
+  anthropic: 'ANTHROPIC_API_KEY',
+  openai: 'OPENAI_API_KEY',
+  deepseek: 'DEEPSEEK_API_KEY',
+  openrouter: 'OPENROUTER_API_KEY'
+};
+
+/**
+ * Which provider serves this model? Returns null when nothing claims it, which
+ * callers turn into a 400 "Unsupported model" - the same answer as before for
+ * anything unmatched.
+ *
+ * ORDER MATTERS: OpenRouter is checked first because its ids are
+ * `vendor/model`, and `deepseek/deepseek-chat` must not be captured by the
+ * direct DeepSeek prefix. (The direct DeepSeek ids are `deepseek-flash` /
+ * `deepseek-v4-pro` - no slash - so the two never actually collide, but the
+ * ordering is what guarantees that stays true if either vendor renames.)
+ */
+function detectProvider(model) {
+  if (typeof model !== 'string' || !model) return null;
+  if (openrouter.isOpenRouterModel(model)) return 'openrouter';
+  if (model.startsWith('claude-')) return 'anthropic';
+  if (model.startsWith('deepseek-')) return 'deepseek';
+  if (model.startsWith('gpt-') || model.startsWith('o1') || model.startsWith('o3')) return 'openai';
+  return null;
+}
+
+function get(name) { return PROVIDERS[name] || null; }
+
+function envKey(name) { return ENV_KEYS[name] || null; }
+
+function names() { return Object.keys(PROVIDERS); }
+
+/** Provider names that look usable given the current environment (presence only, never values). */
+function configured(env = process.env) {
+  return names().filter((n) => !!env[ENV_KEYS[n]]);
+}
+
+module.exports = { PROVIDERS, ENV_KEYS, detectProvider, get, envKey, names, configured };

package/providers/openai.js

@@ -1,114 +1,122 @@
-// model-router/providers/openai.js
-const { OpenAI } = require('openai');
-
-function buildClient(apiKey) {
-  return new OpenAI({ apiKey });
-}
-
-function estimateCost(model, inputTokens, outputTokens) {
-  // Approximate pricing per 1M tokens
-  const rates = {
-    'gpt-4o-mini': { input: 0.15, output: 0.6 },
-    'gpt-4o': { input: 2.5, output: 10.0 }
-  };
-  const rate = rates[model] || { input: 2.5, output: 10.0 };
-  return ((inputTokens * rate.input) + (outputTokens * rate.output)) / 1_000_000;
-}
-
-async function chat(client, payload) {
-  const request = {
-    model: payload.model,
-    messages: payload.messages,
-    temperature: typeof payload.temperature === 'number' ? payload.temperature : 0.0,
-    max_tokens: payload.max_tokens || 1024,
-    ...(payload.tools && { tools: payload.tools }),
-    ...(payload.tool_choice && { tool_choice: payload.tool_choice }),
-    ...(payload.response_format && { response_format: payload.response_format })
-  };
-
-  const start = Date.now();
-  const response = await client.chat.completions.create(request);
-  const latencyMs = Date.now() - start;
-
-  const choice = response.choices[0];
-  const inputTokens = response.usage.prompt_tokens;
-  const outputTokens = response.usage.completion_tokens;
-  const costUsd = estimateCost(payload.model, inputTokens, outputTokens);
-
-  return {
-    provider: 'openai',
-    model: payload.model,
-    latency_ms: latencyMs,
-    usage: { input_tokens: inputTokens, output_tokens: outputTokens },
-    cost_usd: costUsd,
-    content: choice.message.content || '',
-    tool_calls: choice.message.tool_calls,
-    raw: response
-  };
-}
-
-/**
- * Pure state-accumulation for one OpenAI streaming chunk - factored out
- * from chatStream() so the usage/cost extraction is directly
- * unit-testable with canned chunks, no live API needed. Mutates `state`
- * ({content, inputTokens, outputTokens}) and calls onDelta() with each
- * new piece of assistant text.
- *
- * OpenAI only includes `usage` on a final, choice-less chunk, and only
- * when the request explicitly asked for it (`stream_options:
- * {include_usage: true}`, set in chatStream() below) - without that
- * flag a streamed OpenAI response has NO usage data at all, which would
- * silently make cost_usd wrong (stuck at 0) for every streamed OpenAI
- * call. Requesting it explicitly is required, not optional, for the
- * cost tracking this whole project is built around to stay honest.
- */
-function applyStreamChunk(state, chunk, onDelta) {
-  const choice = chunk.choices && chunk.choices[0];
-  if (choice && choice.delta && choice.delta.content) {
-    state.content += choice.delta.content;
-    onDelta(choice.delta.content);
-  }
-  if (chunk.usage) {
-    state.inputTokens = chunk.usage.prompt_tokens;
-    state.outputTokens = chunk.usage.completion_tokens;
-  }
-}
-
-/**
- * Streaming counterpart to chat(). Scope: plain text content only - no
- * tools/tool_choice forwarded (server.js rejects stream:true + tools
- * before this is ever called; see streaming.js for why).
- */
-async function chatStream(client, payload, { onDelta, signal } = {}) {
-  const request = {
-    model: payload.model,
-    messages: payload.messages,
-    temperature: typeof payload.temperature === 'number' ? payload.temperature : 0.0,
-    max_tokens: payload.max_tokens || 1024,
-    stream: true,
-    stream_options: { include_usage: true }
-  };
-
-  const start = Date.now();
-  const stream = await client.chat.completions.create(request, signal ? { signal } : undefined);
-
-  const state = { content: '', inputTokens: 0, outputTokens: 0 };
-  for await (const chunk of stream) {
-    applyStreamChunk(state, chunk, onDelta || (() => {}));
-  }
-
-  const latencyMs = Date.now() - start;
-  const costUsd = estimateCost(payload.model, state.inputTokens, state.outputTokens);
-
-  return {
-    provider: 'openai',
-    model: payload.model,
-    latency_ms: latencyMs,
-    usage: { input_tokens: state.inputTokens, output_tokens: state.outputTokens },
-    cost_usd: costUsd,
-    content: state.content,
-    tool_calls: undefined
-  };
-}
-
-module.exports = { buildClient, chat, chatStream, applyStreamChunk, estimateCost };
+// model-router/providers/openai.js
+const { OpenAI } = require('openai');
+
+function buildClient(apiKey) {
+  return new OpenAI({ apiKey });
+}
+
+function estimateCost(model, inputTokens, outputTokens) {
+  // Approximate pricing per 1M tokens
+  const rates = {
+    'gpt-4o-mini': { input: 0.15, output: 0.6 },
+    'gpt-4o': { input: 2.5, output: 10.0 }
+  };
+  const rate = rates[model] || { input: 2.5, output: 10.0 };
+  return ((inputTokens * rate.input) + (outputTokens * rate.output)) / 1_000_000;
+}
+
+// `options.requestLogprobs` (cascade routing, step 34): ask the API for
+// per-token logprobs so cascade.js can estimate confidence from the response.
+// ONLY set by the router for an OpenAI candidate when cascade is active for
+// that dispatch - a normal caller never sees this, and it's near-zero extra
+// cost on the request it's attached to. top_logprobs: 1 keeps the payload
+// small (one alternative per token) while still carrying the emitted token's
+// own logprob, which is all the confidence math needs.
+async function chat(client, payload, options = {}) {
+  const request = {
+    model: payload.model,
+    messages: payload.messages,
+    temperature: typeof payload.temperature === 'number' ? payload.temperature : 0.0,
+    max_tokens: payload.max_tokens || 1024,
+    ...(payload.tools && { tools: payload.tools }),
+    ...(payload.tool_choice && { tool_choice: payload.tool_choice }),
+    ...(payload.response_format && { response_format: payload.response_format }),
+    ...(options.requestLogprobs && { logprobs: true, top_logprobs: 1 })
+  };
+
+  const start = Date.now();
+  const response = await client.chat.completions.create(request);
+  const latencyMs = Date.now() - start;
+
+  const choice = response.choices[0];
+  const inputTokens = response.usage.prompt_tokens;
+  const outputTokens = response.usage.completion_tokens;
+  const costUsd = estimateCost(payload.model, inputTokens, outputTokens);
+
+  return {
+    provider: 'openai',
+    model: payload.model,
+    latency_ms: latencyMs,
+    usage: { input_tokens: inputTokens, output_tokens: outputTokens },
+    cost_usd: costUsd,
+    content: choice.message.content || '',
+    tool_calls: choice.message.tool_calls,
+    raw: response
+  };
+}
+
+/**
+ * Pure state-accumulation for one OpenAI streaming chunk - factored out
+ * from chatStream() so the usage/cost extraction is directly
+ * unit-testable with canned chunks, no live API needed. Mutates `state`
+ * ({content, inputTokens, outputTokens}) and calls onDelta() with each
+ * new piece of assistant text.
+ *
+ * OpenAI only includes `usage` on a final, choice-less chunk, and only
+ * when the request explicitly asked for it (`stream_options:
+ * {include_usage: true}`, set in chatStream() below) - without that
+ * flag a streamed OpenAI response has NO usage data at all, which would
+ * silently make cost_usd wrong (stuck at 0) for every streamed OpenAI
+ * call. Requesting it explicitly is required, not optional, for the
+ * cost tracking this whole project is built around to stay honest.
+ */
+function applyStreamChunk(state, chunk, onDelta) {
+  const choice = chunk.choices && chunk.choices[0];
+  if (choice && choice.delta && choice.delta.content) {
+    state.content += choice.delta.content;
+    onDelta(choice.delta.content);
+  }
+  if (chunk.usage) {
+    state.inputTokens = chunk.usage.prompt_tokens;
+    state.outputTokens = chunk.usage.completion_tokens;
+  }
+}
+
+/**
+ * Streaming counterpart to chat(). Scope: plain text content only - no
+ * tools/tool_choice forwarded (server.js rejects stream:true + tools
+ * before this is ever called; see streaming.js for why).
+ */
+async function chatStream(client, payload, { onDelta, signal } = {}) {
+  const request = {
+    model: payload.model,
+    messages: payload.messages,
+    temperature: typeof payload.temperature === 'number' ? payload.temperature : 0.0,
+    max_tokens: payload.max_tokens || 1024,
+    stream: true,
+    stream_options: { include_usage: true }
+  };
+
+  const start = Date.now();
+  const stream = await client.chat.completions.create(request, signal ? { signal } : undefined);
+
+  const state = { content: '', inputTokens: 0, outputTokens: 0 };
+  for await (const chunk of stream) {
+    applyStreamChunk(state, chunk, onDelta || (() => {}));
+  }
+
+  const latencyMs = Date.now() - start;
+  const costUsd = estimateCost(payload.model, state.inputTokens, state.outputTokens);
+
+  return {
+    provider: 'openai',
+    model: payload.model,
+    latency_ms: latencyMs,
+    usage: { input_tokens: state.inputTokens, output_tokens: state.outputTokens },
+    cost_usd: costUsd,
+    content: state.content,
+    tool_calls: undefined
+  };
+}
+
+module.exports = { buildClient, chat, chatStream, applyStreamChunk, estimateCost };