@khanglvm/llm-router 2.0.5 → 2.0.6

package/README.md

@@ -2,20 +2,18 @@
 
 LLM Router is a local and Cloudflare-deployable gateway for routing one client endpoint across multiple LLM providers, models, aliases, fallbacks, and rate limits.
 
-The npm package name stays the same:
+**Current version**: `2.0.5`
 
+NPM package:
 ```bash
 @khanglvm/llm-router
 ```
 
-The primary CLI command is now:
-
+Primary CLI command:
 ```bash
 llr
 ```
 
-`2.0.1` is the current public release. It includes the Web UI, AMP routing, and coding-tool integrations introduced in the 2.x line.
-
 ## Install
 
 ```bash
@@ -266,7 +264,18 @@ Run the JavaScript test suite:
 node --test $(rg --files -g "*.test.js" src)
 ```
 
+## Documentation
+
+Comprehensive documentation is available in the `docs/` directory:
+
+- **[Project Overview & PDR](./docs/project-overview-pdr.md)** — Feature matrix, target users, success metrics, constraints
+- **[Codebase Summary](./docs/codebase-summary.md)** — Directory structure, module relationships, entry points, test infrastructure
+- **[Code Standards](./docs/code-standards.md)** — Patterns, naming conventions, testing, error handling
+- **[System Architecture](./docs/system-architecture.md)** — Request lifecycle, subsystem boundaries, data flow, deployment models
+- **[Project Roadmap](./docs/project-roadmap.md)** — Current status, planned phases, timeline, success metrics
+
 ## Security and Releases
 
 - Security: [`SECURITY.md`](https://github.com/khanglvm/llm-router/blob/master/SECURITY.md)
 - Release notes: [`CHANGELOG.md`](https://github.com/khanglvm/llm-router/blob/master/CHANGELOG.md)
+- AMP routing: [`docs/amp-routing.md`](./docs/amp-routing.md)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@khanglvm/llm-router",
-  "version": "2.0.5",
+  "version": "2.0.6",
   "description": "LLM Router: single gateway endpoint for multi-provider LLMs with unified OpenAI+Anthropic format and seamless fallback",
   "keywords": [
     "llm-router",

package/src/runtime/codex-request-transformer.js

@@ -224,10 +224,15 @@ function normalizeInputMessageContent(content, role) {
         ? part.image_url
         : part.image_url?.url;
       if (typeof rawUrl === 'string' && rawUrl.trim()) {
-        parts.push({
+        const imageItem = {
           type: 'input_image',
           image_url: rawUrl
-        });
+        };
+        const detail = part.image_url?.detail || part.detail;
+        if (typeof detail === 'string' && detail.trim()) {
+          imageItem.detail = detail.trim();
+        }
+        parts.push(imageItem);
       }
       continue;
     }
@@ -333,6 +338,21 @@ function normalizeToolChoiceForResponses(toolChoice) {
     const normalizedType = String(toolChoice.type || '').trim().toLowerCase();
     if (normalizedType === 'none') return 'none';
     if (normalizedType === 'required' || normalizedType === 'any' || normalizedType === 'tool') {
+      const functionName = String(
+        toolChoice.function?.name || toolChoice.name || ''
+      ).trim();
+      if (functionName) {
+        return { type: 'function', name: functionName };
+      }
+      return 'required';
+    }
+    if (normalizedType === 'function') {
+      const functionName = String(
+        toolChoice.function?.name || toolChoice.name || ''
+      ).trim();
+      if (functionName) {
+        return { type: 'function', name: functionName };
+      }
       return 'required';
     }
   }

package/src/runtime/config.js

@@ -735,7 +735,11 @@ function normalizeAmpWebSearchConfig(rawWebSearch) {
   return {
     strategy: normalizeAmpWebSearchStrategy(rawWebSearch.strategy),
     count,
-    providers
+    providers,
+    interceptInternalSearch: normalizeBooleanValue(
+      rawWebSearch.interceptInternalSearch ?? rawWebSearch["intercept-internal-search"],
+      false
+    )
   };
 }
 

package/src/runtime/handler/amp-web-search.js

@@ -10,6 +10,7 @@ import {
   resolveRouteReference
 } from "../config.js";
 import { isSubscriptionProvider, makeSubscriptionProviderCall } from "../subscription-provider.js";
+import { jsonResponse } from "./http.js";
 
 const SEARCH_TOOL_NAME = "web_search";
 const READ_WEB_PAGE_TOOL_NAME = "read_web_page";
@@ -2178,6 +2179,135 @@ export async function testHostedWebSearchProviderRoute({
   }, query, runtimeConfig, env);
 }
 
+async function fetchStructuredSearchResults(query, count, provider) {
+  const normalizedQuery = String(query || "").trim();
+  if (!normalizedQuery || !provider) return [];
+
+  const id = provider.id;
+
+  if (id === "brave") {
+    if (!provider.apiKey) return [];
+    const url = `https://api.search.brave.com/res/v1/web/search?q=${encodeURIComponent(normalizedQuery)}&count=${count}&text_decorations=false`;
+    const response = await runFetchWithTimeout(url, {
+      headers: { Accept: "application/json", "X-Subscription-Token": provider.apiKey }
+    });
+    if (!response.ok) return [];
+    const payload = await response.json();
+    return (Array.isArray(payload?.web?.results) ? payload.web.results.slice(0, count) : [])
+      .map((item) => ({ title: String(item?.title || ""), url: String(item?.url || ""), snippet: String(item?.description || "") }));
+  }
+
+  if (id === "tavily") {
+    if (!provider.apiKey) return [];
+    const response = await runFetchWithTimeout("https://api.tavily.com/search", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ api_key: provider.apiKey, query: normalizedQuery, max_results: count, search_depth: "basic" })
+    });
+    if (!response.ok) return [];
+    const payload = await response.json();
+    return (Array.isArray(payload?.results) ? payload.results.slice(0, count) : [])
+      .map((item) => ({ title: String(item?.title || ""), url: String(item?.url || ""), snippet: String(item?.content || "") }));
+  }
+
+  if (id === "exa") {
+    if (!provider.apiKey) return [];
+    const response = await runFetchWithTimeout("https://api.exa.ai/search", {
+      method: "POST",
+      headers: { "Content-Type": "application/json", "x-api-key": provider.apiKey },
+      body: JSON.stringify({ query: normalizedQuery, numResults: count, type: "auto", contents: { text: { maxCharacters: 500 } } })
+    });
+    if (!response.ok) return [];
+    const payload = await response.json();
+    return (Array.isArray(payload?.results) ? payload.results.slice(0, count) : [])
+      .map((item) => ({ title: String(item?.title || ""), url: String(item?.url || ""), snippet: String(item?.text || item?.snippet || "") }));
+  }
+
+  if (id === "searxng") {
+    if (!provider.url) return [];
+    const url = `${provider.url}/search?q=${encodeURIComponent(normalizedQuery)}&format=json&categories=general&language=auto`;
+    const response = await runFetchWithTimeout(url, {
+      headers: { Accept: "application/json", "User-Agent": "llm-router" }
+    });
+    if (!response.ok) return [];
+    const payload = await response.json();
+    return (Array.isArray(payload?.results) ? payload.results.slice(0, count) : [])
+      .map((item) => ({ title: String(item?.title || ""), url: String(item?.url || ""), snippet: String(item?.content || "") }));
+  }
+
+  return [];
+}
+
+export async function executeWebSearchQueries({ queries, maxResults, config, env }) {
+  const normalizedQueries = (Array.isArray(queries) ? queries : []).map((q) => String(q || "").trim()).filter(Boolean).slice(0, 10);
+  if (normalizedQueries.length === 0) return { results: [], provider: "" };
+
+  const count = Math.max(1, Math.min(20, Number(maxResults) || 5));
+  const snapshot = await buildAmpWebSearchSnapshot(config, { env });
+  const readyProviders = snapshot.providers.filter((p) => p.ready && !isHostedSearchProvider(p));
+
+  for (const providerStatus of readyProviders) {
+    try {
+      const allResults = [];
+      const batchResults = await Promise.all(
+        normalizedQueries.map((query) => fetchStructuredSearchResults(query, count, providerStatus))
+      );
+      for (const results of batchResults) allResults.push(...results);
+      if (allResults.length > 0) {
+        return { results: allResults, provider: providerStatus.id };
+      }
+    } catch {
+      continue;
+    }
+  }
+
+  return { results: [], provider: "" };
+}
+
+export async function maybeInterceptAmpInternalSearch(request, url, config, env) {
+  const searchParams = url.searchParams;
+  if (!searchParams.has("webSearch2")) return null;
+
+  const webSearchConfig = config?.webSearch || config?.amp?.webSearch;
+  if (!webSearchConfig?.interceptInternalSearch) return null;
+
+  const providers = Array.isArray(webSearchConfig?.providers) ? webSearchConfig.providers : [];
+  if (providers.length === 0) return null;
+
+  let body;
+  try {
+    body = await request.clone().json();
+  } catch {
+    return null;
+  }
+
+  const params = body?.params;
+  if (!params || !Array.isArray(params.searchQueries) || params.searchQueries.length === 0) return null;
+
+  try {
+    const results = await executeWebSearchQueries({
+      queries: params.searchQueries,
+      maxResults: Number(params.maxResults) || 5,
+      config,
+      env
+    });
+
+    return jsonResponse({
+      result: {
+        results: results.results.map((r) => ({
+          title: r.title || "",
+          url: r.url || "",
+          snippet: r.snippet || "",
+          content: r.snippet || ""
+        }))
+      }
+    });
+  } catch (error) {
+    console.warn(`[llm-router] webSearch2 interception failed: ${error?.message || error}`);
+    return null;
+  }
+}
+
 export async function maybeInterceptAmpWebSearch({
   response,
   providerBody,

package/src/runtime/handler/provider-call.js

@@ -547,7 +547,8 @@ export async function makeProviderCall({
   env,
   clientType,
   runtimeConfig,
-  stateStore
+  stateStore,
+  ampContext
 }) {
   const provider = candidate.provider;
   const targetFormat = candidate.targetFormat;
@@ -566,11 +567,16 @@ export async function makeProviderCall({
     body
   });
 
+  let effectiveBody = body;
+  if (ampContext?.presets?.reasoningEffort && !body?.reasoning_effort && !body?.reasoning?.effort) {
+    effectiveBody = { ...body, reasoning_effort: ampContext.presets.reasoningEffort };
+  }
+
   let activePlan;
   let fallbackPlan = null;
   try {
     activePlan = buildProviderRequestPlan({
-      body,
+      body: effectiveBody,
       sourceFormat,
       targetFormat: preferOpenAIToolRouting ? FORMATS.OPENAI : targetFormat,
       candidate,
@@ -581,7 +587,7 @@ export async function makeProviderCall({
     });
     if (preferOpenAIToolRouting) {
       fallbackPlan = buildProviderRequestPlan({
-        body,
+        body: effectiveBody,
         sourceFormat,
         targetFormat,
         candidate,
@@ -619,6 +625,12 @@ export async function makeProviderCall({
 
   if (isSubscriptionProvider(provider)) {
     const subscriptionType = String(provider?.subscriptionType || provider?.subscription_type || "").trim().toLowerCase();
+    if (subscriptionType === "chatgpt-codex" && ampContext?.threadId) {
+      activePlan.providerBody = {
+        ...activePlan.providerBody,
+        prompt_cache_key: activePlan.providerBody.prompt_cache_key || ampContext.threadId
+      };
+    }
     const executeSubscriptionRequest = async (requestBody) => makeSubscriptionProviderCall({
       provider,
       body: requestBody,

package/src/runtime/handler/provider-translation.js

@@ -694,9 +694,10 @@ export function handleClaudeStreamToOpenAIResponses(response, requestBody, fallb
       const index = Number(payload.index);
       const blockInfo = payload.content_block || {};
       state.activeBlocks.set(index, String(blockInfo.type || "").trim());
+      // Defer text output item creation until first renderable text delta
+      // to avoid emitting empty assistant text scaffolding before tool calls.
       if (blockInfo.type === "text") {
-        ensureOpenAIResponsesTextItem(state, controller, encoder);
-        state.textOpened = true;
+        // Intentionally do NOT open text item yet; wait for renderable text in content_block_delta.
       } else if (blockInfo.type === "thinking" || blockInfo.type === "redacted_thinking") {
         ensureOpenAIResponsesReasoningItem(state, index, controller, encoder);
       } else if (blockInfo.type === "tool_use") {
@@ -709,6 +710,10 @@ export function handleClaudeStreamToOpenAIResponses(response, requestBody, fallb
       const index = Number(payload.index);
       const delta = payload.delta || {};
       if (delta.type === "text_delta" && typeof delta.text === "string") {
+        const hasRenderableText = /\S/.test(delta.text);
+        if (!state.textOpened && !hasRenderableText) {
+          return;
+        }
         ensureOpenAIResponsesTextItem(state, controller, encoder);
         state.textOpened = true;
         state.textBuffer += delta.text;

package/src/runtime/handler/request.js

@@ -461,3 +461,28 @@ export function isStreamingEnabled(sourceFormat, body) {
   // Some clients omit `stream` on follow-up/tool turns and expect JSON responses.
   return body?.stream === true;
 }
+
+const AMP_MODE_PRESETS = new Map([
+  ["smart", { reasoningEffort: "", toolChoice: "" }],
+  ["free",  { reasoningEffort: "", toolChoice: "" }],
+  ["rush",  { reasoningEffort: "low", toolChoice: "" }],
+  ["deep",  { reasoningEffort: "high", toolChoice: "" }],
+  ["large", { reasoningEffort: "", toolChoice: "" }],
+  ["bombadil", { reasoningEffort: "", toolChoice: "" }]
+]);
+
+export function extractAmpContext(request) {
+  const headers = request?.headers;
+  if (!headers || typeof headers.get !== "function") {
+    return { threadId: "", mode: "", overrideProvider: "", feature: "", messageId: "", presets: null };
+  }
+
+  const threadId = String(headers.get("x-amp-thread-id") || "").trim();
+  const mode = String(headers.get("x-amp-mode") || "").trim().toLowerCase();
+  const overrideProvider = String(headers.get("x-amp-override-provider") || "").trim().toLowerCase();
+  const feature = String(headers.get("x-amp-feature") || "").trim();
+  const messageId = String(headers.get("x-amp-message-id") || "").trim();
+  const presets = AMP_MODE_PRESETS.get(mode) || null;
+
+  return { threadId, mode, overrideProvider, feature, messageId, presets };
+}

package/src/runtime/handler.js

@@ -24,6 +24,7 @@ import { corsResponse, jsonResponse } from "./handler/http.js";
 import {
   detectUserRequestFormat,
   estimateRequestContextTokens,
+  extractAmpContext,
   inferAmpContextRequirement,
   isAmpManagementPath,
   isJsonRequest,
@@ -45,7 +46,7 @@ import {
   convertAmpGeminiRequestToOpenAI,
   hasGeminiWebSearchTool
 } from "./handler/amp-gemini.js";
-import { shouldInterceptAmpWebSearch } from "./handler/amp-web-search.js";
+import { shouldInterceptAmpWebSearch, maybeInterceptAmpInternalSearch } from "./handler/amp-web-search.js";
 import {
   isRequestFromAllowedIp,
   resolveAllowedOrigin,
@@ -59,6 +60,7 @@ import {
   resolveRetryPolicy
 } from "./handler/fallback.js";
 import { parseJsonSafely, sleep } from "./handler/utils.js";
+import { createThreadAffinityStore } from "./thread-affinity.js";
 import {
   applyCandidateFailureState,
   applyRuntimeRetryPolicyGuards,
@@ -457,6 +459,9 @@ async function handleRouteRequest(request, env, getConfig, sourceFormatHint, opt
 
   const requestedModel = body?.model || "smart";
   const stream = isStreamingEnabled(sourceFormat, body);
+  const ampContext = options.clientType === "amp"
+    ? extractAmpContext(request)
+    : null;
 
   const interceptAmpWebSearch = shouldInterceptAmpWebSearch({
     clientType: options.clientType,
@@ -481,7 +486,7 @@ async function handleRouteRequest(request, env, getConfig, sourceFormatHint, opt
 
   const resolved = resolveRequestModel(config, requestedModel, sourceFormat, {
     clientType: options.clientType,
-    providerHint: options.providerHint
+    providerHint: ampContext?.overrideProvider || options.providerHint
   });
   if (!resolved.primary) {
     if (options.clientType === "amp" && resolved.allowAmpProxy !== false && isAmpProxyEnabled(config)) {
@@ -604,6 +609,24 @@ async function handleRouteRequest(request, env, getConfig, sourceFormatHint, opt
     recordRouteSkip(routeDebug, skippedEntry.candidate, skippedEntry.skipReasons);
   }
 
+  // Thread affinity: reorder candidates to prefer affinity-bound candidate
+  if (ampContext?.threadId && options.threadAffinityStore) {
+    const affinityCandidateKey = options.threadAffinityStore.getAffinity(ampContext.threadId);
+    if (affinityCandidateKey) {
+      const affinityIndex = ranking.entries.findIndex(
+        (entry) => entry.eligible && entry.candidateKey === affinityCandidateKey
+      );
+      if (affinityIndex > 0) {
+        const [affinityEntry] = ranking.entries.splice(affinityIndex, 1);
+        ranking.entries.unshift(affinityEntry);
+        ranking.selectedEntry = affinityEntry;
+      }
+      if (affinityIndex < 0) {
+        options.threadAffinityStore.clearAffinity(ampContext.threadId);
+      }
+    }
+  }
+
   if (!ranking.selectedEntry) {
     return withRouteDebugHeaders(jsonResponse({
       type: "error",
@@ -652,7 +675,8 @@ async function handleRouteRequest(request, env, getConfig, sourceFormatHint, opt
         env,
         clientType: options.clientType,
         runtimeConfig: config,
-        stateStore
+        stateStore,
+        ampContext
       });
 
       if (!quotaConsumed && shouldConsumeQuotaFromResult(result)) {
@@ -694,6 +718,9 @@ async function handleRouteRequest(request, env, getConfig, sourceFormatHint, opt
           });
           pendingFallbackContext = null;
         }
+        if (ampContext?.threadId && options.threadAffinityStore) {
+          options.threadAffinityStore.setAffinity(ampContext.threadId, entry.candidateKey);
+        }
         return withRouteDebugHeaders(result.response, routeDebug);
       }
 
@@ -784,6 +811,7 @@ export function createFetchHandler(options) {
   }
 
   let stateStoreRef = options.stateStore || null;
+  const threadAffinityStore = createThreadAffinityStore();
   let stateStorePromise = null;
 
   async function ensureStateStore(env = {}, runtimeFlags = {}) {
@@ -906,6 +934,11 @@ export function createFetchHandler(options) {
         return respond(jsonResponse({ error: "Forbidden" }, 403));
       }
 
+      const searchInterceptResult = await maybeInterceptAmpInternalSearch(request, url, config, env);
+      if (searchInterceptResult) {
+        return respond(searchInterceptResult);
+      }
+
       return respond(await proxyAmpUpstreamRequest({ request, config }));
     }
 
@@ -1051,7 +1084,8 @@ export function createFetchHandler(options) {
         providerHint: "google",
         requestKind: "chat-completions",
         stateStore,
-        runtimeFlags
+        runtimeFlags,
+        threadAffinityStore
       });
 
       if (routeResponse.status >= 400) {
@@ -1095,7 +1129,8 @@ export function createFetchHandler(options) {
         providerHint: route.providerHint,
         requestKind: route.requestKind,
         stateStore,
-        runtimeFlags
+        runtimeFlags,
+        threadAffinityStore
       });
       return respond(routeResponse);
     }

package/src/runtime/thread-affinity.js

@@ -0,0 +1,41 @@
+const DEFAULT_AFFINITY_TTL_MS = 60 * 60 * 1000; // 1 hour
+const MAX_BINDINGS = 10_000;
+
+export function createThreadAffinityStore(options = {}) {
+  const ttlMs = options.ttlMs || DEFAULT_AFFINITY_TTL_MS;
+  const bindings = new Map();
+
+  function pruneExpired(now = Date.now()) {
+    for (const [key, binding] of bindings) {
+      if (binding.expiresAt <= now) bindings.delete(key);
+    }
+  }
+
+  function getAffinity(threadId) {
+    if (!threadId) return null;
+    const binding = bindings.get(threadId);
+    if (!binding) return null;
+    if (binding.expiresAt <= Date.now()) {
+      bindings.delete(threadId);
+      return null;
+    }
+    return binding.candidateKey;
+  }
+
+  function setAffinity(threadId, candidateKey) {
+    if (!threadId || !candidateKey) return;
+    const now = Date.now();
+    bindings.set(threadId, {
+      candidateKey,
+      lastSeen: now,
+      expiresAt: now + ttlMs
+    });
+    if (bindings.size > MAX_BINDINGS) pruneExpired(now);
+  }
+
+  function clearAffinity(threadId) {
+    if (threadId) bindings.delete(threadId);
+  }
+
+  return { getAffinity, setAffinity, clearAffinity, pruneExpired, _bindings: bindings };
+}

package/src/translator/response/openai-to-claude.js

@@ -52,7 +52,7 @@ export function openaiToClaudeResponse(chunk, state) {
 
   // Handle regular content
   const textDelta = normalizeTextDelta(delta?.content);
-  if (textDelta) {
+  if (textDelta && (state.textBlockStarted || hasRenderableText(textDelta))) {
     stopThinkingBlock(state, results);
 
     if (!state.textBlockStarted) {
@@ -119,6 +119,10 @@ export function openaiToClaudeResponse(chunk, state) {
   return results.length > 0 ? results : null;
 }
 
+function hasRenderableText(text) {
+  return typeof text === "string" && /\S/.test(text);
+}
+
 function normalizeTextDelta(content) {
   if (typeof content === "string") return content;
   if (!Array.isArray(content)) return "";
@@ -191,6 +195,7 @@ function normalizeMessageToolCalls(message) {
 
 function emitTextDelta(text, state, results) {
   if (!text) return;
+  if (!state.textBlockStarted && !hasRenderableText(text)) return;
   stopThinkingBlock(state, results);
 
   if (!state.textBlockStarted) {