@khanglvm/llm-router 2.3.6 → 2.3.7

package/CHANGELOG.md

@@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [2.3.7] - 2026-04-18
+
+### Fixed
+- Local `llr start` now keeps a fixed-port supervisor in front of the router runtime so CLI and tool traffic can survive backend restarts and upgrades without losing the public router endpoint.
+- `llr update` now upgrades the installed package before asking the live backend to reload, keeping the running router online during the install step and draining in-flight requests before the new version takes over.
+- Requests that arrive during a backend restart window are now deferred and automatically retried through the supervisor instead of failing immediately when the backend socket is briefly unavailable.
+
 ## [2.3.6] - 2026-04-18
 
 ### Fixed

package/README.md

@@ -29,10 +29,17 @@ llr ai-help  # agent-oriented setup brief
 - **Model aliases with routing** — group models into stable alias names with weighted round-robin, quota-aware balancing, and automatic fallback
 - **Rate limiting** — set request caps per model or across all models over configurable time windows
 - **Coding tool routing** — one-click routing config for Codex CLI, Claude Code, Factory Droid, and AMP
+- **Seamless local updates** — `llr update` keeps the fixed local router endpoint online, drains in-flight requests, and automatically retries through backend restart windows
 - **Web search** — built-in web search for AMP and other router-managed tools
 - **Deployable** — run locally or deploy to Cloudflare Workers
 - **AI-agent friendly** — full CLI parity with `llr config --operation=...` so agents can configure everything programmatically
 
+## Local Runtime Reliability
+
+`llr start` keeps a small supervisor bound to the fixed local router port and runs the real router backend behind it on an internal loopback port.
+
+That means `llr update` can install a new package version and gracefully swap the backend without breaking active CLI or tool requests. Requests that arrive during the short backend handoff are deferred and retried automatically instead of failing immediately. The Web UI may reconnect during that window, but router-managed API traffic keeps the same public local endpoint.
+
 ## Web UI
 
 ### Alias & Fallback

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@khanglvm/llm-router",
-  "version": "2.3.6",
+  "version": "2.3.7",
   "description": "LLM Router: single gateway endpoint for multi-provider LLMs with unified OpenAI+Anthropic format and seamless fallback",
   "keywords": [
     "llm-router",

package/src/cli/router-module.js

@@ -6694,7 +6694,7 @@ async function doSetFactoryDroidRouting(context) {
         ["Settings File", patchResult.settingsFilePath],
         ["Backup File", patchResult.backupFilePath],
         ["Base URL", patchResult.baseUrl],
-        ["Provider", patchResult.configuredProvider || "generic-chat-completion-api"],
+        ["Provider", patchResult.configuredProvider || "(not set)"],
         ["Default Model", patchResult.bindings?.defaultModel || "(not set)"],
         ["Mission Orchestrator", patchResult.bindings?.missionOrchestratorModel || "(not set)"],
         ["Mission Worker", patchResult.bindings?.missionWorkerModel || "(not set)"],

package/src/cli-entry.js

@@ -117,14 +117,22 @@ async function promptStartupConflictResolution({ port }) {
   }
 }
 
-async function runStartFastPath(args) {
+async function runStartFastPath(args, { backendMode = false } = {}) {
+  const explicitPort = backendMode
+    ? (() => {
+      const parsed = Number.parseInt(String(args.port ?? ""), 10);
+      return Number.isInteger(parsed) && parsed > 0 ? parsed : FIXED_LOCAL_ROUTER_PORT;
+    })()
+    : resolveListenPort({ explicitPort: args.port });
   const result = await runStartCommand({
     configPath: args.config || args.configPath || getDefaultConfigPath(),
     host: FIXED_LOCAL_ROUTER_HOST,
-    port: resolveListenPort({ explicitPort: args.port }),
+    port: explicitPort,
     watchConfig: parseBoolean(args["watch-config"] ?? args.watchConfig, true),
     watchBinary: parseBoolean(args["watch-binary"] ?? args.watchBinary, true),
     requireAuth: parseBoolean(args["require-auth"] ?? args.requireAuth, false),
+    backendMode,
+    startCommand: backendMode ? "start-runtime" : "start",
     onStartupConflict: (payload) => promptStartupConflictResolution(payload),
     cliPathForWatch: process.argv[1],
     onLine: (line) => console.log(line),
@@ -163,6 +171,7 @@ export async function runCli(argv = process.argv.slice(2), isTTY = undefined, ov
   const parsed = parseSimpleArgs(argv);
   const first = parsed.positional[0];
   const firstIsStart = first === "start";
+  const firstIsStartRuntime = first === "start-runtime";
   const firstIsWeb = first === "web";
   const firstIsConfig = first === "config";
   const firstIsSetup = first === "setup";
@@ -188,6 +197,12 @@ export async function runCli(argv = process.argv.slice(2), isTTY = undefined, ov
     return runStartFastPathImpl(parsedStart.args);
   }
 
+  if (firstIsStartRuntime && !parsed.wantsHelp) {
+    const startArgs = argv.slice(1);
+    const parsedStart = parseSimpleArgs(startArgs);
+    return runStartFastPathImpl(parsedStart.args, { backendMode: true });
+  }
+
   if (firstIsWeb && !parsed.wantsHelp) {
     const webArgs = argv.slice(1);
     const parsedWeb = parseSimpleArgs(webArgs);

package/src/node/coding-tool-config.js

@@ -970,7 +970,38 @@ export async function patchClaudeCodeEffortLevel({
 }
 
 const FACTORY_DROID_ROUTER_MARKER = "_llmRouterManaged";
-const FACTORY_DROID_ROUTER_PROVIDER = "generic-chat-completion-api";
+const FACTORY_DROID_OPENAI_PROVIDER = "openai";
+const FACTORY_DROID_ANTHROPIC_PROVIDER = "anthropic";
+const FACTORY_DROID_ROUTER_PROVIDERS = Object.freeze([
+  FACTORY_DROID_OPENAI_PROVIDER,
+  FACTORY_DROID_ANTHROPIC_PROVIDER
+]);
+
+function dedupeStrings(values = []) {
+  const seen = new Set();
+  const out = [];
+  for (const value of values) {
+    const normalized = String(value || "").trim();
+    if (!normalized || seen.has(normalized)) continue;
+    seen.add(normalized);
+    out.push(normalized);
+  }
+  return out;
+}
+
+function normalizeFactoryDroidFormat(value) {
+  const normalized = String(value || "").trim().toLowerCase();
+  if (normalized === "openai") return "openai";
+  if (normalized === "claude" || normalized === "anthropic") return "claude";
+  return "";
+}
+
+function mapFactoryDroidFormatToProvider(format) {
+  const normalized = normalizeFactoryDroidFormat(format);
+  if (normalized === "claude") return FACTORY_DROID_ANTHROPIC_PROVIDER;
+  if (normalized === "openai") return FACTORY_DROID_OPENAI_PROVIDER;
+  return "";
+}
 
 function normalizeFactoryDroidBindings(bindings = {}) {
   const source = bindings && typeof bindings === "object" && !Array.isArray(bindings) ? bindings : {};
@@ -984,9 +1015,119 @@ function normalizeFactoryDroidBindings(bindings = {}) {
   };
 }
 
-function buildFactoryDroidBaseUrl(endpointUrl) {
+function buildFactoryDroidBaseUrl(endpointUrl, provider = FACTORY_DROID_OPENAI_PROVIDER) {
   const normalized = normalizeHttpUrl(endpointUrl);
-  return normalized ? `${normalized}/openai/v1` : "";
+  const resolvedProvider = String(provider || "").trim().toLowerCase() || FACTORY_DROID_OPENAI_PROVIDER;
+  if (!normalized) return "";
+  return resolvedProvider === FACTORY_DROID_ANTHROPIC_PROVIDER
+    ? `${normalized}/anthropic`
+    : `${normalized}/openai/v1`;
+}
+
+function inferFactoryDroidFormatFromModelId(modelId) {
+  const normalized = String(modelId || "").trim().toLowerCase();
+  if (!normalized) return "";
+  if (/^(?:claude|opus|sonnet|haiku)(?=[-./\s]|$)/i.test(normalized)) return "claude";
+  if (/^gpt(?=[-./\s]|$)/i.test(normalized)) return "openai";
+  return "";
+}
+
+function inferFactoryDroidFormatFromProviderId(providerId) {
+  const normalized = String(providerId || "").trim().toLowerCase();
+  if (!normalized) return "";
+  if (normalized === "anthropic") return "claude";
+  if (normalized === "openai") return "openai";
+  return "";
+}
+
+function getFactoryDroidProviderModelFormats(provider, model, modelId = "") {
+  const resolvedModelId = String(modelId || model?.id || "").trim();
+  const preferredFormat = normalizeFactoryDroidFormat(provider?.lastProbe?.modelPreferredFormat?.[resolvedModelId]);
+  if (preferredFormat) return [preferredFormat];
+
+  return dedupeStrings([
+    ...(provider?.lastProbe?.modelSupport?.[resolvedModelId] || []),
+    ...(model?.formats || []),
+    model?.format
+  ])
+    .map(normalizeFactoryDroidFormat)
+    .filter(Boolean);
+}
+
+function getFactoryDroidProviderFormats(provider) {
+  return dedupeStrings([
+    ...(provider?.formats || []),
+    provider?.format
+  ])
+    .map(normalizeFactoryDroidFormat)
+    .filter(Boolean);
+}
+
+function getFactoryDroidAliasTargetRefs(alias) {
+  const refs = [];
+  const push = (entry) => {
+    const ref = String(
+      typeof entry === "string"
+        ? entry
+        : (entry?.ref || entry?.sourceRef || "")
+    ).trim();
+    if (ref) refs.push(ref);
+  };
+
+  for (const entry of Array.isArray(alias?.targets) ? alias.targets : []) push(entry);
+  for (const entry of Array.isArray(alias?.fallbackTargets) ? alias.fallbackTargets : []) push(entry);
+
+  return refs;
+}
+
+function resolveFactoryDroidRouteFormat(modelRef, config = {}, seen = new Set()) {
+  const normalizedModelRef = String(modelRef || "").trim();
+  if (!normalizedModelRef || seen.has(normalizedModelRef)) return "";
+
+  if (normalizedModelRef.includes("/")) {
+    const separatorIndex = normalizedModelRef.indexOf("/");
+    const providerId = normalizedModelRef.slice(0, separatorIndex).trim();
+    const modelId = normalizedModelRef.slice(separatorIndex + 1).trim();
+    const provider = (Array.isArray(config?.providers) ? config.providers : [])
+      .find((entry) => String(entry?.id || "").trim() === providerId);
+    const model = Array.isArray(provider?.models)
+      ? provider.models.find((entry) => String(entry?.id || "").trim() === modelId)
+      : null;
+    return getFactoryDroidProviderModelFormats(provider, model, modelId)[0]
+      || inferFactoryDroidFormatFromModelId(modelId)
+      || getFactoryDroidProviderFormats(provider)[0]
+      || inferFactoryDroidFormatFromProviderId(providerId)
+      || "";
+  }
+
+  seen.add(normalizedModelRef);
+  const aliases = config?.modelAliases && typeof config.modelAliases === "object" && !Array.isArray(config.modelAliases)
+    ? config.modelAliases
+    : {};
+  const alias = aliases[normalizedModelRef];
+  if (!alias || typeof alias !== "object" || Array.isArray(alias)) return "";
+
+  for (const targetRef of getFactoryDroidAliasTargetRefs(alias)) {
+    const resolved = resolveFactoryDroidRouteFormat(targetRef, config, new Set(seen));
+    if (resolved) return resolved;
+  }
+
+  return "";
+}
+
+function resolveFactoryDroidCustomModelProvider(modelRef, config = {}) {
+  return mapFactoryDroidFormatToProvider(resolveFactoryDroidRouteFormat(modelRef, config))
+    || FACTORY_DROID_OPENAI_PROVIDER;
+}
+
+function resolveFactoryDroidProviderDisplayName(modelRef, config = {}) {
+  const normalizedModelRef = String(modelRef || "").trim();
+  if (!normalizedModelRef.includes("/")) return "";
+  const separatorIndex = normalizedModelRef.indexOf("/");
+  const providerId = normalizedModelRef.slice(0, separatorIndex).trim();
+  const provider = (Array.isArray(config?.providers) ? config.providers : [])
+    .find((entry) => String(entry?.id || "").trim() === providerId);
+  return String(provider?.name || providerId || "").trim();
 }
 
 function collectFactoryDroidAvailableModels(config = {}, bindings = {}) {
@@ -1034,7 +1175,10 @@ function buildFactoryDroidAvailableModelDescriptors(config = {}, bindings = {})
         modelRef,
         kind,
         id: buildFactoryDroidRouterModelId(modelRef, { kind }),
-        displayName: buildFactoryDroidRouterDisplayName(modelRef, { kind })
+        displayName: buildFactoryDroidRouterDisplayName(modelRef, {
+          kind,
+          providerName: kind === "model" ? resolveFactoryDroidProviderDisplayName(modelRef, config) : ""
+        })
       };
     })
     .filter((entry) => String(entry.id || "").trim() && String(entry.modelRef || "").trim());
@@ -1156,14 +1300,17 @@ function isFactoryDroidRouterManagedEntry(entry, { baseUrl = "" } = {}) {
   const entryId = String(entry.id || "").trim();
   if (isFactoryDroidRouterModelId(entryId)) return true;
 
-  const provider = String(entry.provider || "").trim();
-  if (provider !== FACTORY_DROID_ROUTER_PROVIDER) return false;
+  const provider = String(entry.provider || "").trim().toLowerCase();
+  if (!FACTORY_DROID_ROUTER_PROVIDERS.includes(provider)) return false;
 
   const entryBaseUrl = String(entry.baseUrl || "").trim();
   if (baseUrl && entryBaseUrl === String(baseUrl || "").trim()) return true;
 
   const apiKey = String(entry.apiKey || "").trim();
-  return apiKey.startsWith("gw_") && entryBaseUrl.includes("/openai/v1");
+  return apiKey.startsWith("gw_") && (
+    entryBaseUrl.includes("/openai/v1")
+    || entryBaseUrl.includes("/anthropic")
+  );
 }
 
 function stripRouterManagedCustomModels(customModels, { baseUrl = "" } = {}) {
@@ -1310,28 +1457,30 @@ export async function readFactoryDroidRoutingState({
 } = {}) {
   const resolvedSettingsPath = path.resolve(String(settingsFilePath || resolveFactoryDroidSettingsFilePath({ homeDir })).trim());
   const resolvedBackupPath = path.resolve(String(backupFilePath || resolveCodingToolBackupFilePath(resolvedSettingsPath)).trim());
-  const expectedBaseUrl = buildFactoryDroidBaseUrl(endpointUrl);
   const routeLookup = buildFactoryDroidRouteLookup(config);
   const settingsState = await readJsonObjectFile(resolvedSettingsPath, `Factory Droid settings file '${resolvedSettingsPath}'`);
   const backupState = await readJsonObjectFile(resolvedBackupPath, `Backup file '${resolvedBackupPath}'`);
   const customModels = Array.isArray(settingsState.data?.customModels) ? settingsState.data.customModels : [];
-  const routerEntry = getRouterManagedCustomModel(customModels);
-  const configuredBaseUrl = routerEntry ? String(routerEntry.baseUrl || "").trim() : "";
-  const configuredProvider = routerEntry ? String(routerEntry.provider || "").trim() : "";
-  const routedViaRouter = Boolean(
-    expectedBaseUrl
-      && routerEntry
-      && configuredBaseUrl === expectedBaseUrl
-  );
-
   const resolvedDefaultModelValue = getNestedObjectValue(settingsState.data, ["sessionDefaultSettings", "model"])
     || settingsState.data?.model
-    || routerEntry?.id
-    || routerEntry?.model
     || "";
   const resolvedMissionOrchestratorValue = settingsState.data?.missionOrchestratorModel || "";
   const resolvedMissionWorkerValue = getNestedObjectValue(settingsState.data, ["missionModelSettings", "workerModel"]) || "";
   const resolvedMissionValidatorValue = getNestedObjectValue(settingsState.data, ["missionModelSettings", "validationWorkerModel"]) || "";
+  const routerEntry = getFactoryDroidCustomModelEntryByValue(customModels, resolvedDefaultModelValue, { preferRouterManaged: true })
+    || getRouterManagedCustomModel(customModels);
+  const configuredBaseUrl = routerEntry ? String(routerEntry.baseUrl || "").trim() : "";
+  const configuredProvider = routerEntry ? String(routerEntry.provider || "").trim() : "";
+  const expectedBaseUrls = new Set(
+    FACTORY_DROID_ROUTER_PROVIDERS
+      .map((provider) => buildFactoryDroidBaseUrl(endpointUrl, provider))
+      .filter(Boolean)
+  );
+  const routedViaRouter = Boolean(
+    configuredBaseUrl
+      && routerEntry
+      && expectedBaseUrls.has(configuredBaseUrl)
+  );
 
   return {
     tool: "factory-droid",
@@ -1371,7 +1520,7 @@ export async function patchFactoryDroidSettingsFile({
 } = {}) {
   const resolvedSettingsPath = path.resolve(String(settingsFilePath || resolveFactoryDroidSettingsFilePath({ homeDir })).trim());
   const resolvedBackupPath = path.resolve(String(backupFilePath || resolveCodingToolBackupFilePath(resolvedSettingsPath)).trim());
-  const baseUrl = buildFactoryDroidBaseUrl(endpointUrl);
+  const baseUrl = buildFactoryDroidBaseUrl(endpointUrl, FACTORY_DROID_OPENAI_PROVIDER);
   const normalizedApiKey = String(apiKey || "").trim();
   const normalizedBindings = normalizeFactoryDroidBindings(bindings);
   const routeLookup = buildFactoryDroidRouteLookup(config);
@@ -1411,15 +1560,16 @@ export async function patchFactoryDroidSettingsFile({
     ? availableModels.map((descriptor, index) => {
       const entryIndex = routerEntryStartIndex + index;
       const modelId = buildFactoryDroidCustomModelId(descriptor.modelRef, entryIndex);
+      const provider = resolveFactoryDroidCustomModelProvider(descriptor.modelRef, config);
       return {
         [FACTORY_DROID_ROUTER_MARKER]: true,
         model: descriptor.modelRef,
         id: modelId,
         index: entryIndex,
         displayName: descriptor.displayName,
-        baseUrl,
+        baseUrl: buildFactoryDroidBaseUrl(endpointUrl, provider),
         apiKey: normalizedApiKey,
-        provider: FACTORY_DROID_ROUTER_PROVIDER
+        provider
       };
     })
     : [{
@@ -1430,7 +1580,7 @@ export async function patchFactoryDroidSettingsFile({
       displayName: buildFactoryDroidRouterDisplayName("llm-router", { kind: "alias" }),
       baseUrl,
       apiKey: normalizedApiKey,
-      provider: FACTORY_DROID_ROUTER_PROVIDER
+      provider: FACTORY_DROID_OPENAI_PROVIDER
     }];
 
   customModels.push(...routerEntries);
@@ -1483,12 +1633,16 @@ export async function patchFactoryDroidSettingsFile({
   }
 
   await writeJsonObjectFile(resolvedSettingsPath, nextSettings);
+  const primaryEntry = resolvedBindings.defaultModel
+    ? getFactoryDroidCustomModelEntryByValue(allCustomModels, resolvedBindings.defaultModel, { preferRouterManaged: true })
+    : null;
+  const configuredEntry = primaryEntry || getRouterManagedCustomModel(allCustomModels);
   return {
     settingsFilePath: resolvedSettingsPath,
     backupFilePath: resolvedBackupPath,
     settingsCreated: !settingsState.existed,
-    baseUrl,
-    configuredProvider: FACTORY_DROID_ROUTER_PROVIDER,
+    baseUrl: String(configuredEntry?.baseUrl || baseUrl).trim(),
+    configuredProvider: String(configuredEntry?.provider || FACTORY_DROID_OPENAI_PROVIDER).trim(),
     bindings: resolvedBindings,
     bindingIds: normalizeFactoryDroidBindings({
       defaultModel: normalizedBindings.defaultModel

package/src/node/config-store.js

@@ -1,5 +1,5 @@
 /**
- * Local config persistence for ~/.llm-router.json.
+ * Local config persistence for the default and development config files.
  */
 
 import os from "node:os";
@@ -14,11 +14,16 @@ import {
 import { sanitizePersistedLocalServerConfig } from "./local-server-settings.js";
 
 export const DEFAULT_CONFIG_FILENAME = ".llm-router.json";
+export const DEFAULT_DEV_CONFIG_FILENAME = ".llm-router-dev.json";
 
 export function getDefaultConfigPath() {
   return path.join(os.homedir(), DEFAULT_CONFIG_FILENAME);
 }
 
+export function getDefaultDevConfigPath() {
+  return path.join(os.homedir(), DEFAULT_DEV_CONFIG_FILENAME);
+}
+
 function normalizePersistedConfig(config, normalizeOptions = undefined) {
   return sanitizePersistedLocalServerConfig(
     normalizeRuntimeConfig(config, normalizeOptions)

package/src/node/instance-state.js

@@ -6,6 +6,7 @@ import { FIXED_LOCAL_ROUTER_HOST, FIXED_LOCAL_ROUTER_PORT } from "./local-server
 
 const DEFAULT_INSTANCE_STATE_FILENAME = ".llm-router.runtime.json";
 const MAX_START_OUTPUT_CHARS = 4000;
+export const RUNTIME_STATE_PATH_ENV = "LLM_ROUTER_RUNTIME_STATE_PATH";
 
 function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
@@ -80,7 +81,9 @@ function runtimeMatchesStartOptions(runtime, {
     && normalized.requireAuth === normalizeBoolean(requireAuth, false);
 }
 
-export function getRuntimeStatePath() {
+export function getRuntimeStatePath({ env = process.env } = {}) {
+  const override = String(env?.[RUNTIME_STATE_PATH_ENV] || "").trim();
+  if (override) return path.resolve(override);
   return path.join(os.homedir(), DEFAULT_INSTANCE_STATE_FILENAME);
 }
 

package/src/node/local-server.js

@@ -292,8 +292,36 @@ export async function startLocalRouteServer({
   });
 
   const fallbackHost = formatHostForUrl(host, port);
+  let shuttingDown = false;
+  const socketRequestCounts = new Map();
+
+  function closeSocketIfIdle(socket) {
+    if (!socket || socket.destroyed) return;
+    if (Number(socketRequestCounts.get(socket) || 0) > 0) return;
+    socket.end();
+  }
 
   const server = http.createServer(async (req, res) => {
+    const socket = req.socket;
+    socketRequestCounts.set(socket, Number(socketRequestCounts.get(socket) || 0) + 1);
+    let finalized = false;
+    const finalizeRequest = () => {
+      if (finalized) return;
+      finalized = true;
+      const remaining = Math.max(0, Number(socketRequestCounts.get(socket) || 0) - 1);
+      if (remaining > 0) {
+        socketRequestCounts.set(socket, remaining);
+        return;
+      }
+      socketRequestCounts.set(socket, 0);
+      if (shuttingDown) {
+        closeSocketIfIdle(socket);
+      }
+    };
+
+    res.once("finish", finalizeRequest);
+    res.once("close", finalizeRequest);
+
     try {
       const request = nodeRequestToFetchRequest(req, fallbackHost);
       const response = await fetchHandler(request, {}, undefined);
@@ -308,6 +336,13 @@ export async function startLocalRouteServer({
     }
   });
 
+  server.on("connection", (socket) => {
+    socketRequestCounts.set(socket, Number(socketRequestCounts.get(socket) || 0));
+    socket.on("close", () => {
+      socketRequestCounts.delete(socket);
+    });
+  });
+
   await new Promise((resolve, reject) => {
     server.once("error", reject);
     server.listen(port, host, () => {
@@ -318,10 +353,15 @@ export async function startLocalRouteServer({
 
   const originalClose = server.close.bind(server);
   server.close = (callback) => {
+    shuttingDown = true;
     Promise.resolve()
       .then(() => configStore.close())
       .then(() => (typeof fetchHandler.close === "function" ? fetchHandler.close() : undefined))
       .finally(() => {
+        server.closeIdleConnections?.();
+        for (const socket of socketRequestCounts.keys()) {
+          closeSocketIfIdle(socket);
+        }
         originalClose(callback);
       });
     return server;