@khanglvm/llm-router 2.0.0-beta.1 → 2.0.0

package/src/node/litellm-context-catalog.js

@@ -0,0 +1,184 @@
+export const LITELLM_CONTEXT_CATALOG_URL = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json";
+export const LITELLM_CONTEXT_CACHE_TTL_MS = 10 * 60 * 1000;
+
+function normalizeModelLookupName(value) {
+  return String(value || "").trim().toLowerCase();
+}
+
+function tokenizeModelLookupName(value) {
+  return normalizeModelLookupName(value)
+    .split(/[^a-z0-9]+/g)
+    .filter(Boolean);
+}
+
+function buildCanonicalModelLookupVariants(value, provider = "") {
+  const normalizedValue = normalizeModelLookupName(value);
+  const normalizedProvider = normalizeModelLookupName(provider);
+  const variants = new Set();
+
+  if (!normalizedValue) return variants;
+  variants.add(normalizedValue);
+
+  if (!normalizedProvider) return variants;
+  for (const separator of ["/", ":"]) {
+    const prefix = `${normalizedProvider}${separator}`;
+    if (!normalizedValue.startsWith(prefix)) continue;
+    const strippedValue = normalizedValue.slice(prefix.length).trim();
+    if (strippedValue) variants.add(strippedValue);
+  }
+
+  return variants;
+}
+
+function isCanonicalExactModelNameMatch(query, candidate, provider = "") {
+  const queryVariants = buildCanonicalModelLookupVariants(query, provider);
+  const candidateVariants = buildCanonicalModelLookupVariants(candidate, provider);
+
+  if (queryVariants.size === 0 || candidateVariants.size === 0) return false;
+  for (const variant of queryVariants) {
+    if (candidateVariants.has(variant)) return true;
+  }
+  return false;
+}
+
+function scoreLooseModelNameMatch(query, candidate) {
+  const normalizedQuery = normalizeModelLookupName(query);
+  const normalizedCandidate = normalizeModelLookupName(candidate);
+  if (!normalizedQuery || !normalizedCandidate) return 0;
+  if (normalizedQuery === normalizedCandidate) return 1000;
+  if (normalizedCandidate.includes(normalizedQuery)) return 600 - (normalizedCandidate.length - normalizedQuery.length);
+  if (normalizedQuery.includes(normalizedCandidate)) return 500 - (normalizedQuery.length - normalizedCandidate.length);
+
+  const queryTokens = tokenizeModelLookupName(normalizedQuery);
+  const candidateTokens = tokenizeModelLookupName(normalizedCandidate);
+  if (queryTokens.length === 0 || candidateTokens.length === 0) return 0;
+
+  let score = 0;
+  for (const token of queryTokens) {
+    if (candidateTokens.includes(token)) {
+      score += token.length * 10;
+      continue;
+    }
+    const partialMatch = candidateTokens.find((candidateToken) => candidateToken.includes(token) || token.includes(candidateToken));
+    if (partialMatch) score += Math.min(token.length, partialMatch.length) * 4;
+  }
+  return score;
+}
+
+function extractLiteLlmContextWindow(entry) {
+  const maxInputTokens = Number(entry?.max_input_tokens);
+  if (Number.isFinite(maxInputTokens) && maxInputTokens > 0) return Math.floor(maxInputTokens);
+  const maxTokens = Number(entry?.max_tokens);
+  if (Number.isFinite(maxTokens) && maxTokens > 0) return Math.floor(maxTokens);
+  return null;
+}
+
+function createLiteLlmLookupResult(modelName, entry = {}) {
+  return {
+    model: String(modelName || "").trim(),
+    contextWindow: extractLiteLlmContextWindow(entry),
+    provider: String(entry?.litellm_provider || "").trim(),
+    mode: String(entry?.mode || "").trim()
+  };
+}
+
+function extractMedianContextWindow(exactMatch, suggestions = []) {
+  const exactContextWindow = Number(exactMatch?.contextWindow);
+  if (Number.isFinite(exactContextWindow) && exactContextWindow > 0) {
+    return Math.floor(exactContextWindow);
+  }
+
+  const contextWindows = (Array.isArray(suggestions) ? suggestions : [])
+    .map((entry) => Number(entry?.contextWindow))
+    .filter((value) => Number.isFinite(value) && value > 0)
+    .sort((left, right) => left - right);
+
+  if (contextWindows.length === 0) return null;
+  return contextWindows[Math.floor((contextWindows.length - 1) / 2)];
+}
+
+export function createLiteLlmContextLookupHelper({
+  fetchImpl = fetch,
+  catalogUrl = LITELLM_CONTEXT_CATALOG_URL,
+  cacheTtlMs = LITELLM_CONTEXT_CACHE_TTL_MS
+} = {}) {
+  let cachedCatalog = null;
+  let cachedAt = 0;
+  let inFlightPromise = null;
+
+  async function loadCatalog() {
+    const now = Date.now();
+    if (cachedCatalog && (now - cachedAt) < cacheTtlMs) {
+      return cachedCatalog;
+    }
+    if (inFlightPromise) return inFlightPromise;
+
+    inFlightPromise = (async () => {
+      const response = await fetchImpl(catalogUrl);
+      if (!response.ok) {
+        throw new Error(`LiteLLM context catalog request failed with status ${response.status}.`);
+      }
+      const payload = await response.json();
+      if (!payload || typeof payload !== "object" || Array.isArray(payload)) {
+        throw new Error("LiteLLM context catalog returned an invalid payload.");
+      }
+      cachedCatalog = payload;
+      cachedAt = Date.now();
+      return cachedCatalog;
+    })();
+
+    try {
+      return await inFlightPromise;
+    } finally {
+      inFlightPromise = null;
+    }
+  }
+
+  return async function lookupLiteLlmContextWindow({ models = [], limit = 8 } = {}) {
+    const catalog = await loadCatalog();
+    const catalogEntries = Object.entries(catalog);
+    const resolvedLimit = Math.max(1, Math.min(Number(limit) || 8, 20));
+
+    return (Array.isArray(models) ? models : [])
+      .map((model) => String(model || "").trim())
+      .filter(Boolean)
+      .map((modelName) => {
+        const rankedMatches = catalogEntries
+          .map(([candidateName, entry]) => ({
+            result: createLiteLlmLookupResult(candidateName, entry),
+            score: scoreLooseModelNameMatch(modelName, candidateName)
+          }))
+          .filter((entry) => entry.score > 0 && Number.isFinite(entry.result.contextWindow))
+          .sort((left, right) => {
+            if (right.score !== left.score) return right.score - left.score;
+            return left.result.model.localeCompare(right.result.model);
+          });
+
+        const exactMatch = rankedMatches.find((entry) => isCanonicalExactModelNameMatch(
+          modelName,
+          entry.result.model,
+          entry.result.provider
+        ))?.result || null;
+
+        const suggestions = rankedMatches
+          .filter((entry) => {
+            if (!exactMatch) return true;
+            return !(
+              entry.result.model === exactMatch.model
+              && entry.result.provider === exactMatch.provider
+              && entry.result.mode === exactMatch.mode
+              && entry.result.contextWindow === exactMatch.contextWindow
+            );
+          })
+          .slice(0, resolvedLimit)
+          .map((entry) => entry.result);
+
+        return {
+          query: modelName,
+          exactMatch,
+          suggestions,
+          medianContextWindow: extractMedianContextWindow(exactMatch, suggestions)
+        };
+      });
+  };
+}

package/src/node/local-server.js

@@ -9,6 +9,8 @@ import { Readable } from "node:stream";
 import { createFetchHandler } from "../runtime/handler.js";
 import { readConfigFile, getDefaultConfigPath } from "./config-store.js";
 import { FIXED_LOCAL_ROUTER_HOST, FIXED_LOCAL_ROUTER_PORT } from "./local-server-settings.js";
+import { readActivityLogSettings } from "../shared/local-router-defaults.js";
+import { appendActivityLogEntry, resolveActivityLogPath } from "./activity-log.js";
 
 const DEFAULT_CONFIG_RELOAD_DEBOUNCE_MS = 300;
 const MAX_CONFIG_RELOAD_DEBOUNCE_MS = 5000;
@@ -237,6 +239,7 @@ export async function startLocalRouteServer({
   port = FIXED_LOCAL_ROUTER_PORT,
   host = FIXED_LOCAL_ROUTER_HOST,
   configPath = getDefaultConfigPath(),
+  activityLogPath = "",
   watchConfig = true,
   configReloadDebounceMs = process.env.LLM_ROUTER_CONFIG_RELOAD_DEBOUNCE_MS,
   validateConfig,
@@ -245,20 +248,37 @@ export async function startLocalRouteServer({
   requireAuth = false
 } = {}) {
   const reloadDebounceMs = resolveReloadDebounceMs(configReloadDebounceMs);
+  const resolvedActivityLogPath = resolveActivityLogPath(configPath, activityLogPath);
+  let activityLogEnabled = true;
   const configStore = createLiveConfigStore({
     configPath,
     watchConfig,
     reloadDebounceMs,
     validateConfig,
-    onReload: onConfigReload,
+    onReload: (nextConfig, reason) => {
+      activityLogEnabled = readActivityLogSettings(nextConfig).enabled;
+      if (typeof onConfigReload === "function") {
+        onConfigReload(nextConfig, reason);
+      }
+    },
     onReloadError: onConfigReloadError
   });
-  await configStore.getConfig();
+  const initialConfig = await configStore.getConfig();
+  activityLogEnabled = readActivityLogSettings(initialConfig).enabled;
 
   const fetchHandler = createFetchHandler({
     ignoreAuth: !requireAuth,
     getConfig: () => configStore.getConfig(),
-    defaultStateStoreBackend: "file"
+    defaultStateStoreBackend: "file",
+    onActivityLog: (entry) => {
+      if (!activityLogEnabled) return;
+      void appendActivityLogEntry(resolvedActivityLogPath, {
+        ...entry,
+        source: entry?.source || "runtime"
+      }).catch((error) => {
+        console.warn(`[llm-router] Failed writing activity log: ${formatError(error)}`);
+      });
+    }
   });
 
   const fallbackHost = formatHostForUrl(host, port);

package/src/node/port-reclaim.js

@@ -130,7 +130,7 @@ export async function stopStartupManagedListener({ port, line, error }, deps = {
 
   if (!shouldStopStartup) return { ok: true, attempted: false };
 
-  line(`Detected startup-managed llm-router on port ${port}. Stopping startup service before reclaim.`);
+  line(`Detected a startup-managed LLM Router instance on port ${port}. Stopping the startup service before reclaim.`);
   try {
     await stopStartupFn();
     await clearRuntimeStateFn();
@@ -140,7 +140,7 @@ export async function stopStartupManagedListener({ port, line, error }, deps = {
     return {
       ok: false,
       attempted: true,
-      errorMessage: `Port ${port} is occupied by a startup-managed llm-router service and could not be stopped automatically. Stop it with 'llm-router stop' or 'llm-router config --operation=startup-uninstall' and retry.`
+      errorMessage: `Port ${port} is occupied by a startup-managed LLM Router service and could not be stopped automatically. Stop it with 'llr stop' or 'llr config --operation=startup-uninstall' and retry.`
     };
   }
 }

package/src/node/start-command.js

@@ -286,11 +286,11 @@ async function handoffToStartupManagedWithLatest({
     if (!stopped?.ok) {
       return {
         ok: false,
-        errorMessage: stopped?.reason || `Failed to stop existing llm-router pid ${activeRuntime.pid}.`
+        errorMessage: stopped?.reason || `Failed to stop existing LLM Router pid ${activeRuntime.pid}.`
       };
     }
     await clearRuntimeStateFn({ pid: activeRuntime.pid });
-    line(`Stopped manual llm-router on http://${activeRuntime.host}:${activeRuntime.port} so the startup service can own the router.`);
+    line(`Stopped manual LLM Router on http://${activeRuntime.host}:${activeRuntime.port} so the startup service can own the router.`);
   }
 
   const reclaimed = await reclaimPortFn({ port: startArgs.port, line, error });
@@ -319,7 +319,7 @@ async function handoffToStartupManagedWithLatest({
     if (!runtime) {
       return {
         ok: false,
-        errorMessage: `Startup-managed llm-router did not become ready on http://${startArgs.host}:${startArgs.port}.`
+        errorMessage: `Startup-managed LLM Router did not become ready on http://${startArgs.host}:${startArgs.port}.`
       };
     }
     return {
@@ -333,7 +333,7 @@ async function handoffToStartupManagedWithLatest({
     error(`Failed restarting startup-managed service: ${message}`);
     return {
       ok: false,
-      errorMessage: `Failed to restart startup-managed llm-router with latest installed version: ${message}`
+      errorMessage: `Failed to restart the startup-managed LLM Router instance with the latest installed version: ${message}`
     };
   }
 }
@@ -394,7 +394,7 @@ export async function runStartCommand(options = {}) {
       exitCode: 2,
       errorMessage: [
         `Config file not found: ${configPath}`,
-        "Run 'llm-router config' to create provider config or 'llm-router -h' for help."
+        "Run 'llr config' to create provider config or 'llr -h' for help."
       ].join("\n")
     };
   }
@@ -439,7 +439,7 @@ export async function runStartCommand(options = {}) {
       exitCode: 2,
       errorMessage: [
         `No providers configured in ${configPath}`,
-        "Run 'llm-router config' to add a provider or 'llm-router -h' for help."
+        "Run 'llr config' to add a provider or 'llr -h' for help."
       ].join("\n")
     };
   }
@@ -450,7 +450,7 @@ export async function runStartCommand(options = {}) {
       exitCode: 2,
       errorMessage: [
         `Local auth requires masterKey in ${configPath}.`,
-        "Run 'llm-router config --operation=set-master-key --master-key=...' or start without --require-auth."
+        "Run 'llr config --operation=set-master-key --master-key=...' or start without --require-auth."
       ].join("\n")
     };
   }
@@ -494,7 +494,7 @@ export async function runStartCommand(options = {}) {
       ok: true,
       exitCode: 0,
       data: [
-        `Startup-managed llm-router is active on http://${handoff.runtime.host}:${handoff.runtime.port}.`,
+        `Startup-managed LLM Router is active on http://${handoff.runtime.host}:${handoff.runtime.port}.`,
         `manager=${handoff.detail?.manager || startup.manager || "unknown"}`,
         `service=${handoff.detail?.serviceId || startup.serviceId || "unknown"}`
       ].join("\n")
@@ -538,7 +538,7 @@ export async function runStartCommand(options = {}) {
             startArgs: buildStartArgs({ configPath, ...nextLocalServer })
           });
           if (!launch.ok) {
-            error(`Failed to relaunch llm-router after config runtime change: ${launch.error instanceof Error ? launch.error.message : String(launch.error)}`);
+            error(`Failed to relaunch LLM Router after the config runtime change: ${launch.error instanceof Error ? launch.error.message : String(launch.error)}`);
             process.exit(1);
             return;
           }
@@ -562,7 +562,7 @@ export async function runStartCommand(options = {}) {
     return {
       ok: false,
       exitCode: 1,
-      errorMessage: `Another llm-router instance is already running at http://${activeRuntime.host}:${activeRuntime.port}. Stop it before starting a new one.`
+      errorMessage: `Another LLM Router instance is already running at http://${activeRuntime.host}:${activeRuntime.port}. Stop it before starting a new one.`
     };
   }
 
@@ -574,7 +574,7 @@ export async function runStartCommand(options = {}) {
       return {
         ok: false,
         exitCode: 1,
-        errorMessage: `Failed to start llm-router on http://${host}:${port}: ${startError instanceof Error ? startError.message : String(startError)}`
+        errorMessage: `Failed to start LLM Router on http://${host}:${port}: ${startError instanceof Error ? startError.message : String(startError)}`
       };
     }
 
@@ -622,7 +622,7 @@ export async function runStartCommand(options = {}) {
             ok: true,
             exitCode: 0,
             data: [
-              "Restarted startup-managed llm-router instance with latest installed version.",
+              "Restarted the startup-managed LLM Router instance with the latest installed version.",
               `manager=${restarted.detail?.manager || "unknown"}`,
               `service=${restarted.detail?.serviceId || "unknown"}`
             ].join("\n")
@@ -633,7 +633,7 @@ export async function runStartCommand(options = {}) {
           return {
             ok: true,
             exitCode: 0,
-            data: `Startup-managed llm-router is still running on port ${port}. Exiting without changes.`
+            data: `Startup-managed LLM Router is still running on port ${port}. Exiting without changes.`
           };
         }
 
@@ -647,7 +647,7 @@ export async function runStartCommand(options = {}) {
             };
           }
 
-          line("Startup-managed instance stopped. Starting llm-router in this terminal...");
+          line("Startup-managed instance stopped. Starting LLM Router in this terminal...");
           const takeoverStart = await attemptServerStartAfterStartupStop(buildLocalServerOptions, { startLocalRouteServer: startLocalRouteServerFn });
           if (takeoverStart.ok) {
             server = takeoverStart.server;
@@ -656,7 +656,7 @@ export async function runStartCommand(options = {}) {
             return {
               ok: false,
               exitCode: 1,
-              errorMessage: `Failed to start llm-router on http://${host}:${port}: ${takeoverStart.error instanceof Error ? takeoverStart.error.message : String(takeoverStart.error)}`
+              errorMessage: `Failed to start LLM Router on http://${host}:${port}: ${takeoverStart.error instanceof Error ? takeoverStart.error.message : String(takeoverStart.error)}`
             };
           }
         }
@@ -682,7 +682,7 @@ export async function runStartCommand(options = {}) {
         return {
           ok: false,
           exitCode: 1,
-          errorMessage: `Failed to start llm-router after reclaiming port ${port}: ${retryError instanceof Error ? retryError.message : String(retryError)}`
+          errorMessage: `Failed to start LLM Router after reclaiming port ${port}: ${retryError instanceof Error ? retryError.message : String(retryError)}`
         };
       }
     }
@@ -763,7 +763,7 @@ export async function runStartCommand(options = {}) {
       binaryState = nextState;
 
       if (managedByStartup) {
-        line(`Detected llm-router update (${from} -> ${to}). Exiting for startup manager to relaunch latest version.`);
+        line(`Detected LLM Router update (${from} -> ${to}). Exiting for the startup manager to relaunch the latest version.`);
         void shutdown().then(() => {
           process.exit(0);
         });
@@ -774,7 +774,7 @@ export async function runStartCommand(options = {}) {
       if (!cliPath) {
         if (!binaryNoticeSent) {
           binaryNoticeSent = true;
-          line(`Detected llm-router update (${from} -> ${to}). Restart this process to run the new version.`);
+          line(`Detected LLM Router update (${from} -> ${to}). Restart this process to run the new version.`);
         }
         return;
       }
@@ -782,22 +782,22 @@ export async function runStartCommand(options = {}) {
       binaryRelaunching = true;
       void (async () => {
         try {
-          line(`Detected llm-router update (${from} -> ${to}). Relaunching latest version...`);
+          line(`Detected LLM Router update (${from} -> ${to}). Relaunching the latest version...`);
           await shutdown();
           const launch = await spawnReplacementCli({
             cliPath,
             startArgs: buildStartArgs({ configPath, host, port, watchConfig, watchBinary, requireAuth })
           });
           if (!launch.ok) {
-            error(`Failed to relaunch updated llm-router: ${launch.error instanceof Error ? launch.error.message : String(launch.error)}`);
+            error(`Failed to relaunch the updated LLM Router process: ${launch.error instanceof Error ? launch.error.message : String(launch.error)}`);
             process.exit(1);
             return;
           }
 
-          line(`Started updated llm-router process (pid ${launch.pid || "unknown"}).`);
+          line(`Started the updated LLM Router process (pid ${launch.pid || "unknown"}).`);
           process.exit(0);
         } catch (relaunchError) {
-          error(`Failed during llm-router auto-relaunch: ${relaunchError instanceof Error ? relaunchError.message : String(relaunchError)}`);
+          error(`Failed during LLM Router auto-relaunch: ${relaunchError instanceof Error ? relaunchError.message : String(relaunchError)}`);
           process.exit(1);
         }
       })();

package/src/node/startup-manager.js

@@ -1,5 +1,5 @@
 /**
- * OS startup integration for llm-router.
+ * OS startup integration for LLM Router.
  * Supports macOS LaunchAgent and Linux systemd --user service.
  */
 
@@ -58,14 +58,14 @@ export function resolveStartupCliEntryPath({
   }
 
   const nodeBinDir = path.dirname(execPath);
-  for (const binName of ["llm-router", "llm-router-route"]) {
+  for (const binName of ["llr", "llm-router", "llm-router-route"]) {
     const candidate = path.join(nodeBinDir, binName);
     if (exists(candidate)) return candidate;
   }
 
   if (envCliPath) return envCliPath;
   if (argvCliPath) return path.resolve(argvCliPath);
-  throw new Error("Unable to resolve llm-router CLI entry path.");
+  throw new Error("Unable to resolve the LLM Router CLI entry path.");
 }
 
 function makeExecArgs({ configPath }) {

package/src/node/web-command.js

@@ -67,7 +67,7 @@ export async function runWebCommand(options = {}) {
     return {
       ok: false,
       exitCode: 1,
-      errorMessage: `Failed to start llm-router web console: ${startError instanceof Error ? startError.message : String(startError)}`
+      errorMessage: `Failed to start the LLM Router web console: ${startError instanceof Error ? startError.message : String(startError)}`
     };
   }
 

package/src/node/web-console-assets.js

@@ -3,7 +3,7 @@ import { WEB_CONSOLE_CSS } from "./web-console-styles.generated.js";
 export { WEB_CONSOLE_CSS };
 
 export function renderWebConsoleHtml({
-  title = "LLM Router Web",
+  title = "LLM Router Web Console",
   headHtml = "",
   bodyHtml = ""
 } = {}) {