@khanglvm/llm-router 2.4.0 → 2.5.1

package/src/runtime/config.js

@@ -9,6 +9,12 @@ import {
   CLAUDE_CODE_SUBSCRIPTION_MODELS
 } from "./subscription-constants.js";
 import { sanitizeRuntimeMetadata } from "../shared/local-router-defaults.js";
+import {
+  LOCAL_RUNTIME_PROVIDER_TYPE,
+  collectDuplicateLocalVariantModelIds,
+  materializeLocalVariantProvider,
+  normalizeLocalModelsMetadata
+} from "./local-models.js";
 
 export const CONFIG_VERSION = 2;
 export const MIN_SUPPORTED_CONFIG_VERSION = 1;
@@ -1769,12 +1775,15 @@ export function normalizeRuntimeConfig(rawConfig, options = {}) {
   const raw = shouldMigrate
     ? migrateRuntimeConfig(rawInput, { targetVersion })
     : rawInput;
-  const providers = sanitizeModelFallbackReferences(
-    toArray(raw.providers)
-    .map(normalizeProvider)
-    .filter(Boolean)
-    .filter((provider) => provider.enabled !== false)
-  );
+  const localModels = normalizeLocalModelsMetadata(raw.metadata?.localModels);
+  const providers = sanitizeModelFallbackReferences([
+    ...toArray(raw.providers)
+      .map(normalizeProvider)
+      .filter(Boolean)
+      .filter((provider) => provider.enabled !== false)
+      .filter((provider) => provider.type !== LOCAL_RUNTIME_PROVIDER_TYPE),
+    ...materializeLocalVariantProvider({ metadata: { localModels } })
+  ]);
   const modelAliasResult = normalizeModelAliases(raw.modelAliases || raw["model-aliases"]);
   const rawDefaultModel = typeof raw.defaultModel === "string"
     ? raw.defaultModel
@@ -1816,7 +1825,10 @@ export function normalizeRuntimeConfig(rawConfig, options = {}) {
     ...(webSearch ? { webSearch } : {}),
     ...(claudeCode && Object.keys(claudeCode).length > 0 ? { claudeCode } : {}),
     ollama,
-    metadata: sanitizeRuntimeMetadata(raw.metadata)
+    metadata: sanitizeRuntimeMetadata({
+      ...(normalizeMetadataObject(raw.metadata) || {}),
+      localModels
+    })
   };
   Object.defineProperty(normalized, NORMALIZATION_ISSUES_SYMBOL, {
     value: {
@@ -2185,6 +2197,9 @@ export function validateRuntimeConfig(config, { requireMasterKey = false, requir
   validateProviderRateLimits(config, routingIndex, errors);
   validateModelAliases(config, routingIndex, errors);
   validateAmpConfig(config, routingIndex, errors);
+  for (const duplicateModelId of collectDuplicateLocalVariantModelIds(config.metadata?.localModels)) {
+    errors.push(`Duplicate local variant model id '${duplicateModelId}'.`);
+  }
 
   if (requireMasterKey && !config.masterKey) {
     errors.push("masterKey is required for worker deployment/export.");

package/src/runtime/handler/provider-translation.js

@@ -3,7 +3,10 @@ import {
   claudeEventToOpenAIChunks,
   initClaudeToOpenAIState
 } from "../../translator/response/claude-to-openai.js";
-import { finalizeOpenAIToClaudeStream } from "../../translator/response/openai-to-claude.js";
+import {
+  finalizeOpenAIToClaudeStream,
+  normalizeOpenAIUsageToClaude
+} from "../../translator/response/openai-to-claude.js";
 import { passthroughResponseWithCors, withCorsHeaders } from "./http.js";
 
 function normalizeOpenAIContent(content) {
@@ -120,10 +123,7 @@ export function convertOpenAINonStreamToClaude(result, fallbackModel = "unknown"
     content,
     stop_reason: convertOpenAIFinishReason(resolveOpenAINonStreamFinishReason(choice)),
     stop_sequence: null,
-    usage: {
-      input_tokens: result?.usage?.prompt_tokens || 0,
-      output_tokens: result?.usage?.completion_tokens || 0
-    }
+    usage: normalizeOpenAIUsageToClaude(result?.usage)
   };
 }
 

package/src/runtime/local-models.js

@@ -0,0 +1,168 @@
+export const LOCAL_RUNTIME_PROVIDER_TYPE = "local-runtime";
+export const LOCAL_RUNTIME_PROVIDER_ID = "local-models";
+export const LOCAL_RUNTIME_BASE_URL = "http://127.0.0.1:39391/v1";
+
+function isPlainObject(value) {
+  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+
+function clonePlainObject(value) {
+  return isPlainObject(value) ? { ...value } : {};
+}
+
+function normalizeString(value) {
+  return typeof value === "string" ? value.trim() : "";
+}
+
+function normalizePositiveNumber(value) {
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed) || parsed <= 0) return undefined;
+  return Math.floor(parsed);
+}
+
+function normalizeLocalModelLibraryEntry(key, entry) {
+  if (!isPlainObject(entry)) return null;
+
+  const normalized = {
+    ...entry,
+    id: normalizeString(entry.id) || key
+  };
+
+  for (const field of ["source", "displayName", "path", "availability"]) {
+    if (field in normalized) {
+      const value = normalizeString(normalized[field]);
+      if (value) normalized[field] = value;
+      else delete normalized[field];
+    }
+  }
+
+  return normalized;
+}
+
+function normalizeLocalModelVariantEntry(key, entry) {
+  if (!isPlainObject(entry)) return null;
+
+  const normalized = {
+    ...entry,
+    key: normalizeString(entry.key) || key,
+    baseModelId: normalizeString(entry.baseModelId),
+    id: normalizeString(entry.id),
+    name: normalizeString(entry.name),
+    runtime: normalizeString(entry.runtime),
+    enabled: entry.enabled === true,
+    preload: entry.preload === true
+  };
+
+  if ("preset" in normalized) {
+    const preset = normalizeString(normalized.preset);
+    if (preset) normalized.preset = preset;
+    else delete normalized.preset;
+  }
+
+  const contextWindow = normalizePositiveNumber(entry.contextWindow);
+  if (contextWindow !== undefined) normalized.contextWindow = contextWindow;
+  else delete normalized.contextWindow;
+
+  const estimatedBytes = normalizePositiveNumber(entry.estimatedBytes);
+  if (estimatedBytes !== undefined) normalized.estimatedBytes = estimatedBytes;
+  else delete normalized.estimatedBytes;
+
+  if (isPlainObject(entry.capabilities)) normalized.capabilities = { ...entry.capabilities };
+  else delete normalized.capabilities;
+
+  if ("availability" in normalized) {
+    const availability = normalizeString(normalized.availability);
+    if (availability) normalized.availability = availability;
+    else delete normalized.availability;
+  }
+
+  return normalized;
+}
+
+export function normalizeLocalModelsMetadata(raw = {}) {
+  const source = isPlainObject(raw) ? raw : {};
+  const runtime = clonePlainObject(source.runtime);
+  const capacity = clonePlainObject(source.capacity);
+  const library = {};
+  const variants = {};
+
+  for (const [key, value] of Object.entries(clonePlainObject(source.library))) {
+    const normalizedEntry = normalizeLocalModelLibraryEntry(normalizeString(key), value);
+    if (!normalizedEntry) continue;
+    library[normalizedEntry.id || key] = normalizedEntry;
+  }
+
+  for (const [key, value] of Object.entries(clonePlainObject(source.variants))) {
+    const normalizedEntry = normalizeLocalModelVariantEntry(normalizeString(key), value);
+    if (!normalizedEntry?.key) continue;
+    variants[normalizedEntry.key] = normalizedEntry;
+  }
+
+  return {
+    runtime,
+    library,
+    variants,
+    capacity
+  };
+}
+
+export function collectDuplicateLocalVariantModelIds(localModelsMetadata) {
+  const metadata = normalizeLocalModelsMetadata(localModelsMetadata);
+  const seen = new Set();
+  const duplicates = new Set();
+
+  for (const variant of Object.values(metadata.variants)) {
+    const modelId = normalizeString(variant?.id);
+    if (!modelId) continue;
+    if (seen.has(modelId)) duplicates.add(modelId);
+    else seen.add(modelId);
+  }
+
+  return [...duplicates];
+}
+
+export function materializeLocalVariantProvider(config = {}) {
+  const metadata = normalizeLocalModelsMetadata(config?.metadata?.localModels);
+  const models = [];
+
+  for (const variant of Object.values(metadata.variants)) {
+    if (!variant || variant.enabled !== true) continue;
+    if (!variant.id) continue;
+
+    const baseModel = metadata.library[variant.baseModelId] || null;
+    const materialized = {
+      id: variant.id,
+      enabled: true,
+      metadata: {
+        localVariantKey: variant.key,
+        baseModelId: variant.baseModelId,
+        runtime: variant.runtime,
+        preload: variant.preload === true,
+        availability: variant.availability || baseModel?.availability || "available",
+        capacityState: variant.capacityState,
+        estimatedBytes: variant.estimatedBytes
+      }
+    };
+
+    if (variant.name) materialized.name = variant.name;
+    if (variant.contextWindow !== undefined) materialized.contextWindow = variant.contextWindow;
+    if (isPlainObject(variant.capabilities)) materialized.capabilities = { ...variant.capabilities };
+
+    models.push(materialized);
+  }
+
+  if (models.length === 0) return [];
+
+  return [{
+    id: LOCAL_RUNTIME_PROVIDER_ID,
+    name: "Local Models",
+    type: LOCAL_RUNTIME_PROVIDER_TYPE,
+    baseUrl: LOCAL_RUNTIME_BASE_URL,
+    format: "openai",
+    formats: ["openai"],
+    apiKey: "local-runtime",
+    enabled: true,
+    models,
+    rateLimits: []
+  }];
+}

package/src/translator/response/openai-to-claude.js

@@ -4,6 +4,73 @@
 
 import { FORMATS } from "../formats.js";
 
+const DEFAULT_CLAUDE_SERVER_TOOL_USE = Object.freeze({
+  web_search_requests: 0,
+  web_fetch_requests: 0
+});
+
+const DEFAULT_CLAUDE_CACHE_CREATION = Object.freeze({
+  ephemeral_1h_input_tokens: 0,
+  ephemeral_5m_input_tokens: 0
+});
+
+function toNonNegativeNumber(value, fallback = 0) {
+  const parsed = Number(value);
+  return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback;
+}
+
+function normalizeClaudeServerToolUse(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return { ...DEFAULT_CLAUDE_SERVER_TOOL_USE };
+  }
+
+  return {
+    web_search_requests: toNonNegativeNumber(value.web_search_requests),
+    web_fetch_requests: toNonNegativeNumber(value.web_fetch_requests)
+  };
+}
+
+function normalizeClaudeCacheCreation(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return { ...DEFAULT_CLAUDE_CACHE_CREATION };
+  }
+
+  return {
+    ephemeral_1h_input_tokens: toNonNegativeNumber(value.ephemeral_1h_input_tokens),
+    ephemeral_5m_input_tokens: toNonNegativeNumber(value.ephemeral_5m_input_tokens)
+  };
+}
+
+export function normalizeOpenAIUsageToClaude(rawUsage) {
+  const usage = rawUsage && typeof rawUsage === "object" && !Array.isArray(rawUsage)
+    ? rawUsage
+    : {};
+  const cacheCreation = normalizeClaudeCacheCreation(usage.cache_creation);
+  const inputTokens = usage.prompt_tokens ?? usage.input_tokens;
+  const outputTokens = usage.completion_tokens ?? usage.output_tokens;
+  const cacheCreationInputTokens = usage.cache_creation_input_tokens
+    ?? (cacheCreation.ephemeral_1h_input_tokens + cacheCreation.ephemeral_5m_input_tokens);
+  const speed = typeof usage.speed === "string" && usage.speed.trim()
+    ? usage.speed.trim()
+    : "standard";
+  const serviceTier = typeof usage.service_tier === "string" && usage.service_tier.trim()
+    ? usage.service_tier.trim()
+    : "standard";
+
+  return {
+    input_tokens: toNonNegativeNumber(inputTokens),
+    cache_creation_input_tokens: toNonNegativeNumber(cacheCreationInputTokens),
+    cache_read_input_tokens: toNonNegativeNumber(usage.cache_read_input_tokens),
+    output_tokens: toNonNegativeNumber(outputTokens),
+    server_tool_use: normalizeClaudeServerToolUse(usage.server_tool_use),
+    service_tier: serviceTier,
+    cache_creation: cacheCreation,
+    inference_geo: typeof usage.inference_geo === "string" ? usage.inference_geo : "",
+    iterations: Array.isArray(usage.iterations) ? usage.iterations : [],
+    speed
+  };
+}
+
 /**
  * Convert OpenAI stream chunk to Claude format
  */
@@ -16,13 +83,7 @@ export function openaiToClaudeResponse(chunk, state) {
 
   // Track usage
   if (chunk.usage && typeof chunk.usage === "object") {
-    const promptTokens = chunk.usage.prompt_tokens || 0;
-    const outputTokens = chunk.usage.completion_tokens || 0;
-    
-    state.usage = {
-      input_tokens: promptTokens,
-      output_tokens: outputTokens
-    };
+    state.usage = normalizeOpenAIUsageToClaude(chunk.usage);
   }
 
   // First chunk - send message_start
@@ -264,7 +325,7 @@ function ensureMessageStart(state, results, chunk = undefined) {
       content: [],
       stop_reason: null,
       stop_sequence: null,
-      usage: { input_tokens: 0, output_tokens: 0 }
+      usage: normalizeOpenAIUsageToClaude(state.usage)
     }
   });
 }
@@ -297,7 +358,7 @@ export function finalizeOpenAIToClaudeStream(state, { force = false } = {}) {
     results.push({
       type: "message_delta",
       delta: { stop_reason: convertFinishReason(normalizedFinishReason) },
-      usage: state.usage || { input_tokens: 0, output_tokens: 0 }
+      usage: normalizeOpenAIUsageToClaude(state.usage)
     });
     state.messageDeltaSent = true;
   }