@khanglvm/llm-router 2.5.1 → 2.6.0

package/src/runtime/handler/provider-call.js

@@ -135,6 +135,21 @@ function queueLargeRequestEvent(onLargeRequestLog, payload) {
   }
 }
 
+async function resolveRequestProviderUrl(provider, plan, candidate, runtimeFlags) {
+  if (provider?.type === "local-runtime" && typeof runtimeFlags?.resolveLocalRuntimeBaseUrl === "function") {
+    const dynamicBaseUrl = await runtimeFlags.resolveLocalRuntimeBaseUrl({
+      candidate,
+      targetFormat: plan.targetFormat,
+      requestKind: plan.requestKind
+    });
+    if (dynamicBaseUrl) {
+      return resolveProviderUrl({ ...provider, baseUrl: dynamicBaseUrl }, plan.targetFormat, plan.requestKind);
+    }
+  }
+
+  return resolveProviderUrl(provider, plan.targetFormat, plan.requestKind);
+}
+
 function maybeQueueLargeRequestLog({
   env,
   onLargeRequestLog,
@@ -1074,8 +1089,27 @@ export async function makeProviderCall({
   }
 
   const executeHttpProviderRequest = async (plan) => {
-    const providerUrl = resolveProviderUrl(provider, plan.targetFormat, plan.requestKind);
-    if (!providerUrl) return null;
+    let providerUrl;
+    try {
+      providerUrl = await resolveRequestProviderUrl(provider, plan, candidate, runtimeFlags);
+    } catch (error) {
+      return jsonResponse({
+        type: "error",
+        error: {
+          type: "api_error",
+          message: error instanceof Error ? error.message : String(error)
+        }
+      }, 503);
+    }
+    if (!providerUrl) {
+      return jsonResponse({
+        type: "error",
+        error: {
+          type: "configuration_error",
+          message: `Provider ${provider.id} has invalid baseUrl.`
+        }
+      }, 500);
+    }
     const headers = mergeCachingHeaders(
       buildProviderHeaders(provider, env, plan.targetFormat),
       requestHeaders,
@@ -1114,22 +1148,6 @@ export async function makeProviderCall({
     }
   };
 
-  if (!resolveProviderUrl(provider, activePlan.targetFormat, activePlan.requestKind)) {
-    return {
-      ok: false,
-      status: 500,
-      retryable: false,
-      errorKind: "configuration_error",
-      response: jsonResponse({
-        type: "error",
-        error: {
-          type: "configuration_error",
-          message: `Provider ${provider.id} has invalid baseUrl.`
-        }
-      }, 500)
-    };
-  }
-
   let response;
   try {
     response = await executeHttpProviderRequest(activePlan);

package/src/runtime/handler/runtime-policy.js

@@ -27,7 +27,10 @@ export function resolveRuntimeFlags(options = {}, env = {}) {
     workerRuntime,
     workerSafeMode,
     allowBestEffortStatefulRouting,
-    statefulRoutingEnabled: !workerSafeMode || allowBestEffortStatefulRouting
+    statefulRoutingEnabled: !workerSafeMode || allowBestEffortStatefulRouting,
+    ...(typeof options.resolveLocalRuntimeBaseUrl === "function"
+      ? { resolveLocalRuntimeBaseUrl: options.resolveLocalRuntimeBaseUrl }
+      : {})
   };
 }
 

package/src/runtime/local-models.js

@@ -20,6 +20,34 @@ function normalizePositiveNumber(value) {
   return Math.floor(parsed);
 }
 
+function normalizeRuntimeProfile(raw = {}) {
+  const source = isPlainObject(raw) ? raw : {};
+  const overrides = isPlainObject(source.overrides) ? { ...source.overrides } : {};
+  const extraArgs = Array.isArray(source.extraArgs)
+    ? source.extraArgs.map((value) => normalizeString(value)).filter(Boolean)
+    : [];
+
+  return {
+    mode: normalizeString(source.mode) === "custom" ? "custom" : "auto",
+    preset: normalizeString(source.preset) || "balanced",
+    overrides,
+    extraArgs,
+    lastKnownGood: isPlainObject(source.lastKnownGood) ? { ...source.lastKnownGood } : null,
+    lastFailure: isPlainObject(source.lastFailure) ? { ...source.lastFailure } : null
+  };
+}
+
+function normalizeRuntimeStatus(raw = {}) {
+  const source = isPlainObject(raw) ? raw : {};
+
+  return {
+    activeInstanceId: normalizeString(source.activeInstanceId),
+    lastFailure: isPlainObject(source.lastFailure) ? { ...source.lastFailure } : null,
+    lastStartedAt: normalizeString(source.lastStartedAt),
+    lastHealthyAt: normalizeString(source.lastHealthyAt)
+  };
+}
+
 function normalizeLocalModelLibraryEntry(key, entry) {
   if (!isPlainObject(entry)) return null;
 
@@ -76,6 +104,14 @@ function normalizeLocalModelVariantEntry(key, entry) {
     else delete normalized.availability;
   }
 
+  if (normalized.runtime === "llamacpp") {
+    normalized.runtimeProfile = normalizeRuntimeProfile(entry.runtimeProfile);
+    normalized.runtimeStatus = normalizeRuntimeStatus(entry.runtimeStatus);
+  } else {
+    delete normalized.runtimeProfile;
+    delete normalized.runtimeStatus;
+  }
+
   return normalized;
 }