@khanglvm/llm-router 2.6.0 → 2.6.2

package/src/node/llamacpp-runtime.js

@@ -2,11 +2,6 @@ import path from "node:path";
 import os from "node:os";
 import { existsSync } from "node:fs";
 import { spawn, spawnSync } from "node:child_process";
-import { setTimeout as delay } from "node:timers/promises";
-import { deriveLlamacppLaunchProfile } from "./llamacpp-runtime-profile.js";
-import { createLlamacppManagedRuntimeRegistry } from "./llamacpp-managed-runtime.js";
-import { listListeningPids as listListeningPidsForPort } from "./port-reclaim.js";
-import { stopProcessByPid as stopProcessByPidForRuntime } from "./instance-state.js";
 
 export const LLAMACPP_DEFAULT_HOST = "127.0.0.1";
 export const LLAMACPP_DEFAULT_PORT = 39391;
@@ -21,8 +16,8 @@ const COMMON_SOURCE_BUILD_PATHS = Object.freeze([
   "src/llama-cpp-turboquant/build/bin/llama-server",
   "src/llama.cpp-turboquant/build/bin/llama-server"
 ]);
-const managedLlamacppRuntimeRegistry = createLlamacppManagedRuntimeRegistry();
-let inFlightConfiguredStartCount = 0;
+
+let managedLlamacppRuntime = null;
 
 function isPlainObject(value) {
   return Boolean(value) && typeof value === "object" && !Array.isArray(value);
@@ -44,34 +39,6 @@ function normalizePathEntries(entries) {
     : [];
 }
 
-function buildRuntimeProfileHash({ command, host, port, args = [] } = {}) {
-  const normalizedArgs = Array.isArray(args) ? args.filter(Boolean) : [];
-  return `${normalizeString(command)}|${normalizeString(host)}|${String(normalizePort(port, LLAMACPP_DEFAULT_PORT))}|${normalizedArgs.join("\u001f")}`;
-}
-
-function isManagedRuntimeAlive(instance) {
-  const child = instance?.child;
-  if (!child) return false;
-  return child.exitCode === null && child.killed !== true;
-}
-
-function normalizeListeningPidResult(result) {
-  if (result && typeof result === "object" && result.ok === false) {
-    return { ok: false, pids: [] };
-  }
-  if (Array.isArray(result)) {
-    return result
-      .map((value) => Number(value))
-      .filter((pid) => Number.isInteger(pid) && pid > 0);
-  }
-  if (result && typeof result === "object" && Array.isArray(result.pids)) {
-    return result.pids
-      .map((value) => Number(value))
-      .filter((pid) => Number.isInteger(pid) && pid > 0);
-  }
-  return [];
-}
-
 function readConfiguredLlamacppRuntime(config) {
   const runtime = config?.metadata?.localModels?.runtime?.llamacpp;
   if (!isPlainObject(runtime)) {
@@ -105,8 +72,6 @@ function buildPreloadModels(config) {
     if (!modelPath) continue;
     preloadModels.push({
       variantId: normalizeString(variant.id),
-      variant,
-      baseModel,
       modelPath,
       contextWindow: Number.isFinite(Number(variant.contextWindow)) ? Number(variant.contextWindow) : undefined
     });
@@ -114,15 +79,6 @@ function buildPreloadModels(config) {
   return preloadModels;
 }
 
-function detectLlamacppSystemProfile(system = {}) {
-  const totalMemoryBytes = Number(system?.totalMemoryBytes);
-  return {
-    platform: normalizeString(system?.platform) || process.platform,
-    unifiedMemory: system?.unifiedMemory === true || process.platform === "darwin",
-    totalMemoryBytes: Number.isFinite(totalMemoryBytes) && totalMemoryBytes > 0 ? totalMemoryBytes : os.totalmem()
-  };
-}
-
 export function detectLlamacppCandidates({
   envPathEntries = process.env.PATH?.split(path.delimiter) || [],
   homeDir = os.homedir(),
@@ -166,18 +122,16 @@ export function buildLlamacppLaunchArgs({
   command,
   host = LLAMACPP_DEFAULT_HOST,
   port = LLAMACPP_DEFAULT_PORT,
-  preloadModels = [],
-  launchProfile = null
+  preloadModels = []
 } = {}) {
   const firstModel = Array.isArray(preloadModels) ? preloadModels[0] : null;
   const args = [
     normalizeString(command),
     "--host", normalizeString(host) || LLAMACPP_DEFAULT_HOST,
-    "--port", String(normalizePort(port, LLAMACPP_DEFAULT_PORT)),
-    ...((Array.isArray(launchProfile?.args) ? launchProfile.args : []).filter(Boolean))
+    "--port", String(normalizePort(port, LLAMACPP_DEFAULT_PORT))
   ];
 
-  if (!launchProfile && firstModel?.modelPath) {
+  if (firstModel?.modelPath) {
     args.push("-m", firstModel.modelPath);
     if (Number.isFinite(Number(firstModel.contextWindow)) && Number(firstModel.contextWindow) > 0) {
       args.push("-c", String(Math.floor(Number(firstModel.contextWindow))));
@@ -187,31 +141,6 @@ export function buildLlamacppLaunchArgs({
   return args.filter(Boolean);
 }
 
-export async function spawnManagedLlamacppRuntime({
-  command,
-  host = LLAMACPP_DEFAULT_HOST,
-  port = LLAMACPP_DEFAULT_PORT,
-  launchProfile
-} = {}, {
-  spawnImpl = spawn
-} = {}) {
-  const args = buildLlamacppLaunchArgs({
-    command,
-    host,
-    port,
-    launchProfile
-  });
-  const child = spawnImpl(args[0], args.slice(1), { stdio: "ignore" });
-  return {
-    pid: child?.pid,
-    child,
-    host,
-    port,
-    baseUrl: `http://${host}:${port}/v1`,
-    args
-  };
-}
-
 export function parseLlamacppValidationOutput(output = "") {
   const text = String(output || "").trim();
   const lowered = text.toLowerCase();
@@ -270,142 +199,78 @@ async function startConfiguredRuntime(config, {
   requireAutostart = true
 } = {}, {
   spawnSyncImpl = spawnSync,
-  spawnImpl = spawn,
-  system = undefined,
-  listListeningPids = undefined,
-  stopProcessByPid = undefined
+  spawnImpl = spawn
 } = {}) {
-  inFlightConfiguredStartCount += 1;
-  try {
-    const runtime = readConfiguredLlamacppRuntime(config);
-    if (requireAutostart && !runtime.startWithRouter) {
-      return { ok: true, skipped: true, reason: "autostart-disabled" };
-    }
+  const runtime = readConfiguredLlamacppRuntime(config);
+  if (requireAutostart && !runtime.startWithRouter) {
+    return { ok: true, skipped: true, reason: "autostart-disabled" };
+  }
 
-    if (!runtime.command) {
-      const errorMessage = "llama.cpp autostart is enabled, but no runtime command is configured.";
-      error(errorMessage);
-      return { ok: false, errorMessage };
-    }
+  if (!runtime.command) {
+    const errorMessage = "llama.cpp autostart is enabled, but no runtime command is configured.";
+    error(errorMessage);
+    return { ok: false, errorMessage };
+  }
 
-    const preloadModels = buildPreloadModels(config);
-    const firstModel = Array.isArray(preloadModels) ? preloadModels[0] : null;
-    const launchProfile = firstModel?.variant && firstModel?.baseModel
-      ? deriveLlamacppLaunchProfile({
-        variant: firstModel.variant,
-        baseModel: firstModel.baseModel,
-        system: detectLlamacppSystemProfile(system)
-      })
-      : null;
-    const args = buildLlamacppLaunchArgs({
-      command: runtime.command,
-      host: runtime.host,
-      port: runtime.port,
-      preloadModels,
-      launchProfile
-    });
-    const variantKey = normalizeString(firstModel?.variant?.key || firstModel?.variantId) || "default";
-    const profileHash = buildRuntimeProfileHash({
-      command: runtime.command,
-      host: runtime.host,
-      port: runtime.port,
-      args: args.slice(1)
-    });
-    const listListeningPidsFn = typeof listListeningPids === "function"
-      ? listListeningPids
-      : (port) => listListeningPidsForPort(port, { spawnSync: spawnSyncImpl });
-    const stopProcessByPidFn = typeof stopProcessByPid === "function"
-      ? stopProcessByPid
-      : (pid) => stopProcessByPidForRuntime(pid);
-    await managedLlamacppRuntimeRegistry.reconcile({
-      listListeningPids: async (port) => normalizeListeningPidResult(await listListeningPidsFn(port)),
-      stopProcessByPid: async (pid) => stopProcessByPidFn(pid)
-    });
+  if (managedLlamacppRuntime
+    && managedLlamacppRuntime.command === runtime.command
+    && managedLlamacppRuntime.host === runtime.host
+    && managedLlamacppRuntime.port === runtime.port
+    && managedLlamacppRuntime.child?.exitCode === null
+    && managedLlamacppRuntime.child?.killed !== true) {
+    return { ok: true, alreadyRunning: true, runtime: managedLlamacppRuntime };
+  }
 
-    const existing = managedLlamacppRuntimeRegistry
-      .snapshot()
-      .find((instance) => (
-        instance.variantKey === variantKey
-        && instance.profileHash === profileHash
-        && isManagedRuntimeAlive(instance)
-      ));
-    if (existing) {
-      return { ok: true, alreadyRunning: true, runtime: existing };
-    }
+  const validation = validateLlamacppCommand(runtime.command, { spawnSyncImpl });
+  if (!validation.ok) {
+    error(validation.errorMessage || `Failed validating llama.cpp runtime '${runtime.command}'.`);
+    return validation;
+  }
 
-    const validation = validateLlamacppCommand(runtime.command, { spawnSyncImpl });
-    if (!validation.ok) {
-      error(validation.errorMessage || `Failed validating llama.cpp runtime '${runtime.command}'.`);
-      return validation;
-    }
+  const preloadModels = buildPreloadModels(config);
+  const args = buildLlamacppLaunchArgs({
+    command: runtime.command,
+    host: runtime.host,
+    port: runtime.port,
+    preloadModels
+  });
 
-    try {
-      const managedRuntime = await managedLlamacppRuntimeRegistry.ensureRuntimeForVariant({
-        variantKey,
-        profileHash,
-        launchArgs: args.slice(1),
-        preferredPort: runtime.port
-      }, {
-        spawnRuntime: async ({ port }) => new Promise((resolve, reject) => {
-          let settled = false;
-          const allocatedArgs = buildLlamacppLaunchArgs({
-            command: runtime.command,
-            host: runtime.host,
-            port,
-            preloadModels,
-            launchProfile
-          });
-          const child = spawnImpl(allocatedArgs[0], allocatedArgs.slice(1), {
-            stdio: "ignore"
-          });
-          const expectedInstanceId = `${variantKey}:${profileHash}:${port}`;
-          if (child && child.__llamacppManagedExitHookAttached !== true) {
-            child.__llamacppManagedExitHookAttached = true;
-            child.once("exit", () => {
-              void managedLlamacppRuntimeRegistry.untrackInstance(expectedInstanceId);
-            });
-          }
-
-          const settleResolve = (value) => {
-            if (settled) return;
-            settled = true;
-            resolve(value);
-          };
-          const settleReject = (reason) => {
-            if (settled) return;
-            settled = true;
-            reject(reason);
-          };
-
-          child.once("spawn", () => {
-            if (typeof child.unref === "function") child.unref();
-            settleResolve({
-              pid: child?.pid,
-              child,
-              command: runtime.command,
-              host: runtime.host,
-              port,
-              args: allocatedArgs,
-              baseUrl: `http://${runtime.host}:${port}/v1`
-            });
-          });
-          child.once("error", (spawnError) => {
-            settleReject(spawnError);
-          });
-        }),
-        waitForHealthy: async (instance) => instance
+  return new Promise((resolve) => {
+    let settled = false;
+    const child = spawnImpl(args[0], args.slice(1), {
+      stdio: "ignore"
+    });
+
+    const finish = (result) => {
+      if (settled) return;
+      settled = true;
+      resolve(result);
+    };
+
+    child.once("spawn", () => {
+      managedLlamacppRuntime = {
+        child,
+        command: runtime.command,
+        host: runtime.host,
+        port: runtime.port,
+        args
+      };
+      child.once("exit", () => {
+        if (managedLlamacppRuntime?.child === child) {
+          managedLlamacppRuntime = null;
+        }
       });
+      if (typeof child.unref === "function") child.unref();
+      line(`Started llama.cpp runtime on http://${runtime.host}:${runtime.port}${validation.isTurboQuant ? " (TurboQuant detected)" : ""}.`);
+      finish({ ok: true, runtime: managedLlamacppRuntime, validation });
+    });
 
-      line(`Started llama.cpp runtime on http://${managedRuntime.host}:${managedRuntime.port}${validation.isTurboQuant ? " (TurboQuant detected)" : ""}.`);
-      return { ok: true, runtime: managedRuntime, validation };
-    } catch (spawnError) {
+    child.once("error", (spawnError) => {
       const errorMessage = spawnError instanceof Error ? spawnError.message : String(spawnError);
       error(`Failed starting llama.cpp runtime: ${errorMessage}`);
-      return { ok: false, errorMessage };
-    }
-  } finally {
-    inFlightConfiguredStartCount = Math.max(0, inFlightConfiguredStartCount - 1);
-  }
+      finish({ ok: false, errorMessage });
+    });
+  });
 }
 
 export async function ensureConfiguredLlamacppRuntimeStarted(config, callbacks = {}, deps = {}) {
@@ -422,66 +287,23 @@ export async function startConfiguredLlamacppRuntime(config, callbacks = {}, dep
   }, deps);
 }
 
-export function getManagedLlamacppRuntimeSnapshot() {
-  return managedLlamacppRuntimeRegistry.snapshot().map((instance) => {
-    const { child: _child, ...rest } = instance || {};
-    return JSON.parse(JSON.stringify(rest));
-  });
-}
-
 export async function stopManagedLlamacppRuntime({
   line = () => {},
   error = () => {}
 } = {}) {
-  while (inFlightConfiguredStartCount > 0) {
-    await delay(0);
-  }
-  if (typeof managedLlamacppRuntimeRegistry.waitForInFlightStarts === "function") {
-    await managedLlamacppRuntimeRegistry.waitForInFlightStarts();
-  }
-  const instances = managedLlamacppRuntimeRegistry.snapshot();
-  if (instances.length === 0) {
+  const active = managedLlamacppRuntime;
+  if (!active?.child) {
     return { ok: true, skipped: true, reason: "not-running" };
   }
 
-  const failures = [];
-  let stoppedCount = 0;
-  let pendingExitCount = 0;
-  for (const instance of instances) {
-    try {
-      if (instance?.owner === "llm-router" && typeof instance?.child?.kill === "function") {
-        const killResult = instance.child.kill("SIGTERM");
-        if (killResult !== false) {
-          stoppedCount += 1;
-        }
-      }
-      if (!isManagedRuntimeAlive(instance)) {
-        await managedLlamacppRuntimeRegistry.untrackInstance(instance?.instanceId);
-      } else {
-        pendingExitCount += 1;
-      }
-    } catch (stopError) {
-      const errorMessage = stopError instanceof Error ? stopError.message : String(stopError);
-      failures.push(errorMessage);
-      error(`Failed stopping llama.cpp runtime: ${errorMessage}`);
-    }
-  }
-
-  if (stoppedCount > 0) {
-    if (pendingExitCount === 0) {
-      line(stoppedCount === 1 ? "Stopped managed llama.cpp runtime." : `Stopped ${stoppedCount} managed llama.cpp runtimes.`);
-    } else {
-      line(stoppedCount === 1
-        ? "Stop signal sent to managed llama.cpp runtime; waiting for exit."
-        : `Stop signal sent to ${stoppedCount} managed llama.cpp runtimes; waiting for exits.`);
-    }
+  managedLlamacppRuntime = null;
+  try {
+    active.child.kill("SIGTERM");
+    line("Stopped managed llama.cpp runtime.");
+    return { ok: true };
+  } catch (stopError) {
+    const errorMessage = stopError instanceof Error ? stopError.message : String(stopError);
+    error(`Failed stopping llama.cpp runtime: ${errorMessage}`);
+    return { ok: false, errorMessage };
   }
-
-  const completed = failures.length === 0 && pendingExitCount === 0;
-  return {
-    ok: completed,
-    stoppedCount,
-    pendingExitCount,
-    ...(failures.length > 0 ? { errorMessage: failures.join("; ") } : {})
-  };
 }

package/src/node/local-models-service.js

@@ -212,42 +212,19 @@ export async function saveLocalModelVariant(config, draft, {
       activeVariants,
       totalMemoryBytes: system.totalMemoryBytes
     });
-  if (!decision.allowed) {
+    if (!decision.allowed) {
       throw new Error(decision.reason);
     }
   }
 
-  const normalizedMetadata = normalizeLocalModelsMetadata({
-    variants: {
-      draft: normalizedDraft
-    }
-  });
-  const normalizedVariantDraft = Object.values(normalizedMetadata.variants)[0] || {};
-  const previousVariant = isPlainObject(next.metadata.localModels.variants[key])
-    ? next.metadata.localModels.variants[key]
-    : {};
-
   next.metadata.localModels.variants[key] = {
-    ...previousVariant,
+    ...(isPlainObject(next.metadata.localModels.variants[key]) ? next.metadata.localModels.variants[key] : {}),
     key,
     baseModelId,
     id: modelId,
     name,
     runtime,
     preset: normalizeString(normalizedDraft.preset),
-    runtimeProfile: runtime === "llamacpp"
-      ? normalizedVariantDraft.runtimeProfile
-      : undefined,
-    runtimeStatus: runtime === "llamacpp"
-      ? (isPlainObject(previousVariant.runtimeStatus)
-        ? previousVariant.runtimeStatus
-        : {
-          activeInstanceId: "",
-          lastFailure: null,
-          lastStartedAt: "",
-          lastHealthyAt: ""
-        })
-      : undefined,
     enabled: normalizedDraft.enabled === true,
     preload: normalizedDraft.preload === true,
     contextWindow: Number.isFinite(Number(normalizedDraft.contextWindow)) ? Number(normalizedDraft.contextWindow) : undefined,

package/src/node/local-server.js

@@ -13,10 +13,6 @@ import { readActivityLogSettings } from "../shared/local-router-defaults.js";
 import { appendActivityLogEntry, resolveActivityLogPath } from "./activity-log.js";
 import { appendLargeRequestLogEntry, resolveLargeRequestLogPath } from "./large-request-log.js";
 import { isLargeRequestLoggingEnabled } from "../runtime/handler/large-request-log.js";
-import {
-  startConfiguredLlamacppRuntime,
-  stopManagedLlamacppRuntime
-} from "./llamacpp-runtime.js";
 
 const DEFAULT_CONFIG_RELOAD_DEBOUNCE_MS = 300;
 const MAX_CONFIG_RELOAD_DEBOUNCE_MS = 5000;
@@ -38,10 +34,6 @@ function formatError(error) {
   return error instanceof Error ? error.message : String(error);
 }
 
-function normalizeString(value) {
-  return typeof value === "string" ? value.trim() : "";
-}
-
 function createLiveConfigStore({
   configPath,
   watchConfig = true,
@@ -245,39 +237,6 @@ async function writeFetchResponseToNode(res, response) {
   readable.pipe(res);
 }
 
-function buildVariantLlamacppRuntimeConfig(config, variantKey) {
-  const normalizedVariantKey = normalizeString(variantKey);
-  const runtime = config?.metadata?.localModels?.runtime?.llamacpp;
-  const variants = config?.metadata?.localModels?.variants;
-  const library = config?.metadata?.localModels?.library;
-  const variant = variants?.[normalizedVariantKey];
-  if (!runtime || !variant || variant.runtime !== "llamacpp") return null;
-
-  const baseModelId = normalizeString(variant?.baseModelId);
-  const baseModel = library?.[baseModelId];
-  if (!baseModel) return null;
-
-  return {
-    metadata: {
-      localModels: {
-        runtime: {
-          llamacpp: { ...runtime }
-        },
-        library: {
-          [baseModelId]: { ...baseModel }
-        },
-        variants: {
-          [normalizedVariantKey]: {
-            ...variant,
-            enabled: true,
-            preload: true
-          }
-        }
-      }
-    }
-  };
-}
-
 export async function startLocalRouteServer({
   port = FIXED_LOCAL_ROUTER_PORT,
   host = FIXED_LOCAL_ROUTER_HOST,
@@ -289,10 +248,7 @@ export async function startLocalRouteServer({
   validateConfig,
   onConfigReload,
   onConfigReloadError,
-  requireAuth = false,
-  createFetchHandlerImpl = createFetchHandler,
-  startConfiguredLlamacppRuntimeImpl = startConfiguredLlamacppRuntime,
-  stopManagedLlamacppRuntimeImpl = stopManagedLlamacppRuntime
+  requireAuth = false
 } = {}) {
   const reloadDebounceMs = resolveReloadDebounceMs(configReloadDebounceMs);
   const resolvedActivityLogPath = resolveActivityLogPath(configPath, activityLogPath);
@@ -314,22 +270,9 @@ export async function startLocalRouteServer({
   const initialConfig = await configStore.getConfig();
   activityLogEnabled = readActivityLogSettings(initialConfig).enabled;
 
-  const fetchHandler = createFetchHandlerImpl({
+  const fetchHandler = createFetchHandler({
     ignoreAuth: !requireAuth,
-    runtime: "node",
     getConfig: () => configStore.getConfig(),
-    resolveLocalRuntimeBaseUrl: async ({ candidate }) => {
-      const variantKey = candidate?.model?.metadata?.localVariantKey;
-      const config = await configStore.getConfig();
-      const targetedConfig = buildVariantLlamacppRuntimeConfig(config, variantKey);
-      if (!targetedConfig) return "";
-
-      const started = await startConfiguredLlamacppRuntimeImpl(targetedConfig);
-      if (!started?.ok) {
-        throw new Error(started?.errorMessage || `Failed starting local runtime for ${normalizeString(variantKey) || "unknown variant"}.`);
-      }
-      return normalizeString(started?.runtime?.baseUrl);
-    },
     defaultStateStoreBackend: "file",
     onActivityLog: (entry) => {
       if (!activityLogEnabled) return;
@@ -412,7 +355,6 @@ export async function startLocalRouteServer({
   server.close = (callback) => {
     shuttingDown = true;
     Promise.resolve()
-      .then(() => stopManagedLlamacppRuntimeImpl().catch(() => {}))
       .then(() => configStore.close())
       .then(() => (typeof fetchHandler.close === "function" ? fetchHandler.close() : undefined))
       .finally(() => {

package/src/node/provider-probe.js

@@ -310,6 +310,15 @@ function isTransientModelRuntimeError(result, message) {
   return patterns.some((pattern) => pattern.test(text));
 }
 
+function isOutputLimitReachedMessage(message) {
+  const text = String(message || "").toLowerCase();
+  if (!text) return false;
+  return (
+    text.includes("max_tokens") &&
+    (text.includes("output limit") || text.includes("token limit") || text.includes("finish"))
+  );
+}
+
 function isRateLimitResult(result, message) {
   const status = Number(result?.status || 0);
   if (status === 429) return true;
@@ -377,6 +386,15 @@ function classifyModelProbeResult(format, result) {
     };
   }
 
+  if (isOutputLimitReachedMessage(message)) {
+    return {
+      supported: true,
+      confirmed: true,
+      outcome: "output-limit",
+      message: message || "Request reached model but the probe token budget was too small."
+    };
+  }
+
   if (isUnsupportedModelMessage(message)) {
     return {
       supported: false,