@khanglvm/llm-router 2.5.2 → 2.6.0

package/CHANGELOG.md

@@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [2.6.0] - 2026-04-23
+
+### Added
+- Local `llama.cpp` variants can now persist a per-model runtime profile, including auto-tuned presets and custom launch overrides, so each GGUF variant can run with settings that match its own size and context shape instead of sharing one global `llama-server` startup profile.
+- The Web UI now exposes managed `llama.cpp` runtime health for Local Models, including tracked instance counts, healthy/stale summaries, and persisted runtime-profile data for each saved variant.
+
+### Changed
+- Local variant requests are now resolved through a managed per-variant `llama.cpp` runtime layer that can reuse compatible instances, allocate fallback ports safely, and start the right runtime configuration for the specific model variant without exposing multi-process lifecycle management to the user.
+- Hugging Face GGUF search/download flows now surface file size plus estimated runtime memory guidance directly in the Local Models workflow, making it easier to choose a viable quantization before download.
+
+### Fixed
+- Managed `llama.cpp` runtimes now reconcile stale tracked instances before reuse, avoid reserving dead immediate-exit servers, and drain pending shutdown/startup edges more reliably so local per-model routing does not leave behind stale `llama-server` processes.
+
 ## [2.5.2] - 2026-04-23
 
 ### Fixed

package/README.md

@@ -44,6 +44,9 @@ Open `llr` and use the **Local Models** tab to manage local inference sources al
 - **Native macOS browsing** — use the built-in file picker to choose a single GGUF file, scan a folder recursively for GGUF models, or browse directly to a local `llama-server` binary
 - **Managed + attached model library** — stale or moved files stay visible instead of crashing the app, and can be repaired by locating the file again or removed cleanly
 - **Router-visible local variants** — create friendly model variants with bounded presets, context-window metadata, preload toggles, and Mac unified-memory fit guidance with clearer safe/tight recommendations
+- **Per-variant llama.cpp tuning** — each local variant can store its own runtime profile so balanced, throughput, long-context, low-memory, or custom launch overrides do not fight over one shared global `llama-server` config
+- **Managed per-model runtimes** — the router automatically starts, reuses, and stops the right `llama.cpp` instance for the requested local variant, with stale-runtime cleanup handled internally instead of asking the user to manage separate servers
+- **GGUF size + memory guidance** — Hugging Face search results now show model file size plus estimated runtime memory fit guidance before download, helping choose viable quantizations faster
 - **Alias-ready local routing** — once saved, local variants behave like normal router models and can be used in aliases, capability flags, and fallback chains
 
 For v1, the managed download flow only searches public Hugging Face GGUF files and the fit guidance is tuned for Macs with unified memory.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@khanglvm/llm-router",
-  "version": "2.5.2",
+  "version": "2.6.0",
   "description": "LLM Router: single gateway endpoint for multi-provider LLMs with unified OpenAI+Anthropic format and seamless fallback",
   "keywords": [
     "llm-router",

package/src/node/huggingface-gguf.js

@@ -1,5 +1,6 @@
 import path from "node:path";
 import { promises as fs } from "node:fs";
+import { estimateLlamacppRuntimeBytes } from "./llamacpp-runtime-profile.js";
 
 const HUGGING_FACE_API_URL = "https://huggingface.co/api/models";
 const HUGGING_FACE_BASE_URL = "https://huggingface.co";
@@ -154,6 +155,13 @@ export function shapeHuggingFaceGgufResults(files, systemInfo = {}) {
       expectedContextWindow: systemInfo?.expectedContextWindow
     }, systemInfo);
     const quantization = parseQuantizationFromFileName(file);
+    const estimatedRuntimeBytes = sizeBytes
+      ? estimateLlamacppRuntimeBytes({
+        sizeBytes,
+        contextWindow: systemInfo?.expectedContextWindow,
+        preset: status.fit === "tight" ? "memory-safe" : "balanced"
+      })
+      : undefined;
     const fitScore = status.fit === "safe" ? 30 : status.fit === "tight" ? 15 : status.fit === "unknown" ? 8 : -20;
     const rankingScore = fitScore
       + (status.disabled ? -100 : 0)
@@ -166,6 +174,10 @@ export function shapeHuggingFaceGgufResults(files, systemInfo = {}) {
       file,
       quantization,
       sizeBytes,
+      estimatedRuntimeBytes,
+      memoryLabel: estimatedRuntimeBytes
+        ? `${(estimatedRuntimeBytes / (1024 ** 3)).toFixed(1)} GB runtime est.`
+        : "Runtime estimate unavailable",
       disabled: status.disabled,
       disabledReason: status.reason,
       fit: status.fit,

package/src/node/llamacpp-managed-runtime.js

@@ -0,0 +1,202 @@
+export function createLlamacppManagedRuntimeRegistry(deps = {}) {
+  const instances = new Map();
+  const inFlightStarts = new Map();
+  let nextPort = 39391;
+  const MIN_PORT = 1;
+  const MAX_PORT = 65535;
+
+  function resolveSpawnRuntime(overrides = {}) {
+    if (typeof overrides.spawnRuntime === "function") return overrides.spawnRuntime;
+    if (typeof deps.spawnRuntime === "function") return deps.spawnRuntime;
+    return async ({ host = "127.0.0.1", port } = {}) => ({
+      pid: undefined,
+      host,
+      port,
+      baseUrl: `http://${host}:${port}/v1`
+    });
+  }
+
+  function resolveWaitForHealthy(overrides = {}) {
+    if (typeof overrides.waitForHealthy === "function") return overrides.waitForHealthy;
+    if (typeof deps.waitForHealthy === "function") return deps.waitForHealthy;
+    return async (instance) => ({ ...instance, healthy: true });
+  }
+
+  function resolveListListeningPids(overrides = {}) {
+    if (typeof overrides.listListeningPids === "function") return overrides.listListeningPids;
+    if (typeof deps.listListeningPids === "function") return deps.listListeningPids;
+    return async () => [];
+  }
+
+  function resolveStopProcessByPid(overrides = {}) {
+    if (typeof overrides.stopProcessByPid === "function") return overrides.stopProcessByPid;
+    if (typeof deps.stopProcessByPid === "function") return deps.stopProcessByPid;
+    return async () => {};
+  }
+
+  function isTrackedInstanceReusable(instance) {
+    if (instance?.healthy !== true) return false;
+    const child = instance?.child;
+    if (child) {
+      return child.exitCode === null && child.killed !== true;
+    }
+    return true;
+  }
+
+  function isChildAlive(child) {
+    if (!child) return true;
+    return child.exitCode === null && child.killed !== true;
+  }
+
+  function normalizeRuntimePort(value, fallback = null) {
+    const parsed = Number(value);
+    if (!Number.isInteger(parsed) || parsed < MIN_PORT || parsed > MAX_PORT) return fallback;
+    return parsed;
+  }
+
+  function buildCompatibilityKey(variantKey, profileHash) {
+    return `${String(variantKey || "")}::${String(profileHash || "")}`;
+  }
+
+  function buildReservedPorts() {
+    const reserved = new Set();
+    for (const instance of instances.values()) {
+      if (!isChildAlive(instance?.child)) continue;
+      const port = normalizeRuntimePort(instance?.port);
+      if (port !== null) reserved.add(port);
+    }
+    for (const start of inFlightStarts.values()) {
+      const port = normalizeRuntimePort(start?.reservedPort);
+      if (port !== null) reserved.add(port);
+    }
+    return reserved;
+  }
+
+  function pruneDeadInstances() {
+    for (const [instanceId, instance] of instances.entries()) {
+      if (!isChildAlive(instance?.child)) {
+        instances.delete(instanceId);
+      }
+    }
+  }
+
+  function allocatePort(preferredPort) {
+    const reservedPorts = buildReservedPorts();
+    const preferred = normalizeRuntimePort(preferredPort);
+    if (preferred !== null && !reservedPorts.has(preferred)) {
+      if (preferred >= nextPort) nextPort = preferred + 1;
+      return preferred;
+    }
+
+    let port = Math.max(39391, nextPort);
+    if (port > MAX_PORT) {
+      port = 39391;
+    }
+    const startPort = port;
+    while (reservedPorts.has(port)) {
+      port += 1;
+      if (port > MAX_PORT) {
+        port = 39391;
+      }
+      if (port === startPort) {
+        throw new Error("No available managed runtime port.");
+      }
+    }
+
+    nextPort = port + 1;
+    return port;
+  }
+
+  async function ensureRuntimeForVariant({ variantKey, profileHash, launchArgs, preferredPort } = {}, runtimeDeps = {}) {
+    const spawnRuntime = resolveSpawnRuntime(runtimeDeps);
+    const waitForHealthy = resolveWaitForHealthy(runtimeDeps);
+    const compatibilityKey = buildCompatibilityKey(variantKey, profileHash);
+    pruneDeadInstances();
+
+    for (const instance of instances.values()) {
+      if (
+        instance.profileHash === profileHash
+        && instance.variantKey === variantKey
+        && isTrackedInstanceReusable(instance)
+      ) {
+        return instance;
+      }
+    }
+
+    const inFlight = inFlightStarts.get(compatibilityKey);
+    if (inFlight?.promise) {
+      return inFlight.promise;
+    }
+
+    const port = allocatePort(preferredPort);
+    const startPromise = (async () => {
+      const spawned = await spawnRuntime({ variantKey, profileHash, launchArgs, port });
+      const healthy = await waitForHealthy(spawned);
+      const assignedPort = normalizeRuntimePort(healthy?.port, port);
+      if (!isChildAlive(healthy?.child)) {
+        throw new Error("Managed runtime exited before becoming healthy.");
+      }
+      const instance = {
+        instanceId: `${variantKey}:${profileHash}:${assignedPort}`,
+        owner: "llm-router",
+        variantKey,
+        profileHash,
+        healthy: true,
+        ...healthy,
+        port: assignedPort
+      };
+      instances.set(instance.instanceId, instance);
+      return instance;
+    })().finally(() => {
+      inFlightStarts.delete(compatibilityKey);
+    });
+
+    inFlightStarts.set(compatibilityKey, { promise: startPromise, reservedPort: port });
+    return startPromise;
+  }
+
+  async function reconcile(runtimeDeps = {}) {
+    const listListeningPids = resolveListListeningPids(runtimeDeps);
+    const stopProcessByPid = resolveStopProcessByPid(runtimeDeps);
+    for (const [instanceId, instance] of instances.entries()) {
+      const probe = await listListeningPids(instance.port).catch(() => null);
+      const livePids = Array.isArray(probe)
+        ? probe
+        : (probe && typeof probe === "object" && Array.isArray(probe.pids) ? probe.pids : null);
+      const probeFailed = Boolean(probe && typeof probe === "object" && probe.ok === false);
+      if (probeFailed || !Array.isArray(livePids)) continue;
+      if (livePids.includes(instance.pid)) continue;
+      if (instance.owner === "llm-router") {
+        await stopProcessByPid(instance.pid).catch(() => {});
+      }
+      instances.delete(instanceId);
+    }
+  }
+
+  async function waitForInFlightStarts() {
+    while (inFlightStarts.size > 0) {
+      const pending = [...inFlightStarts.values()]
+        .map((entry) => entry?.promise)
+        .filter(Boolean)
+        .map((promise) => promise.catch(() => null));
+      if (pending.length === 0) return;
+      await Promise.all(pending);
+    }
+  }
+
+  return {
+    ensureRuntimeForVariant,
+    reconcile,
+    waitForInFlightStarts,
+    trackInstance: async (instance) => {
+      instances.set(instance.instanceId, { ...instance });
+    },
+    untrackInstance: async (instanceId) => {
+      instances.delete(instanceId);
+    },
+    clear: async () => {
+      instances.clear();
+    },
+    snapshot: () => [...instances.values()]
+  };
+}

package/src/node/llamacpp-runtime-profile.js

@@ -0,0 +1,133 @@
+function isPlainObject(value) {
+  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+
+function normalizeString(value) {
+  return typeof value === "string" ? value.trim() : "";
+}
+
+function toGiB(bytes) {
+  return Math.round((Number(bytes || 0) / (1024 ** 3)) * 10) / 10;
+}
+
+function normalizePositiveInteger(value, fallback) {
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
+  return Math.floor(parsed);
+}
+
+const LLAMACPP_PRESET_TUNING = Object.freeze({
+  balanced: Object.freeze({
+    canonicalPreset: "balanced",
+    batchSize: 64,
+    ubatchSize: 16,
+    gpuLayers: { darwin: 99, other: 0 },
+    penaltyRatio: 0.10,
+    noContBatching: false
+  }),
+  "long-context": Object.freeze({
+    canonicalPreset: "long-context",
+    batchSize: 32,
+    ubatchSize: 8,
+    gpuLayers: { darwin: 80, other: 0 },
+    penaltyRatio: 0.16,
+    noContBatching: false
+  }),
+  "low-memory": Object.freeze({
+    canonicalPreset: "low-memory",
+    batchSize: 32,
+    ubatchSize: 8,
+    gpuLayers: { darwin: 0, other: 0 },
+    penaltyRatio: 0.04,
+    noContBatching: true
+  }),
+  "fast-response": Object.freeze({
+    canonicalPreset: "fast-response",
+    batchSize: 16,
+    ubatchSize: 8,
+    gpuLayers: { darwin: 40, other: 0 },
+    penaltyRatio: 0.07,
+    noContBatching: false
+  }),
+  "cpu-safe": Object.freeze({
+    canonicalPreset: "cpu-safe",
+    batchSize: 32,
+    ubatchSize: 8,
+    gpuLayers: { darwin: 0, other: 0 },
+    penaltyRatio: 0.04,
+    noContBatching: true
+  })
+});
+
+function resolveCanonicalPreset(requestedPreset) {
+  const normalizedPreset = normalizeString(requestedPreset).toLowerCase();
+  if (normalizedPreset === "throughput") return LLAMACPP_PRESET_TUNING["fast-response"];
+  if (normalizedPreset === "memory-safe") return LLAMACPP_PRESET_TUNING["low-memory"];
+  return LLAMACPP_PRESET_TUNING[normalizedPreset] || LLAMACPP_PRESET_TUNING.balanced;
+}
+
+export function estimateLlamacppRuntimeBytes({
+  sizeBytes = 0,
+  contextWindow = 0,
+  preset = "balanced"
+} = {}) {
+  const base = Number(sizeBytes || 0);
+  const contextBytes = Number(contextWindow || 0) * 163840;
+  const tuning = resolveCanonicalPreset(preset);
+  const presetPenalty = Math.floor(base * tuning.penaltyRatio);
+  return base + contextBytes + presetPenalty;
+}
+
+export function deriveLlamacppLaunchProfile({
+  variant,
+  baseModel,
+  system
+} = {}) {
+  const requestedPreset = normalizeString(variant?.preset)
+    || normalizeString(variant?.runtimeProfile?.preset)
+    || "balanced";
+  const failureCategory = normalizeString(
+    variant?.runtimeProfile?.lastFailure?.category || variant?.runtimeStatus?.lastFailure?.category
+  );
+  const tuning = resolveCanonicalPreset(failureCategory === "metal-oom" ? "cpu-safe" : requestedPreset);
+  const preset = tuning.canonicalPreset;
+  const contextWindow = normalizePositiveInteger(variant?.contextWindow, 2048);
+  const overrides = isPlainObject(variant?.runtimeProfile?.overrides) ? variant.runtimeProfile.overrides : {};
+  const extraArgs = Array.isArray(variant?.runtimeProfile?.extraArgs)
+    ? variant.runtimeProfile.extraArgs.map((value) => normalizeString(value)).filter(Boolean)
+    : [];
+  const gpuLayers = Number.isFinite(Number(overrides.gpuLayers))
+    ? Math.floor(Number(overrides.gpuLayers))
+    : (system?.platform === "darwin" ? tuning.gpuLayers.darwin : tuning.gpuLayers.other);
+  const batchSize = Number.isFinite(Number(overrides.batchSize))
+    ? Math.floor(Number(overrides.batchSize))
+    : tuning.batchSize;
+  const ubatchSize = Number.isFinite(Number(overrides.ubatchSize))
+    ? Math.floor(Number(overrides.ubatchSize))
+    : tuning.ubatchSize;
+  const estimatedRuntimeBytes = estimateLlamacppRuntimeBytes({
+    sizeBytes: baseModel?.metadata?.sizeBytes,
+    contextWindow,
+    preset
+  });
+  const args = [
+    "-m", normalizeString(baseModel?.path),
+    "-a", normalizeString(variant?.id),
+    "-c", String(contextWindow),
+    "-np", "1",
+    "-b", String(batchSize),
+    "-ub", String(ubatchSize),
+    "--cache-ram", "0",
+    "--no-warmup"
+  ];
+
+  if (tuning.noContBatching) args.push("--no-cont-batching");
+  args.push("-ngl", String(gpuLayers), ...extraArgs);
+
+  return {
+    preset,
+    args: args.filter(Boolean),
+    estimatedRuntimeBytes,
+    memoryLabel: `${toGiB(estimatedRuntimeBytes)} GB`
+  };
+}