@khanglvm/llm-router 2.4.1 → 2.5.1

package/src/node/local-model-browser.js

@@ -0,0 +1,132 @@
+import path from "node:path";
+import { execFile } from "node:child_process";
+import { promises as fs } from "node:fs";
+
+const GGUF_PATTERN = /\.gguf$/i;
+
+function normalizeString(value) {
+  return typeof value === "string" ? value.trim() : "";
+}
+
+function formatScanEntry(filePath, stats = null) {
+  return {
+    filePath,
+    fileName: path.basename(filePath),
+    sizeBytes: Number.isFinite(Number(stats?.size)) ? Number(stats.size) : undefined
+  };
+}
+
+async function collectGgufFiles(targetPath, entries = []) {
+  const stats = await fs.stat(targetPath);
+  if (stats.isFile()) {
+    if (GGUF_PATTERN.test(targetPath)) entries.push(formatScanEntry(targetPath, stats));
+    return entries;
+  }
+
+  if (!stats.isDirectory()) return entries;
+
+  const children = await fs.readdir(targetPath, { withFileTypes: true });
+  for (const child of children) {
+    const childPath = path.join(targetPath, child.name);
+    if (child.isDirectory()) {
+      await collectGgufFiles(childPath, entries);
+      continue;
+    }
+    if (!child.isFile() || !GGUF_PATTERN.test(child.name)) continue;
+    const childStats = await fs.stat(childPath);
+    entries.push(formatScanEntry(childPath, childStats));
+  }
+  return entries;
+}
+
+function buildBrowseAppleScript(selection) {
+  if (selection === "directory") {
+    return [
+      "try",
+      "POSIX path of (choose folder with prompt \"Select a folder to scan for GGUF files\")",
+      "on error number -128",
+      "return \"\"",
+      "end try"
+    ];
+  }
+
+  if (selection === "runtime") {
+    return [
+      "try",
+      "POSIX path of (choose file with prompt \"Select a llama.cpp runtime binary (llama-server)\")",
+      "on error number -128",
+      "return \"\"",
+      "end try"
+    ];
+  }
+
+  return [
+    "try",
+    "POSIX path of (choose file with prompt \"Select a GGUF file\")",
+    "on error number -128",
+    "return \"\"",
+    "end try"
+  ];
+}
+
+export async function browseForLocalModelPath({
+  selection = "file"
+} = {}, {
+  platform = process.platform,
+  execFileImpl = execFile
+} = {}) {
+  if (platform !== "darwin") {
+    return {
+      canceled: true,
+      reason: "Native local-model browse is currently available on macOS only.",
+      selection
+    };
+  }
+
+  const scriptLines = buildBrowseAppleScript(selection);
+  const args = scriptLines.flatMap((line) => ["-e", line]);
+  const result = await runExecFile(execFileImpl, "osascript", args, { encoding: "utf8" });
+  const output = normalizeString(result?.stdout || "");
+  if (!output) {
+    return { canceled: true, selection };
+  }
+
+  return {
+    canceled: false,
+    selection,
+    path: output
+  };
+}
+
+export async function scanLocalModelPath(targetPath) {
+  const resolvedPath = normalizeString(targetPath);
+  if (!resolvedPath) return [];
+
+  const matches = await collectGgufFiles(resolvedPath);
+  return matches.sort((left, right) => left.fileName.localeCompare(right.fileName));
+}
+async function runExecFile(execFileImpl, command, args, options) {
+  if (execFileImpl === execFile) {
+    return new Promise((resolve, reject) => {
+      execFile(command, args, options, (error, stdout, stderr) => {
+        if (error) reject(error);
+        else resolve({ stdout, stderr });
+      });
+    });
+  }
+
+  if (typeof execFileImpl !== "function") {
+    throw new Error("execFile implementation is required.");
+  }
+
+  if (execFileImpl.length >= 4) {
+    return new Promise((resolve, reject) => {
+      execFileImpl(command, args, options, (error, stdout, stderr) => {
+        if (error) reject(error);
+        else resolve({ stdout, stderr });
+      });
+    });
+  }
+
+  return execFileImpl(command, args, options);
+}

package/src/node/local-model-capacity.js

@@ -0,0 +1,39 @@
+function normalizePositiveNumber(value) {
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed) || parsed <= 0) return 0;
+  return parsed;
+}
+
+function calculateEstimatedBytes(variant = {}) {
+  const sizeBytes = normalizePositiveNumber(variant.sizeBytes);
+  const contextWindow = normalizePositiveNumber(variant.contextWindow);
+  const contextBytes = contextWindow * 163840;
+  const preloadPenalty = variant.preload === true ? Math.floor(sizeBytes * 0.15) : 0;
+  return sizeBytes + contextBytes + preloadPenalty;
+}
+
+export function classifyVariantCapacity(variant, system = {}) {
+  const estimatedBytes = calculateEstimatedBytes(variant);
+  const totalMemoryBytes = normalizePositiveNumber(system.totalMemoryBytes);
+  const safeBudget = Math.floor(totalMemoryBytes * 0.72);
+  const tightBudget = Math.floor(totalMemoryBytes * 0.82);
+
+  if (system.platform === "darwin" && system.unifiedMemory === true && estimatedBytes > tightBudget) {
+    return { fit: "over-budget", estimatedBytes };
+  }
+  if (system.platform === "darwin" && system.unifiedMemory === true && estimatedBytes > safeBudget) {
+    return { fit: "tight", estimatedBytes };
+  }
+  return { fit: "safe", estimatedBytes };
+}
+
+export function canActivateVariant({ candidate, activeVariants, totalMemoryBytes }) {
+  const safeBudget = Math.floor(normalizePositiveNumber(totalMemoryBytes) * 0.72);
+  const activeBytes = (Array.isArray(activeVariants) ? activeVariants : [])
+    .reduce((sum, variant) => sum + normalizePositiveNumber(variant?.estimatedBytes), 0);
+  const nextBytes = activeBytes + normalizePositiveNumber(candidate?.estimatedBytes);
+
+  return nextBytes <= safeBudget
+    ? { allowed: true, reason: "" }
+    : { allowed: false, reason: "Enabling this variant would exceed the local capacity budget." };
+}

package/src/node/local-models-service.js

@@ -0,0 +1,238 @@
+import os from "node:os";
+import path from "node:path";
+import { promises as fs } from "node:fs";
+import { normalizeLocalModelsMetadata } from "../runtime/local-models.js";
+import { canActivateVariant, classifyVariantCapacity } from "./local-model-capacity.js";
+
+function isPlainObject(value) {
+  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+
+function cloneConfig(config) {
+  return isPlainObject(config) ? structuredClone(config) : {};
+}
+
+function normalizeString(value) {
+  return typeof value === "string" ? value.trim() : "";
+}
+
+function normalizeManagedMetadata(metadata) {
+  return isPlainObject(metadata) ? metadata : {};
+}
+
+function ensureLocalModelsState(config) {
+  const next = cloneConfig(config);
+  next.metadata = isPlainObject(next.metadata) ? next.metadata : {};
+  next.metadata.localModels = normalizeLocalModelsMetadata(next.metadata.localModels);
+  return next;
+}
+
+async function defaultPathExists(filePath) {
+  try {
+    await fs.access(filePath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export function getManagedLocalModelsDir({ homeDir = os.homedir() } = {}) {
+  return path.join(homeDir, ".llm-router", "local-models");
+}
+
+export async function registerAttachedLlamacppModel(config, {
+  id,
+  displayName,
+  filePath,
+  metadata = {}
+} = {}) {
+  const baseModelId = normalizeString(id);
+  const modelPath = normalizeString(filePath);
+  const label = normalizeString(displayName);
+
+  if (!baseModelId) throw new Error("Attached local model id is required.");
+  if (!modelPath) throw new Error("Attached local model path is required.");
+
+  const next = ensureLocalModelsState(config);
+  next.metadata.localModels.library[baseModelId] = {
+    id: baseModelId,
+    source: "llamacpp-attached",
+    displayName: label || baseModelId,
+    path: modelPath,
+    availability: "available",
+    metadata: isPlainObject(metadata) ? metadata : {},
+    managed: false
+  };
+
+  return next;
+}
+
+export async function registerManagedLlamacppModel(config, {
+  id,
+  displayName,
+  filePath,
+  repo = "",
+  file = "",
+  sizeBytes = undefined,
+  metadata = {}
+} = {}) {
+  const baseModelId = normalizeString(id);
+  const modelPath = normalizeString(filePath);
+  const label = normalizeString(displayName);
+
+  if (!baseModelId) throw new Error("Managed local model id is required.");
+  if (!modelPath) throw new Error("Managed local model path is required.");
+
+  const next = ensureLocalModelsState(config);
+  next.metadata.localModels.library[baseModelId] = {
+    id: baseModelId,
+    source: "llamacpp-managed",
+    displayName: label || baseModelId,
+    path: modelPath,
+    availability: "available",
+    metadata: {
+      ...normalizeManagedMetadata(metadata),
+      repo: normalizeString(repo),
+      file: normalizeString(file),
+      ...(Number.isFinite(Number(sizeBytes)) ? { sizeBytes: Number(sizeBytes) } : {})
+    },
+    managed: true
+  };
+
+  return next;
+}
+
+export async function reconcileLocalModelPaths(config, {
+  pathExists = defaultPathExists
+} = {}) {
+  const next = ensureLocalModelsState(config);
+  const { library, variants } = next.metadata.localModels;
+
+  for (const baseModel of Object.values(library)) {
+    const baseModelPath = normalizeString(baseModel?.path);
+    if (!baseModelPath) {
+      baseModel.availability = "stale";
+      continue;
+    }
+
+    const exists = await pathExists(baseModelPath);
+    baseModel.availability = exists ? "available" : "stale";
+  }
+
+  for (const variant of Object.values(variants)) {
+    const baseModel = library[variant?.baseModelId];
+    variant.availability = baseModel?.availability || "stale";
+  }
+
+  return next;
+}
+
+export async function removeLocalBaseModel(config, baseModelId) {
+  const targetId = normalizeString(baseModelId);
+  const next = ensureLocalModelsState(config);
+
+  if (!targetId) return next;
+
+  delete next.metadata.localModels.library[targetId];
+  for (const [variantKey, variant] of Object.entries(next.metadata.localModels.variants)) {
+    if (variant?.baseModelId === targetId) {
+      delete next.metadata.localModels.variants[variantKey];
+    }
+  }
+
+  return next;
+}
+
+export async function updateLocalBaseModelPath(config, baseModelId, filePath) {
+  const targetId = normalizeString(baseModelId);
+  const nextPath = normalizeString(filePath);
+  const next = ensureLocalModelsState(config);
+
+  if (!targetId) throw new Error("Base model id is required.");
+  if (!nextPath) throw new Error("Updated local model path is required.");
+
+  const baseModel = next.metadata.localModels.library[targetId];
+  if (!baseModel) {
+    throw new Error(`Base model '${targetId}' was not found.`);
+  }
+
+  baseModel.path = nextPath;
+  baseModel.availability = "available";
+
+  for (const variant of Object.values(next.metadata.localModels.variants)) {
+    if (variant?.baseModelId === targetId) {
+      variant.availability = "available";
+    }
+  }
+
+  return next;
+}
+
+export async function saveLocalModelVariant(config, draft, {
+  system = {}
+} = {}) {
+  const next = ensureLocalModelsState(config);
+  const normalizedDraft = isPlainObject(draft) ? draft : {};
+  const key = normalizeString(normalizedDraft.key || normalizedDraft.id);
+  const baseModelId = normalizeString(normalizedDraft.baseModelId);
+  const modelId = normalizeString(normalizedDraft.id);
+  const name = normalizeString(normalizedDraft.name);
+  const runtime = normalizeString(normalizedDraft.runtime);
+
+  if (!key || !baseModelId || !modelId || !name || !runtime) {
+    throw new Error("key, baseModelId, id, name, and runtime are required.");
+  }
+
+  const baseModel = next.metadata.localModels.library[baseModelId];
+  if (!baseModel) {
+    throw new Error(`Base model '${baseModelId}' was not found.`);
+  }
+
+  const sizeBytes = Number(baseModel?.metadata?.sizeBytes || 0);
+  const capacity = classifyVariantCapacity({
+    sizeBytes,
+    contextWindow: normalizedDraft.contextWindow,
+    preload: normalizedDraft.preload === true
+  }, system);
+
+  const shouldActivate = normalizedDraft.enabled === true || normalizedDraft.preload === true;
+  if (shouldActivate && system?.platform === "darwin" && system?.unifiedMemory === true) {
+    const activeVariants = Object.values(next.metadata.localModels.variants)
+      .filter((variant) => variant?.key !== key)
+      .filter((variant) => variant?.enabled === true || variant?.preload === true)
+      .map((variant) => ({
+        estimatedBytes: Number(variant?.estimatedBytes || 0),
+        preload: variant?.preload === true
+      }));
+    const decision = canActivateVariant({
+      candidate: {
+        estimatedBytes: capacity.estimatedBytes,
+        preload: normalizedDraft.preload === true
+      },
+      activeVariants,
+      totalMemoryBytes: system.totalMemoryBytes
+    });
+    if (!decision.allowed) {
+      throw new Error(decision.reason);
+    }
+  }
+
+  next.metadata.localModels.variants[key] = {
+    ...(isPlainObject(next.metadata.localModels.variants[key]) ? next.metadata.localModels.variants[key] : {}),
+    key,
+    baseModelId,
+    id: modelId,
+    name,
+    runtime,
+    preset: normalizeString(normalizedDraft.preset),
+    enabled: normalizedDraft.enabled === true,
+    preload: normalizedDraft.preload === true,
+    contextWindow: Number.isFinite(Number(normalizedDraft.contextWindow)) ? Number(normalizedDraft.contextWindow) : undefined,
+    capabilities: isPlainObject(normalizedDraft.capabilities) ? normalizedDraft.capabilities : undefined,
+    availability: normalizeString(baseModel?.availability) || "available",
+    capacityState: capacity.fit,
+    estimatedBytes: capacity.estimatedBytes
+  };
+
+  return next;
+}

package/src/node/start-command.js

@@ -14,6 +14,7 @@ import { startLocalRouteServer } from "./local-server.js";
 import { startRouterSupervisor } from "./router-supervisor.js";
 import { reclaimPort, stopStartupManagedListener } from "./port-reclaim.js";
 import { installStartup, startupStatus } from "./startup-manager.js";
+import { ensureConfiguredLlamacppRuntimeStarted, stopManagedLlamacppRuntime } from "./llamacpp-runtime.js";
 import { configHasProvider, sanitizeConfigForDisplay } from "../runtime/config.js";
 
 function summarizeConfig(config, configPath) {
@@ -888,6 +889,12 @@ async function runRouterSupervisorCommand(options = {}) {
   const startRouterSupervisorFn = typeof options.startRouterSupervisor === "function"
     ? options.startRouterSupervisor
     : (startOptions) => startRouterSupervisor(startOptions, options);
+  const ensureConfiguredLlamacppRuntimeStartedFn = typeof options.ensureConfiguredLlamacppRuntimeStarted === "function"
+    ? options.ensureConfiguredLlamacppRuntimeStarted
+    : ensureConfiguredLlamacppRuntimeStarted;
+  const stopManagedLlamacppRuntimeFn = typeof options.stopManagedLlamacppRuntime === "function"
+    ? options.stopManagedLlamacppRuntime
+    : stopManagedLlamacppRuntime;
 
   if (!(await configFileExists(configPath))) {
     return {
@@ -956,6 +963,8 @@ async function runRouterSupervisorCommand(options = {}) {
     };
   }
 
+  await ensureConfiguredLlamacppRuntimeStartedFn(config, { line, error });
+
   const requestedStartArgs = {
     configPath,
     host,
@@ -1103,6 +1112,7 @@ async function runRouterSupervisorCommand(options = {}) {
     if (shutdownPromise) return shutdownPromise;
     shuttingDown = true;
     shutdownPromise = (async () => {
+      await stopManagedLlamacppRuntimeFn({ line, error });
       await new Promise((resolve) => server.close(() => resolve()));
       await clearRuntimeStateFn({ pid: process.pid });
     })();
@@ -1127,6 +1137,8 @@ async function runRouterSupervisorCommand(options = {}) {
   await donePromise;
   if (shutdownPromise) {
     await shutdownPromise;
+  } else {
+    await stopManagedLlamacppRuntimeFn({ line, error });
   }
 
   process.removeListener("SIGINT", handleSigInt);