@khanglvm/llm-router 2.6.0 → 2.6.1

package/src/node/web-console-server.js

@@ -65,7 +65,6 @@ import { detectOllamaInstallation, installOllama, startOllamaServer, stopOllamaS
 import { browseForLocalModelPath, scanLocalModelPath } from "./local-model-browser.js";
 import {
   detectLlamacppCandidates,
-  getManagedLlamacppRuntimeSnapshot,
   startConfiguredLlamacppRuntime,
   stopManagedLlamacppRuntime,
   validateLlamacppCommand
@@ -84,6 +83,7 @@ import {
   downloadManagedHuggingFaceGguf,
   searchHuggingFaceGgufCandidates
 } from "./huggingface-gguf.js";
+import { createQuotaProbeRunner } from "./quota-probe-runner.js";
 import {
   CONFIG_VERSION,
   DEFAULT_MODEL_ALIAS_ID,
@@ -869,10 +869,6 @@ function routeSnapshotDocument(configState) {
   return configState.parseError ? null : (configState.normalizedConfig || buildDefaultConfigObject());
 }
 
-function isObjectRecord(value) {
-  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
-}
-
 function readConfiguredLlamacppRuntime(config = {}) {
   const runtime = config?.metadata?.localModels?.runtime?.llamacpp;
   if (!runtime || typeof runtime !== "object") {
@@ -900,46 +896,17 @@ function readConfiguredLlamacppRuntime(config = {}) {
   };
 }
 
-function buildLlamacppRuntimePayload(runtime = {}, validation = {}, candidates = [], registrySnapshot = []) {
+function buildLlamacppRuntimePayload(runtime = {}, validation = {}, candidates = []) {
   const selectedCommand = String(runtime.selectedCommand || runtime.manualCommand || runtime.command || "").trim();
-  const runtimeInstances = Array.isArray(registrySnapshot) ? registrySnapshot : [];
-  const healthyInstances = runtimeInstances.filter((entry) => entry?.healthy === true);
   return {
     ...runtime,
     selectedCommand,
     selectedDirectory: selectedCommand ? path.dirname(selectedCommand) : "",
-    managedInstanceCount: runtimeInstances.length,
-    healthyInstanceCount: healthyInstances.length,
-    staleRuntimeCount: runtimeInstances.filter((entry) => entry?.healthy !== true).length,
-    instances: runtimeInstances,
     ...(validation && typeof validation === "object" ? validation : {}),
     candidates
   };
 }
 
-function buildRouteSnapshotDocument(configState, registrySnapshot = []) {
-  const document = routeSnapshotDocument(configState);
-  if (!isObjectRecord(document)) return document;
-
-  const runtimeInstances = Array.isArray(registrySnapshot) ? registrySnapshot : [];
-  if (!isObjectRecord(document.metadata?.localModels) && runtimeInstances.length === 0) {
-    return document;
-  }
-
-  const nextDocument = JSON.parse(JSON.stringify(document));
-  if (!isObjectRecord(nextDocument.metadata)) nextDocument.metadata = {};
-  if (!isObjectRecord(nextDocument.metadata.localModels)) nextDocument.metadata.localModels = {};
-  if (!isObjectRecord(nextDocument.metadata.localModels.runtime)) nextDocument.metadata.localModels.runtime = {};
-
-  nextDocument.metadata.localModels.runtime.llamacpp = buildLlamacppRuntimePayload(
-    readConfiguredLlamacppRuntime(nextDocument),
-    {},
-    [],
-    runtimeInstances
-  );
-  return nextDocument;
-}
-
 function updateLlamacppRuntimeConfig(config = {}, runtimePatch = {}) {
   const nextConfig = JSON.parse(JSON.stringify(config || {}));
   if (!nextConfig.metadata || typeof nextConfig.metadata !== "object" || Array.isArray(nextConfig.metadata)) {
@@ -1045,9 +1012,6 @@ export async function startWebConsoleServer(options = {}, deps = {}) {
   const stopManagedLlamacppRuntimeFn = typeof deps.stopManagedLlamacppRuntime === "function"
     ? deps.stopManagedLlamacppRuntime
     : (callbacks = {}) => stopManagedLlamacppRuntime(callbacks);
-  const getManagedLlamacppRuntimeSnapshotFn = typeof deps.getManagedLlamacppRuntimeSnapshot === "function"
-    ? deps.getManagedLlamacppRuntimeSnapshot
-    : getManagedLlamacppRuntimeSnapshot;
   const validateLlamacppCommandFn = typeof deps.validateLlamacppCommand === "function"
     ? deps.validateLlamacppCommand
     : validateLlamacppCommand;
@@ -1072,6 +1036,8 @@ export async function startWebConsoleServer(options = {}, deps = {}) {
     requireAuth: routerRequireAuth === true
   };
 
+  const quotaProbeRunner = createQuotaProbeRunner({ fetchImpl: globalThis.fetch });
+
   async function readWebSearchState(config = null) {
     if (!config || typeof config !== "object") return null;
     const runtimeFlags = resolveRuntimeFlags({ runtime: "node" }, runtimeEnv);
@@ -2497,24 +2463,6 @@ export async function startWebConsoleServer(options = {}, deps = {}) {
 
   let routerRestartPromise = null;
 
-  async function readManagedLlamacppRuntimeSnapshot() {
-    try {
-      const snapshot = await Promise.resolve(getManagedLlamacppRuntimeSnapshotFn());
-      return Array.isArray(snapshot) ? snapshot : [];
-    } catch {
-      return [];
-    }
-  }
-
-  async function buildManagedLlamacppRuntimePayload(runtime = {}, validation = {}, candidates = []) {
-    return buildLlamacppRuntimePayload(
-      runtime,
-      validation,
-      candidates,
-      await readManagedLlamacppRuntimeSnapshot()
-    );
-  }
-
   async function restartManagedRouterWithSettings(settings, {
     reason = "Restarting managed router.",
     configStateOverride = null
@@ -2570,7 +2518,6 @@ export async function startWebConsoleServer(options = {}, deps = {}) {
     const claudeCodeGlobal = await readClaudeCodeGlobalRoutingState(configLocalServer, configState.normalizedConfig);
     const factoryDroidGlobal = await readFactoryDroidGlobalRoutingState(configLocalServer, configState.normalizedConfig);
     const webSearch = await readWebSearchState(configState.normalizedConfig).catch(() => null);
-    const managedLlamacppRegistrySnapshot = await readManagedLlamacppRuntimeSnapshot();
     const ollamaConfig = configState.normalizedConfig?.ollama;
     const ollamaBaseUrl = ollamaConfig?.baseUrl || "http://localhost:11434";
     const ollamaInstallation = detectOllamaInstallation();
@@ -2593,7 +2540,7 @@ export async function startWebConsoleServer(options = {}, deps = {}) {
       },
       config: {
         ...configState.summary,
-        document: buildRouteSnapshotDocument(configState, managedLlamacppRegistrySnapshot),
+        document: routeSnapshotDocument(configState),
         localServer: configLocalServer
       },
       router: routerSnapshot,
@@ -3089,6 +3036,102 @@ export async function startWebConsoleServer(options = {}, deps = {}) {
         return;
       }
 
+      // ── Quota Probe routes ──────────────────────────────────────────
+      const quotaProbeTestMatch = requestUrl.pathname.match(/^\/api\/providers\/([^/]+)\/quota-probe\/test$/);
+      if (method === "POST" && quotaProbeTestMatch) {
+        const providerId = decodeURIComponent(quotaProbeTestMatch[1]);
+        const body = await readJsonBody(req);
+        const configState = await readConfigState(configPath);
+        const provider = (configState.normalizedConfig?.providers || []).find((entry) => entry.id === providerId);
+        if (!provider) {
+          sendJson(res, 404, { error: "Provider not found." });
+          return;
+        }
+        const shortcodeCtx = {
+          providerApiKey: resolveProviderApiKey(provider, process.env) || "",
+          providerBaseUrl: provider.baseUrl || "",
+          providerId: provider.id
+        };
+        const probeConfig = {
+          ...(provider.quotaProbe || {}),
+          enabled: true,
+          capKind: body.capKind || provider.quotaProbe?.capKind || "dollars",
+          mode: body.mode || provider.quotaProbe?.mode || "http",
+          http: body.http || provider.quotaProbe?.http,
+          custom: body.custom || provider.quotaProbe?.custom
+        };
+        const tempRunner = createQuotaProbeRunner({ fetchImpl: globalThis.fetch });
+        const now = Date.now();
+        const startMs = now;
+        try {
+          const snapshot = await tempRunner.executeProbe({ providerId, probeConfig, shortcodeCtx, env: process.env, now });
+          const latencyMs = Date.now() - startMs;
+          sendJson(res, 200, { snapshot, raw: snapshot.raw, latencyMs, error: snapshot.error });
+        } finally {
+          tempRunner.dispose();
+        }
+        return;
+      }
+
+      const quotaProbeRefreshMatch = requestUrl.pathname.match(/^\/api\/providers\/([^/]+)\/quota-probe\/refresh$/);
+      if (method === "POST" && quotaProbeRefreshMatch) {
+        const providerId = decodeURIComponent(quotaProbeRefreshMatch[1]);
+        const configState = await readConfigState(configPath);
+        const provider = (configState.normalizedConfig?.providers || []).find((entry) => entry.id === providerId);
+        if (!provider) {
+          sendJson(res, 404, { error: "Provider not found." });
+          return;
+        }
+        if (!provider.quotaProbe?.enabled) {
+          sendJson(res, 400, { error: "Quota probe not enabled for this provider." });
+          return;
+        }
+        const shortcodeCtx = {
+          providerApiKey: resolveProviderApiKey(provider, process.env) || "",
+          providerBaseUrl: provider.baseUrl || "",
+          providerId: provider.id
+        };
+        const snapshot = await quotaProbeRunner.enqueueRefresh({
+          providerId,
+          probeConfig: provider.quotaProbe,
+          shortcodeCtx,
+          env: process.env,
+          bypassCircuit: true
+        });
+        sendJson(res, 200, { snapshot });
+        return;
+      }
+
+      const quotaProbeSnapshotMatch = requestUrl.pathname.match(/^\/api\/providers\/([^/]+)\/quota-probe\/snapshot$/);
+      if (method === "GET" && quotaProbeSnapshotMatch) {
+        const providerId = decodeURIComponent(quotaProbeSnapshotMatch[1]);
+        sendJson(res, 200, { snapshot: quotaProbeRunner.getSnapshot(providerId) });
+        return;
+      }
+
+      const quotaProbeSaveMatch = requestUrl.pathname.match(/^\/api\/providers\/([^/]+)\/quota-probe\/save$/);
+      if (method === "POST" && quotaProbeSaveMatch) {
+        const providerId = decodeURIComponent(quotaProbeSaveMatch[1]);
+        const body = await readJsonBody(req);
+        const configState = await readConfigState(configPath);
+        if (configState.parseError) {
+          sendJson(res, 400, { error: `Config parse error: ${configState.parseError}` });
+          return;
+        }
+        const rawConfig = configState.rawConfig || {};
+        const providerList = Array.isArray(rawConfig.providers) ? rawConfig.providers : [];
+        const providerIndex = providerList.findIndex((entry) => entry?.id === providerId);
+        if (providerIndex === -1) {
+          sendJson(res, 404, { error: "Provider not found." });
+          return;
+        }
+        providerList[providerIndex] = { ...providerList[providerIndex], quotaProbe: body.quotaProbe || null };
+        rawConfig.providers = providerList;
+        const { snapshot } = await writeAndBroadcastConfig(rawConfig, { source: "quota-probe-save" });
+        sendJson(res, 200, { ok: true, snapshot });
+        return;
+      }
+
       if (method === "POST" && requestUrl.pathname === "/api/config/test-provider") {
         const body = await readJsonBody(req);
         const endpoints = Array.isArray(body?.endpoints) ? body.endpoints.map((entry) => String(entry || "").trim()).filter(Boolean) : [];
@@ -3464,7 +3507,7 @@ export async function startWebConsoleServer(options = {}, deps = {}) {
         });
 
         sendJson(res, 200, {
-          runtime: await buildManagedLlamacppRuntimePayload(configuredRuntime, {}, hydratedCandidates)
+          runtime: buildLlamacppRuntimePayload(configuredRuntime, {}, hydratedCandidates)
         });
         return;
       }
@@ -3491,7 +3534,7 @@ export async function startWebConsoleServer(options = {}, deps = {}) {
         if (!validation?.ok) {
           sendJson(res, 400, {
             error: validation?.errorMessage || `Failed validating llama.cpp runtime '${command}'.`,
-            runtime: await buildManagedLlamacppRuntimePayload(readConfiguredLlamacppRuntime(configState.rawConfig || {}), validation)
+            runtime: buildLlamacppRuntimePayload(readConfiguredLlamacppRuntime(configState.rawConfig || {}), validation)
           });
           return;
         }
@@ -3507,7 +3550,7 @@ export async function startWebConsoleServer(options = {}, deps = {}) {
         const configuredRuntime = readConfiguredLlamacppRuntime(savedConfig || {});
         sendJson(res, 200, {
           ok: true,
-          runtime: await buildManagedLlamacppRuntimePayload(configuredRuntime, {
+          runtime: buildLlamacppRuntimePayload(configuredRuntime, {
             ...validation,
             status: configuredRuntime.status || "stopped"
           })
@@ -3533,7 +3576,7 @@ export async function startWebConsoleServer(options = {}, deps = {}) {
         });
         sendJson(res, 200, {
           ok: true,
-          runtime: await buildManagedLlamacppRuntimePayload(readConfiguredLlamacppRuntime(savedConfig || {}))
+          runtime: buildLlamacppRuntimePayload(readConfiguredLlamacppRuntime(savedConfig || {}))
         });
         return;
       }
@@ -3579,7 +3622,7 @@ export async function startWebConsoleServer(options = {}, deps = {}) {
         });
         sendJson(res, 200, {
           ok: true,
-          runtime: await buildManagedLlamacppRuntimePayload(readConfiguredLlamacppRuntime(savedConfig || {}), {
+          runtime: buildLlamacppRuntimePayload(readConfiguredLlamacppRuntime(savedConfig || {}), {
             ...(validation && typeof validation === "object" ? validation : {}),
             status: "running"
           })
@@ -3615,7 +3658,7 @@ export async function startWebConsoleServer(options = {}, deps = {}) {
         });
         sendJson(res, 200, {
           ok: true,
-          runtime: await buildManagedLlamacppRuntimePayload(readConfiguredLlamacppRuntime(savedConfig || {}), {
+          runtime: buildLlamacppRuntimePayload(readConfiguredLlamacppRuntime(savedConfig || {}), {
             status: "stopped"
           })
         });