@khanglvm/llm-router 2.6.0 → 2.6.1

package/src/node/local-server.js

@@ -13,10 +13,6 @@ import { readActivityLogSettings } from "../shared/local-router-defaults.js";
 import { appendActivityLogEntry, resolveActivityLogPath } from "./activity-log.js";
 import { appendLargeRequestLogEntry, resolveLargeRequestLogPath } from "./large-request-log.js";
 import { isLargeRequestLoggingEnabled } from "../runtime/handler/large-request-log.js";
-import {
-  startConfiguredLlamacppRuntime,
-  stopManagedLlamacppRuntime
-} from "./llamacpp-runtime.js";
 
 const DEFAULT_CONFIG_RELOAD_DEBOUNCE_MS = 300;
 const MAX_CONFIG_RELOAD_DEBOUNCE_MS = 5000;
@@ -38,10 +34,6 @@ function formatError(error) {
   return error instanceof Error ? error.message : String(error);
 }
 
-function normalizeString(value) {
-  return typeof value === "string" ? value.trim() : "";
-}
-
 function createLiveConfigStore({
   configPath,
   watchConfig = true,
@@ -245,39 +237,6 @@ async function writeFetchResponseToNode(res, response) {
   readable.pipe(res);
 }
 
-function buildVariantLlamacppRuntimeConfig(config, variantKey) {
-  const normalizedVariantKey = normalizeString(variantKey);
-  const runtime = config?.metadata?.localModels?.runtime?.llamacpp;
-  const variants = config?.metadata?.localModels?.variants;
-  const library = config?.metadata?.localModels?.library;
-  const variant = variants?.[normalizedVariantKey];
-  if (!runtime || !variant || variant.runtime !== "llamacpp") return null;
-
-  const baseModelId = normalizeString(variant?.baseModelId);
-  const baseModel = library?.[baseModelId];
-  if (!baseModel) return null;
-
-  return {
-    metadata: {
-      localModels: {
-        runtime: {
-          llamacpp: { ...runtime }
-        },
-        library: {
-          [baseModelId]: { ...baseModel }
-        },
-        variants: {
-          [normalizedVariantKey]: {
-            ...variant,
-            enabled: true,
-            preload: true
-          }
-        }
-      }
-    }
-  };
-}
-
 export async function startLocalRouteServer({
   port = FIXED_LOCAL_ROUTER_PORT,
   host = FIXED_LOCAL_ROUTER_HOST,
@@ -289,10 +248,7 @@ export async function startLocalRouteServer({
   validateConfig,
   onConfigReload,
   onConfigReloadError,
-  requireAuth = false,
-  createFetchHandlerImpl = createFetchHandler,
-  startConfiguredLlamacppRuntimeImpl = startConfiguredLlamacppRuntime,
-  stopManagedLlamacppRuntimeImpl = stopManagedLlamacppRuntime
+  requireAuth = false
 } = {}) {
   const reloadDebounceMs = resolveReloadDebounceMs(configReloadDebounceMs);
   const resolvedActivityLogPath = resolveActivityLogPath(configPath, activityLogPath);
@@ -314,22 +270,9 @@ export async function startLocalRouteServer({
   const initialConfig = await configStore.getConfig();
   activityLogEnabled = readActivityLogSettings(initialConfig).enabled;
 
-  const fetchHandler = createFetchHandlerImpl({
+  const fetchHandler = createFetchHandler({
     ignoreAuth: !requireAuth,
-    runtime: "node",
     getConfig: () => configStore.getConfig(),
-    resolveLocalRuntimeBaseUrl: async ({ candidate }) => {
-      const variantKey = candidate?.model?.metadata?.localVariantKey;
-      const config = await configStore.getConfig();
-      const targetedConfig = buildVariantLlamacppRuntimeConfig(config, variantKey);
-      if (!targetedConfig) return "";
-
-      const started = await startConfiguredLlamacppRuntimeImpl(targetedConfig);
-      if (!started?.ok) {
-        throw new Error(started?.errorMessage || `Failed starting local runtime for ${normalizeString(variantKey) || "unknown variant"}.`);
-      }
-      return normalizeString(started?.runtime?.baseUrl);
-    },
     defaultStateStoreBackend: "file",
     onActivityLog: (entry) => {
       if (!activityLogEnabled) return;
@@ -412,7 +355,6 @@ export async function startLocalRouteServer({
   server.close = (callback) => {
     shuttingDown = true;
     Promise.resolve()
-      .then(() => stopManagedLlamacppRuntimeImpl().catch(() => {}))
       .then(() => configStore.close())
       .then(() => (typeof fetchHandler.close === "function" ? fetchHandler.close() : undefined))
       .finally(() => {

package/src/node/provider-probe.js

@@ -310,6 +310,15 @@ function isTransientModelRuntimeError(result, message) {
   return patterns.some((pattern) => pattern.test(text));
 }
 
+function isOutputLimitReachedMessage(message) {
+  const text = String(message || "").toLowerCase();
+  if (!text) return false;
+  return (
+    text.includes("max_tokens") &&
+    (text.includes("output limit") || text.includes("token limit") || text.includes("finish"))
+  );
+}
+
 function isRateLimitResult(result, message) {
   const status = Number(result?.status || 0);
   if (status === 429) return true;
@@ -377,6 +386,15 @@ function classifyModelProbeResult(format, result) {
     };
   }
 
+  if (isOutputLimitReachedMessage(message)) {
+    return {
+      supported: true,
+      confirmed: true,
+      outcome: "output-limit",
+      message: message || "Request reached model but the probe token budget was too small."
+    };
+  }
+
   if (isUnsupportedModelMessage(message)) {
     return {
       supported: false,

package/src/node/quota-probe-mapping.js

@@ -0,0 +1,215 @@
+/**
+ * Pure helper functions for resolving values from arbitrary JSON responses
+ * and coercing them to expected types. No IO, no side effects.
+ */
+
+/**
+ * Resolves a dot-path from a JSON object.
+ * Supports `$.foo.bar` syntax (leading `$` or `$.` is stripped).
+ * Supports array indices: `$.data[0].amount`.
+ * Returns undefined for missing paths or null/undefined intermediates.
+ * @param {any} obj
+ * @param {string} pathStr
+ * @returns {any}
+ */
+export function resolvePath(obj, pathStr) {
+  if (obj == null || typeof pathStr !== "string") return undefined;
+
+  // Strip leading $ or $.
+  let cleaned = pathStr;
+  if (cleaned.startsWith("$.")) cleaned = cleaned.slice(2);
+  else if (cleaned.startsWith("$")) cleaned = cleaned.slice(1);
+
+  if (!cleaned) return obj;
+
+  // Tokenize: split on dots, then expand array indices.
+  // "data[0].amount" → ["data", "0", "amount"]
+  const segments = [];
+  for (const part of cleaned.split(".")) {
+    if (!part) continue;
+    // Handle array indices like "data[0]" or just "[0]"
+    const bracketRe = /([^\[]*)\[(\d+)\]/g;
+    let match;
+    let lastIndex = 0;
+    let hasMatch = false;
+    while ((match = bracketRe.exec(part)) !== null) {
+      hasMatch = true;
+      if (match[1]) segments.push(match[1]);
+      segments.push(match[2]);
+      lastIndex = bracketRe.lastIndex;
+    }
+    if (!hasMatch) {
+      segments.push(part);
+    } else if (lastIndex < part.length) {
+      segments.push(part.slice(lastIndex));
+    }
+  }
+
+  let current = obj;
+  for (const seg of segments) {
+    if (current == null) return undefined;
+    if (Array.isArray(current)) {
+      const idx = Number(seg);
+      if (!Number.isInteger(idx) || idx < 0) return undefined;
+      current = current[idx];
+    } else if (typeof current === "object") {
+      current = current[seg];
+    } else {
+      return undefined;
+    }
+  }
+  return current;
+}
+
+const FALSE_SET = new Set([0, "0", "false", "no", null, undefined, false, ""]);
+
+const DURATION_RE = /^PT?(?:(\d+(?:\.\d+)?)H)?(?:(\d+(?:\.\d+)?)M)?(?:(\d+(?:\.\d+)?)S)?$/i;
+const SHORT_DURATION_RE = /^(\d+(?:\.\d+)?)\s*(h|m|s)$/i;
+
+/**
+ * @param {string} str
+ * @returns {number|undefined} duration in milliseconds
+ */
+function parseDuration(str) {
+  // Try short form: "2h", "30m", "45s"
+  let m = SHORT_DURATION_RE.exec(str);
+  if (m) {
+    const val = parseFloat(m[1]);
+    const unit = m[2].toLowerCase();
+    if (unit === "h") return val * 3600_000;
+    if (unit === "m") return val * 60_000;
+    if (unit === "s") return val * 1000;
+  }
+  // Try ISO 8601 duration: "PT2H", "PT30M", "PT2H30M"
+  m = DURATION_RE.exec(str);
+  if (m && (m[1] || m[2] || m[3])) {
+    const hours = parseFloat(m[1] || "0");
+    const minutes = parseFloat(m[2] || "0");
+    const seconds = parseFloat(m[3] || "0");
+    return (hours * 3600 + minutes * 60 + seconds) * 1000;
+  }
+  return undefined;
+}
+
+/**
+ * Type coercion for mapped values.
+ * @param {any} value
+ * @param {string} as - "number" | "dollars-from-cents" | "boolean" | "datetime" | "raw"
+ * @param {{ now?: number }} [opts]
+ * @returns {any}
+ */
+export function coerceValue(value, as, { now } = {}) {
+  switch (as) {
+    case "number": {
+      if (value == null) return undefined;
+      const n = Number(value);
+      return Number.isNaN(n) ? undefined : n;
+    }
+    case "dollars-from-cents": {
+      if (value == null) return undefined;
+      const n = Number(value);
+      return Number.isNaN(n) ? undefined : n / 100;
+    }
+    case "boolean": {
+      return !FALSE_SET.has(value);
+    }
+    case "datetime": {
+      if (value == null) return undefined;
+      if (typeof value === "string") {
+        // Try duration first
+        const dur = parseDuration(value.trim());
+        if (dur !== undefined) {
+          return (now ?? Date.now()) + dur;
+        }
+        // Try ISO-8601
+        const d = new Date(value);
+        if (!Number.isNaN(d.getTime())) return d.getTime();
+        return undefined;
+      }
+      if (typeof value === "number") {
+        if (!Number.isFinite(value)) return undefined;
+        // Epoch seconds vs ms heuristic
+        return value < 1e12 ? value * 1000 : value;
+      }
+      return undefined;
+    }
+    case "raw":
+    default:
+      return value;
+  }
+}
+
+const SHORTCODE_RE = /\{\{([^}]+)\}\}/g;
+
+const KNOWN_CTX_KEYS = new Set([
+  "providerApiKey",
+  "providerBaseUrl",
+  "providerId"
+]);
+
+/**
+ * Replace `{{shortcode}}` placeholders in a template string.
+ * @param {any} template
+ * @param {Record<string, string>} ctx
+ * @param {Record<string, string>} [env]
+ * @returns {any}
+ */
+export function interpolateShortcodes(template, ctx, env = {}) {
+  if (typeof template !== "string") return template;
+  return template.replace(SHORTCODE_RE, (_, key) => {
+    const trimmed = key.trim();
+    if (KNOWN_CTX_KEYS.has(trimmed)) return ctx[trimmed] ?? "";
+    const envMatch = trimmed.match(/^env\.(.+)$/);
+    if (envMatch) return env[envMatch[1]] ?? "";
+    return "";
+  });
+}
+
+const MAPPED_FIELDS = ["used", "limit", "remaining", "resetAt", "isUnlimited"];
+
+/**
+ * Extract normalized fields from a raw API response using a mapping config.
+ * @param {any} rawResponse
+ * @param {Record<string, any>} mapping
+ * @returns {Record<string, any>}
+ */
+export function extractMappedSnapshot(rawResponse, mapping) {
+  const result = {};
+  const now = Date.now();
+
+  for (const field of MAPPED_FIELDS) {
+    let value;
+
+    // Try primary path
+    const fieldMapping = mapping[field];
+    if (fieldMapping && fieldMapping.path) {
+      const raw = resolvePath(rawResponse, fieldMapping.path);
+      if (raw != null) {
+        value = coerceValue(raw, fieldMapping.as || "raw", { now });
+      }
+    }
+
+    // For "limit" field: try limitFallbacks chain if still null/undefined
+    if (value == null && field === "limit" && Array.isArray(mapping.limitFallbacks)) {
+      for (const fallbackPath of mapping.limitFallbacks) {
+        const raw = resolvePath(rawResponse, fallbackPath);
+        if (raw != null) {
+          const as = fieldMapping?.as || "number";
+          value = coerceValue(raw, as, { now });
+          if (value != null) break;
+        }
+      }
+    }
+
+    // Try constants as final fallback
+    if (value == null && mapping.constants && mapping.constants[field] != null) {
+      value = mapping.constants[field];
+    }
+
+    if (value !== undefined) {
+      result[field] = value;
+    }
+  }
+
+  return result;
+}

package/src/node/quota-probe-runner.js

@@ -0,0 +1,234 @@
+/**
+ * IO layer: executes quota probes (HTTP or custom JS), caches snapshots,
+ * and manages a per-provider circuit breaker.
+ */
+import { createContext, Script } from "node:vm";
+import { validateSnapshot, deriveSnapshot } from "../runtime/quota-probe.js";
+import { extractMappedSnapshot, interpolateShortcodes } from "./quota-probe-mapping.js";
+
+const CIRCUIT_THRESHOLD = 3;
+const CIRCUIT_PAUSE_MS = 5 * 60 * 1000;
+
+function makeErroredSnapshot(capKind, now, error, lastKnownGood) {
+  return {
+    capKind,
+    state: "errored",
+    error: { message: String(error) },
+    fetchedAt: now,
+    raw: null,
+    lastKnownGood: lastKnownGood || null,
+  };
+}
+
+async function executeHttp(probeConfig, shortcodeCtx, env, fetchFn) {
+  const http = probeConfig.http;
+  const url = interpolateShortcodes(http.url, shortcodeCtx, env);
+  const headers = {};
+  for (const h of http.headers || []) {
+    const headerKey = String(h.key || h.name || "").trim();
+    if (!headerKey) continue;
+    headers[interpolateShortcodes(headerKey, shortcodeCtx, env)] =
+      interpolateShortcodes(String(h.value || ""), shortcodeCtx, env);
+  }
+  const opts = { method: http.method || "GET", headers };
+  if (http.body !== undefined && opts.method !== "GET") {
+    opts.body = typeof http.body === "string"
+      ? interpolateShortcodes(http.body, shortcodeCtx, env)
+      : JSON.stringify(http.body);
+  }
+
+  const ac = new AbortController();
+  const timer = setTimeout(() => ac.abort(), http.timeoutMs || 5000);
+  opts.signal = ac.signal;
+
+  try {
+    const res = await fetchFn(url, opts);
+    clearTimeout(timer);
+    const body = await res.json().catch(() => null);
+    if (!res.ok) {
+      const err = new Error(`HTTP ${res.status}`);
+      err.responseBody = body;
+      throw err;
+    }
+    return body;
+  } catch (err) {
+    clearTimeout(timer);
+    if (err.responseBody !== undefined) {
+      const wrapped = new Error(err.message);
+      wrapped.responseBody = err.responseBody;
+      throw wrapped;
+    }
+    const msg = err.name === "AbortError" ? "timeout" : err.message;
+    throw new Error(msg);
+  }
+}
+
+async function executeCustom(probeConfig, shortcodeCtx, fetchFn, now) {
+  const { source, timeoutMs } = probeConfig.custom;
+  const sandbox = Object.freeze({
+    ctx: Object.freeze({
+      fetch: fetchFn,
+      providerApiKey: shortcodeCtx.providerApiKey,
+      providerBaseUrl: shortcodeCtx.providerBaseUrl,
+      providerId: shortcodeCtx.providerId,
+      log: () => {},
+      now,
+      timeoutMs,
+    }),
+  });
+  const vmCtx = createContext(sandbox);
+  const wrapped = `(async () => { ${source}\n return fetchUsage(ctx); })()`;
+  const script = new Script(wrapped, { timeout: timeoutMs });
+  return await script.runInContext(vmCtx, { timeout: timeoutMs });
+}
+
+export function createQuotaProbeRunner({ fetchImpl } = {}) {
+  const fetchFn = fetchImpl || globalThis.fetch;
+  const cache = new Map();
+  const circuits = new Map();
+
+  function getCircuit(providerId) {
+    if (!circuits.has(providerId)) {
+      circuits.set(providerId, { failures: 0, openUntil: 0 });
+    }
+    return circuits.get(providerId);
+  }
+
+  function isCircuitOpen(providerId, now) {
+    const c = circuits.get(providerId);
+    return !!c && c.failures >= CIRCUIT_THRESHOLD && now < c.openUntil;
+  }
+
+  function resetCircuit(providerId) {
+    circuits.delete(providerId);
+  }
+
+  async function executeProbe({ providerId, probeConfig, shortcodeCtx, env, now }) {
+    const capKind = probeConfig.capKind;
+    const prev = cache.get(providerId);
+    const lastKnownGood = prev?.state === "fresh" ? prev : prev?.lastKnownGood || null;
+
+    try {
+      let result;
+      if (probeConfig.mode === "custom") {
+        result = await executeCustom(probeConfig, shortcodeCtx, fetchFn, now);
+      } else {
+        const rawJson = await executeHttp(probeConfig, shortcodeCtx, env, fetchFn);
+        const mapped = extractMappedSnapshot(rawJson, probeConfig.http.mapping);
+        result = { ...mapped, capKind, raw: rawJson };
+      }
+
+      const toValidate = { ...result, capKind: result.capKind || capKind };
+      const validation = validateSnapshot(toValidate);
+      if (!validation.valid) {
+        const err = new Error(validation.error);
+        err.responseBody = result.raw;
+        throw err;
+      }
+
+      const derived = deriveSnapshot(toValidate);
+      const snapshot = {
+        capKind: derived.capKind,
+        used: derived.used,
+        limit: derived.limit,
+        remaining: derived.remaining,
+        resetAt: derived.resetAt,
+        isUnlimited: derived.isUnlimited,
+        state: "fresh",
+        fetchedAt: now,
+        error: null,
+        raw: result.raw ?? null,
+        lastKnownGood: null,
+      };
+
+      cache.set(providerId, snapshot);
+      resetCircuit(providerId);
+      return snapshot;
+    } catch (err) {
+      const circuit = getCircuit(providerId);
+      circuit.failures++;
+      if (circuit.failures >= CIRCUIT_THRESHOLD) {
+        circuit.openUntil = now + CIRCUIT_PAUSE_MS;
+      }
+      const snapshot = makeErroredSnapshot(capKind, now, err.message, lastKnownGood);
+      snapshot.raw = err.responseBody ?? null;
+      cache.set(providerId, snapshot);
+      return snapshot;
+    }
+  }
+
+  function getSnapshot(providerId) {
+    return cache.get(providerId) || null;
+  }
+
+  function getAllSnapshots() {
+    return new Map(cache);
+  }
+
+  // ── Refresh trigger management ──────────────────────────────────────
+  const pendingRefreshes = new Map();
+  const resetAtTimers = new Map();
+  const MAX_CONCURRENT_PROBES = 4;
+  const MAX_RESET_AT_DELAY_MS = 24 * 60 * 60 * 1000;
+  let activeConcurrent = 0;
+  let _onTriggerRefresh = null;
+
+  function scheduleResetAtRefresh(providerId, snapshot, probeConfig, shortcodeCtx, env) {
+    if (resetAtTimers.has(providerId)) {
+      clearTimeout(resetAtTimers.get(providerId));
+      resetAtTimers.delete(providerId);
+    }
+    if (!probeConfig.refreshTriggers?.onResetAt) return;
+    if (!snapshot.resetAt) return;
+    const delay = snapshot.resetAt - Date.now();
+    if (delay <= 0 || delay > MAX_RESET_AT_DELAY_MS) return;
+
+    const timerId = setTimeout(() => {
+      resetAtTimers.delete(providerId);
+      if (_onTriggerRefresh) _onTriggerRefresh({ providerId, trigger: "scheduler.resetAt" });
+      enqueueRefresh({ providerId, probeConfig, shortcodeCtx, env });
+    }, delay);
+    resetAtTimers.set(providerId, timerId);
+  }
+
+  async function enqueueRefresh({ providerId, probeConfig, shortcodeCtx, env, bypassCircuit }) {
+    if (isCircuitOpen(providerId, Date.now()) && !bypassCircuit) {
+      return getSnapshot(providerId);
+    }
+    if (pendingRefreshes.has(providerId)) {
+      return pendingRefreshes.get(providerId);
+    }
+
+    const run = async () => {
+      while (activeConcurrent >= MAX_CONCURRENT_PROBES) {
+        await new Promise(r => setTimeout(r, 50));
+      }
+      activeConcurrent++;
+      try {
+        const snap = await executeProbe({ providerId, probeConfig, shortcodeCtx, env, now: Date.now() });
+        scheduleResetAtRefresh(providerId, snap, probeConfig, shortcodeCtx, env);
+        return snap;
+      } finally {
+        activeConcurrent--;
+        pendingRefreshes.delete(providerId);
+      }
+    };
+
+    const promise = run();
+    pendingRefreshes.set(providerId, promise);
+    return promise;
+  }
+
+  function dispose() {
+    for (const id of resetAtTimers.values()) clearTimeout(id);
+    resetAtTimers.clear();
+  }
+
+  const runner = { executeProbe, getSnapshot, getAllSnapshots, isCircuitOpen, resetCircuit, enqueueRefresh, dispose };
+  Object.defineProperty(runner, "onTriggerRefresh", {
+    get() { return _onTriggerRefresh; },
+    set(fn) { _onTriggerRefresh = fn; },
+    enumerable: true,
+  });
+  return runner;
+}