kiku-agent 1.3.1 → 1.3.3

package/server/agent/engineService.js

@@ -0,0 +1,591 @@
+import { execSync, spawn } from 'node:child_process';
+import os from 'node:os';
+import {
+  buildLmsLoadArgs,
+  loadLmStudioModel,
+  unloadLmStudioModel,
+  wakeUpLmStudio,
+  probeLmStudioRuntime,
+} from './lmStudioWakeup.js';
+import {
+  registerBackgroundProcess,
+  getBackgroundProcess,
+  stopBackgroundProcess,
+  killWslProcessByName,
+} from './processManager.js';
+
+let cachedWslInfo = null;
+let lastWslCheck = 0;
+const WSL_CACHE_TTL_MS = 60000;
+
+
+export function detectPlatformAndWsl(bypassCache = false) {
+  const now = Date.now();
+  if (!bypassCache && cachedWslInfo && (now - lastWslCheck < WSL_CACHE_TTL_MS)) {
+    return cachedWslInfo;
+  }
+
+  const platform = process.platform;
+  let isWsl = false;
+  let distros = [];
+  let activeDistro = null;
+  let wslAvailable = false;
+
+  if (platform === 'linux') {
+    try {
+      const release = os.release().toLowerCase();
+      isWsl = release.includes('microsoft') || release.includes('wsl');
+    } catch {}
+    cachedWslInfo = {
+      platform: 'linux',
+      isNativeLinux: !isWsl,
+      isWsl,
+      wslAvailable: false,
+      distros: [],
+      activeDistro: isWsl ? 'wsl-current' : null,
+    };
+    lastWslCheck = now;
+    return cachedWslInfo;
+  }
+
+  if (platform === 'darwin') {
+    cachedWslInfo = {
+      platform: 'darwin',
+      isNativeLinux: false,
+      isWsl: false,
+      wslAvailable: false,
+      distros: [],
+      activeDistro: null,
+    };
+    lastWslCheck = now;
+    return cachedWslInfo;
+  }
+
+  if (platform === 'win32') {
+    try {
+      const stdout = execSync('wsl --list --quiet', {
+        encoding: 'utf8',
+        timeout: 3000,
+        stdio: ['pipe', 'pipe', 'ignore'],
+        windowsHide: true,
+      });
+      distros = stdout
+        .split(/\r?\n/)
+        .map(d => d.replace(/\0/g, '').trim())
+        .filter(Boolean);
+      wslAvailable = distros.length > 0;
+      activeDistro = distros[0] || null;
+    } catch {
+      wslAvailable = false;
+    }
+
+    cachedWslInfo = {
+      platform: 'win32',
+      isNativeLinux: false,
+      isWsl: false,
+      wslAvailable,
+      distros,
+      activeDistro,
+    };
+    lastWslCheck = now;
+    return cachedWslInfo;
+  }
+
+  cachedWslInfo = {
+    platform,
+    isNativeLinux: false,
+    isWsl: false,
+    wslAvailable: false,
+    distros: [],
+    activeDistro: null,
+  };
+  lastWslCheck = now;
+  return cachedWslInfo;
+}
+
+
+export function clearWslCache() {
+  cachedWslInfo = null;
+  lastWslCheck = 0;
+}
+
+
+export function checkEngineCompatibility(engine = 'lms', customInfo = null) {
+  const normEngine = String(engine).toLowerCase().trim();
+  const info = customInfo || detectPlatformAndWsl();
+
+  if (normEngine === 'lms' || normEngine === 'lmstudio') {
+    return {
+      supported: true,
+      engine: 'lms',
+      name: 'LM Studio (llama.cpp)',
+      platform: info.platform,
+      supportedOs: ['windows', 'macos', 'linux'],
+      targetLevel: 'consumer',
+      description: 'Consumer & workstation inference engine running on llama.cpp with universal 3-OS support.',
+    };
+  }
+
+  if (normEngine === 'vllm') {
+    if (info.platform === 'linux') {
+      return {
+        supported: true,
+        isEnterprise: true,
+        engine: 'vllm',
+        name: 'vLLM (Native Linux)',
+        platform: 'linux',
+        isWsl: Boolean(info.isWsl),
+        distro: info.activeDistro,
+        supportedOs: ['linux'],
+        targetLevel: 'enterprise',
+        description: 'Enterprise-grade PagedAttention engine for high-throughput Data Center workloads.',
+      };
+    }
+
+    if (info.platform === 'win32') {
+      const hasWsl = info.wslAvailable !== undefined ? Boolean(info.wslAvailable) : Boolean(info.hasWsl);
+      if (hasWsl) {
+        return {
+          supported: true,
+          isEnterprise: true,
+          viaWsl: true,
+          engine: 'vllm',
+          name: `vLLM (WSL2 ${info.activeDistro || 'Linux'})`,
+          platform: 'win32',
+          isWsl: true,
+          distro: info.activeDistro,
+          distros: info.distros || [],
+          supportedOs: ['linux'],
+          targetLevel: 'enterprise',
+          description: `Enterprise-grade PagedAttention engine running inside WSL2 Linux distribution "${info.activeDistro || 'Linux'}".`,
+        };
+      }
+      return {
+        supported: false,
+        isEnterprise: true,
+        engine: 'vllm',
+        name: 'vLLM (WSL Required)',
+        platform: 'win32',
+        isWsl: false,
+        supportedOs: ['linux'],
+        targetLevel: 'enterprise',
+        reason: 'vLLM is an enterprise Linux-only engine. On Windows, it requires WSL2 (Windows Subsystem for Linux). Install with: wsl --install',
+        installHint: 'wsl --install',
+      };
+    }
+
+    if (info.platform === 'darwin') {
+      return {
+        supported: false,
+        isEnterprise: true,
+        engine: 'vllm',
+        name: 'vLLM (Unsupported on macOS)',
+        platform: 'darwin',
+        isWsl: false,
+        supportedOs: ['linux'],
+        targetLevel: 'enterprise',
+        reason: 'vLLM is designed exclusively for Linux with CUDA/ROCm hardware. macOS is not supported natively.',
+      };
+    }
+  }
+
+  return {
+    supported: false,
+    engine: normEngine,
+    name: normEngine,
+    platform: info.platform,
+    reason: `Unknown engine: ${engine}`,
+  };
+}
+
+
+export function buildVllmServeArgs(model, options = {}) {
+  const targetModel = model || 'google/gemma-4-e4b';
+  const args = ['serve', targetModel];
+
+  const port = Number(options.port) || 8000;
+  args.push('--port', String(port));
+
+  const host = options.host || '0.0.0.0';
+  args.push('--host', host);
+
+  const memUtil = options.gpuMemoryUtilization !== undefined ? Number(options.gpuMemoryUtilization) : 0.90;
+  if (Number.isFinite(memUtil) && memUtil >= 0.1 && memUtil <= 1.0) {
+    args.push('--gpu-memory-utilization', String(memUtil));
+  }
+
+  const maxLen = Number(options.maxModelLen);
+  if (Number.isFinite(maxLen) && maxLen >= 512 && maxLen <= 131072) {
+    args.push('--max-model-len', String(Math.floor(maxLen)));
+  }
+
+  const tp = Number(options.tensorParallelSize);
+  if (Number.isFinite(tp) && tp >= 1 && tp <= 8) {
+    args.push('--tensor-parallel-size', String(Math.floor(tp)));
+  }
+
+  if (options.dtype && ['auto', 'half', 'float16', 'bfloat16', 'float'].includes(String(options.dtype).toLowerCase())) {
+    args.push('--dtype', String(options.dtype).toLowerCase());
+  }
+
+  if (options.enforceEager) {
+    args.push('--enforce-eager');
+  }
+
+  if (options.kvCacheDtype && ['auto', 'fp8', 'fp8_e5m2', 'fp8_e4m3'].includes(String(options.kvCacheDtype).toLowerCase())) {
+    args.push('--kv-cache-dtype', String(options.kvCacheDtype).toLowerCase());
+  }
+
+  if (options.enablePrefixCaching) {
+    args.push('--enable-prefix-caching');
+  }
+
+  if (options.trustRemoteCode) {
+    args.push('--trust-remote-code');
+  }
+
+  return args;
+}
+
+
+export function buildVllmExecutionPlan(model, options = {}) {
+  const customInfo = options.customInfo || (options.platform ? {
+    platform: options.platform,
+    isWsl: options.platform === 'win32',
+    wslAvailable: options.platform === 'win32',
+    hasWsl: options.platform === 'win32',
+    activeDistro: options.distro || 'Ubuntu',
+    distros: [options.distro || 'Ubuntu'],
+  } : null);
+  const compatibility = checkEngineCompatibility('vllm', customInfo);
+  if (!compatibility.supported) {
+    return { ok: false, reason: compatibility.reason, compatibility };
+  }
+
+  const serveArgs = buildVllmServeArgs(model, options);
+  const serveCmdStr = `vllm ${serveArgs.join(' ')}`;
+
+  if (compatibility.platform === 'win32') {
+    const distro = options.distro || compatibility.distro || 'Ubuntu-26.04';
+    const customVenv = options.venvPath || process.env.VLLM_VENV_PATH || '';
+    const dynamicVenvCheck = [
+      customVenv ? `if [ -f "${customVenv}/bin/activate" ]; then source "${customVenv}/bin/activate" 2>/dev/null; fi;` : '',
+      'if ! command -v vllm >/dev/null 2>&1; then',
+      '  for act in ~/.venvs/*/bin/activate ~/.venv/bin/activate ~/venv/bin/activate ./.venv/bin/activate; do',
+      '    if [ -f "$act" ]; then source "$act" 2>/dev/null; if command -v vllm >/dev/null 2>&1; then break; fi; fi;',
+      '  done;',
+      'fi;',
+      'export PATH="$HOME/.local/bin:/usr/local/bin:$PATH";',
+    ].filter(Boolean).join(' ');
+    const fullBashCmd = `${dynamicVenvCheck} ${serveCmdStr}`;
+    const rawCommand = `wsl -d ${distro} bash -c "${fullBashCmd}"`;
+    return {
+      ok: true,
+      engine: 'vllm',
+      isWsl: true,
+      distro,
+      binary: 'wsl.exe',
+      command: 'wsl.exe',
+      args: ['-d', distro, 'bash', '-c', fullBashCmd],
+      rawCommand,
+      displayCommand: rawCommand,
+      compatibility,
+      serveArgs,
+      port: options.port || 8000,
+    };
+  }
+
+  return {
+    ok: true,
+    engine: 'vllm',
+    isWsl: Boolean(compatibility.isWsl),
+    binary: 'vllm',
+    command: 'vllm',
+    args: serveArgs,
+    rawCommand: serveCmdStr,
+    displayCommand: serveCmdStr,
+    compatibility,
+    serveArgs,
+    port: options.port || 8000,
+  };
+}
+
+
+export async function getEnginesHealth(endpoints = {}) {
+  const lmsUrl = endpoints.lms || 'http://127.0.0.1:1234/v1';
+  const vllmUrl = endpoints.vllm || 'http://127.0.0.1:8000/v1';
+
+  const lmsCompat = checkEngineCompatibility('lms');
+  const vllmCompat = checkEngineCompatibility('vllm');
+
+  const lmsProbe = await probeLmStudioRuntime(lmsUrl, { timeoutMs: 2000, bypassCache: true }).catch(() => null);
+
+  let vllmProbe = { reachable: false, models: [], loadedModel: null };
+  try {
+    const root = vllmUrl.replace(/\/v1\/?$/, '').replace(/\/+$/, '');
+    const res = await fetch(`${root}/v1/models`, { signal: AbortSignal.timeout(2000) });
+    if (res.ok) {
+      const json = await res.json();
+      const modelList = Array.isArray(json?.data) ? json.data.map(m => m.id) : [];
+      vllmProbe = {
+        reachable: true,
+        models: modelList,
+        loadedModel: modelList[0] || null,
+        count: modelList.length,
+      };
+    }
+  } catch {}
+
+  return {
+    engines: {
+      lms: {
+        id: 'lms',
+        name: lmsCompat.name,
+        targetLevel: lmsCompat.targetLevel,
+        supported: lmsCompat.supported,
+        compatibility: lmsCompat,
+        reachable: Boolean(lmsProbe?.reachable),
+        loadedModel: lmsProbe?.loadedModel || null,
+        loadedModels: lmsProbe?.loadedModels || [],
+        models: lmsProbe?.models || [],
+        port: 1234,
+        baseUrl: lmsUrl,
+        process: getBackgroundProcess('lmstudio'),
+      },
+      vllm: {
+        id: 'vllm',
+        name: vllmCompat.name,
+        targetLevel: vllmCompat.targetLevel,
+        supported: vllmCompat.supported,
+        compatibility: vllmCompat,
+        reachable: Boolean(vllmProbe.reachable),
+        loadedModel: vllmProbe.loadedModel,
+        models: vllmProbe.models,
+        port: 8000,
+        baseUrl: vllmUrl,
+        process: getBackgroundProcess('vllm'),
+      },
+    },
+    platform: detectPlatformAndWsl(),
+  };
+}
+
+
+export function calculateAutoTensorSplit(gpuMetrics, reservedMainGpuGb = 3.5, mainGpuIndex = 0) {
+  const gpus = Array.isArray(gpuMetrics)
+    ? gpuMetrics
+    : (Array.isArray(gpuMetrics?.gpus) ? gpuMetrics.gpus : []);
+
+  if (gpus.length < 2) {
+    const singleVram = gpus[0]
+      ? (Number(gpus[0].freeGb) || (gpus[0].freeMb ? gpus[0].freeMb / 1024 : 0) || Number(gpus[0].totalGb) || 24)
+      : 24;
+    return {
+      applicable: false,
+      tensorSplit: '1',
+      ratios: [1],
+      usableGb: [Math.max(0.5, Number(Number(singleVram).toFixed(1)))],
+      reason: 'Requires at least 2 GPUs for multi-GPU tensor split',
+      toString() { return this.tensorSplit; },
+    };
+  }
+
+  const usableGb = gpus.map((gpu, idx) => {
+    const free = Number(gpu.freeGb) || (gpu.freeMb ? gpu.freeMb / 1024 : 0);
+    const total = Number(gpu.totalGb) || (gpu.totalMb ? gpu.totalMb / 1024 : 0);
+    const rawVram = free > 0 ? free : (total > 0 ? total : 24);
+    const headroom = (idx === mainGpuIndex) ? reservedMainGpuGb : 0;
+    return Math.max(0.5, Number((rawVram - headroom).toFixed(2)));
+  });
+
+  const ratios = usableGb.map(gb => Math.max(1, Math.round(gb)));
+  const tensorSplit = ratios.join(',');
+
+  return {
+    applicable: true,
+    tensorSplit,
+    ratios,
+    usableGb,
+    mainGpu: mainGpuIndex,
+    reservedMainGpuGb,
+    toString() { return this.tensorSplit; },
+  };
+}
+
+
+export async function loadModelOnEngine(engine = 'lms', model, options = {}) {
+  const normEngine = String(engine).toLowerCase().trim();
+
+  if (normEngine === 'lms' || normEngine === 'lmstudio') {
+    const res = await wakeUpLmStudio({
+      baseUrl: options.baseUrl || 'http://127.0.0.1:1234/v1',
+      model,
+      quiet: options.quiet ?? false,
+      gpuOffload: options.gpuOffload,
+      cpuThreads: options.cpuThreads,
+      contextLength: options.contextLength,
+      splitMode: options.splitMode,
+      tensorSplit: options.tensorSplit,
+      mainGpu: options.mainGpu,
+      flashAttn: options.flashAttn,
+      batchSize: options.batchSize,
+      ubatchSize: options.ubatchSize,
+      htMode: options.htMode || options.hyperThreadingMode,
+      physicalCores: options.physicalCores,
+      hyperThreading: options.hyperThreading,
+    });
+    return {
+      engine: 'lms',
+      ...res,
+    };
+  }
+
+  if (normEngine === 'vllm') {
+    const plan = buildVllmExecutionPlan(model, options);
+    if (!plan.ok) {
+      return {
+        ok: false,
+        engine: 'vllm',
+        reason: plan.reason,
+      };
+    }
+
+    const port = options.port || 8000;
+    const vllmUrl = options.baseUrl || `http://127.0.0.1:${port}/v1`;
+
+    
+    try {
+      const root = vllmUrl.replace(/\/v1\/?$/, '');
+      const ping = await fetch(`${root}/v1/models`, { signal: AbortSignal.timeout(1500) });
+      if (ping.ok) {
+        const json = await ping.json();
+        const models = (json?.data || []).map(m => m.id);
+        const hasModel = models.some(m => m === model || m.includes(model));
+        if (hasModel) {
+          return {
+            ok: true,
+            engine: 'vllm',
+            alreadyRunning: true,
+            modelLoaded: true,
+            loadedModel: model,
+          };
+        }
+      }
+    } catch {}
+
+
+    try {
+      const child = spawn(plan.binary, plan.args, {
+        detached: true,
+        stdio: 'ignore',
+        windowsHide: true,
+      });
+      if (child.pid) {
+        registerBackgroundProcess('vllm', child, {
+          command: plan.binary,
+          args: plan.args,
+          metadata: {
+            engine: 'vllm',
+            model,
+            isWsl: plan.isWsl,
+            distro: plan.distro,
+            port,
+            wslProcessPattern: 'vllm',
+          },
+        });
+      }
+      child.unref();
+
+      const maxWaitMs = options.maxWaitMs || 45000;
+      const start = Date.now();
+      while (Date.now() - start < maxWaitMs) {
+        await new Promise(r => setTimeout(r, 2000));
+        try {
+          const root = vllmUrl.replace(/\/v1\/?$/, '');
+          const res = await fetch(`${root}/v1/models`, { signal: AbortSignal.timeout(1500) });
+          if (res.ok) {
+            const data = await res.json();
+            const models = (data?.data || []).map(m => m.id);
+            return {
+              ok: true,
+              engine: 'vllm',
+              awakened: true,
+              modelLoaded: true,
+              loadedModel: models[0] || model,
+              models,
+              pid: child.pid,
+            };
+          }
+        } catch {}
+      }
+
+      return {
+        ok: false,
+        engine: 'vllm',
+        reason: `vLLM server started via ${plan.isWsl ? `WSL (${plan.distro})` : 'Linux'} but port ${port} did not respond within ${Math.round(maxWaitMs / 1000)}s`,
+        rawCommand: plan.rawCommand,
+        pid: child.pid,
+      };
+    } catch (err) {
+      return {
+        ok: false,
+        engine: 'vllm',
+        reason: `Failed to spawn vLLM: ${err.message}`,
+      };
+    }
+  }
+
+  return {
+    ok: false,
+    engine: normEngine,
+    reason: `Unsupported engine "${normEngine}"`,
+  };
+}
+
+export async function stopEngine(engine = 'vllm', options = {}) {
+  const normEngine = String(engine).toLowerCase().trim();
+  if (normEngine === 'vllm') {
+    const stopRes = stopBackgroundProcess('vllm', options);
+    const distro = options.distro || 'Ubuntu-26.04';
+    if (process.platform === 'win32') {
+      killWslProcessByName(distro, 'vllm');
+    }
+    return { ok: true, engine: 'vllm', stopped: stopRes.ok };
+  }
+  if (normEngine === 'lms' || normEngine === 'lmstudio') {
+    const stopRes1 = stopBackgroundProcess('lmstudio', options);
+    const stopRes2 = stopBackgroundProcess('lmstudio-server', options);
+    try {
+      const { resolveLmsCli } = await import('./lmStudioWakeup.js');
+      const lmsCmd = resolveLmsCli();
+      const { spawnSync } = await import('node:child_process');
+      spawnSync(lmsCmd, ['server', 'stop'], { stdio: 'ignore', windowsHide: true });
+    } catch {}
+    return { ok: true, engine: 'lms', stopped: stopRes1.ok || stopRes2.ok };
+  }
+  return { ok: false, reason: `Unknown engine "${engine}"` };
+}
+
+export async function unloadModelFromEngine(engine = 'lms', model = null, options = {}) {
+  const normEngine = String(engine).toLowerCase().trim();
+  if (normEngine === 'lms' || normEngine === 'lmstudio') {
+    return await unloadLmStudioModel(model, options);
+  }
+  if (normEngine === 'vllm') {
+    const stopRes = await stopEngine('vllm', options);
+    return {
+      ok: stopRes.ok,
+      unloaded: stopRes.ok,
+      engine: 'vllm',
+      model: model || 'all',
+      message: stopRes.ok ? 'vLLM server stopped and GPU VRAM freed' : 'Failed to stop vLLM server'
+    };
+  }
+  return {
+    ok: false,
+    engine: normEngine,
+    message: `Unsupported engine "${normEngine}" for unload`
+  };
+}
+
+export { buildLmsLoadArgs, loadLmStudioModel, unloadLmStudioModel, wakeUpLmStudio };