kiku-agent 1.2.0 → 1.3.0

package/server/agent/vramService.js

@@ -0,0 +1,572 @@
+import { spawn } from 'node:child_process';
+import os from 'node:os';
+import {
+  classifyAllocation,
+  estimateVramFootprint,
+  gbToBytes,
+  knownSwaProfile,
+  normalizeQuantName,
+  recommendAdjustments,
+} from '../../src/lib/vram/estimator.js';
+import { matchModelIdentifier } from './config.js';
+import { getGpuMetrics } from './gpuMetrics.js';
+import { listInstalledGgufs, resolveModelConfig } from './modelConfig.js';
+import { classifyHardwareTier, calculateDynamicVramBudget, HARDWARE_TIERS } from './telemetry/hardwareTiers.js';
+import { techPowerUpClient } from './telemetry/techPowerUpClient.js';
+
+function runProcessCapture(cmd, args, timeoutMs = 2500) {
+  return new Promise((resolve) => {
+    let stdout = '';
+    let stderr = '';
+    let settled = false;
+    const child = spawn(cmd, args, { stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true });
+    const finish = (result) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      resolve(result);
+    };
+    const timer = setTimeout(() => {
+      try { child.kill('SIGKILL'); } catch { /* already dead */ }
+      finish({ ok: false, stdout: '', stderr: 'timeout' });
+    }, timeoutMs);
+    child.stdout?.on('data', d => { stdout += d.toString(); });
+    child.stderr?.on('data', d => { stderr += d.toString(); });
+    child.on('error', (err) => finish({ ok: false, stdout: '', stderr: err.message }));
+    child.on('close', (code) => finish({ ok: code === 0, stdout: stdout.trim(), stderr: stderr.trim() }));
+  });
+}
+
+function parseRocmSmiOutput(stdout) {
+
+  const rows = stdout.split('\n').map(l => l.trim()).filter(Boolean);
+  const gpus = [];
+  for (const row of rows) {
+    const cells = row.split(',').map(c => c.trim());
+    if (cells[0] === '' || Number.isNaN(Number(cells[0]))) continue;
+    const totalBytes = Number(cells[1]);
+    const usedBytes = cells.length > 2 ? Number(cells[2]) : NaN;
+    if (Number.isFinite(totalBytes) && totalBytes > 0) {
+      gpus.push({ totalBytes, usedBytes: Number.isFinite(usedBytes) ? usedBytes : null });
+    }
+  }
+  return gpus;
+}
+
+async function detectAmdGpu() {
+  const smi = await runProcessCapture('rocm-smi', ['--showmeminfo', 'vram', '--csv']);
+  if (!smi.ok || !smi.stdout) return null;
+  const gpus = parseRocmSmiOutput(smi.stdout);
+  if (gpus.length === 0) return null;
+  return {
+    available: true,
+    vendor: 'AMD',
+    name: 'AMD GPU (ROCm)',
+    totalBytes: gpus[0].totalBytes,
+    freeBytes: gpus[0].usedBytes !== null ? gpus[0].totalBytes - gpus[0].usedBytes : null,
+    usedBytes: gpus[0].usedBytes,
+    source: 'rocm-smi',
+    approximate: false,
+    count: gpus.length,
+  };
+}
+
+function appleUnifiedMemoryTarget() {
+  const totalBytes = Math.round(os.totalmem() * 0.70);
+  return {
+    available: true,
+    vendor: 'Apple',
+    name: `${os.cpus()[0]?.model?.includes('Apple') ? 'Apple Silicon' : 'Unified Memory'} (Metal)`,
+    totalBytes,
+    freeBytes: null,
+    usedBytes: null,
+    source: 'unified-memory-heuristic',
+    approximate: true,
+    count: 1,
+  };
+}
+
+
+export async function getHardwareTarget() {
+  const envOverride = Number(process.env.KIKU_VRAM_TOTAL_GB);
+  if (Number.isFinite(envOverride) && envOverride > 0) {
+    return {
+      available: true,
+      vendor: 'Configured',
+      name: 'Configured VRAM budget (KIKU_VRAM_TOTAL_GB)',
+      totalBytes: Math.round(gbToBytes(envOverride)),
+      freeBytes: null,
+      usedBytes: null,
+      source: 'env-override',
+      approximate: false,
+      count: 1,
+    };
+  }
+
+  try {
+    const probe = await getGpuMetrics();
+    if (probe?.available && probe.primary) {
+      const primary = probe.primary;
+      if (primary.vendor === 'Apple' && process.platform === 'darwin') {
+        return appleUnifiedMemoryTarget();
+      }
+    
+      if (primary.totalMb > 0) {
+        return {
+          available: true,
+          vendor: primary.vendor,
+          name: primary.name,
+          totalBytes: Math.round(primary.totalMb * 1024 * 1024),
+          freeBytes: primary.freeMb !== null && primary.freeMb !== undefined
+            ? Math.round(primary.freeMb * 1024 * 1024)
+            : null,
+          usedBytes: primary.usedMb !== null && primary.usedMb !== undefined
+            ? Math.round(primary.usedMb * 1024 * 1024)
+            : null,
+          source: primary.source || 'gpu-probe',
+          approximate: Boolean(primary.approximate),
+          count: probe.count,
+        };
+      }
+    }
+  } catch {
+   
+  }
+
+  if (process.platform !== 'win32') {
+    const amd = await detectAmdGpu();
+    if (amd) return amd;
+    if (process.platform === 'darwin') return appleUnifiedMemoryTarget();
+  }
+
+  return {
+    available: false,
+    vendor: null,
+    name: null,
+    totalBytes: null,
+    freeBytes: null,
+    usedBytes: null,
+    source: null,
+    approximate: false,
+    reason: 'No discrete GPU with queryable VRAM detected. Set KIKU_VRAM_TOTAL_GB to evaluate a known budget.',
+  };
+}
+
+
+export async function buildHardwareReport() {
+  const [target, probe] = await Promise.all([
+    getHardwareTarget(),
+    getGpuMetrics().catch(() => ({ available: false })),
+  ]);
+  return {
+    available: Boolean(target.available),
+    installedModels: listInstalledGgufs(),
+    target: {
+      vendor: target.vendor,
+      name: target.name,
+      totalBytes: target.totalBytes,
+      freeBytes: target.freeBytes,
+      usedBytes: target.usedBytes ?? null,
+      totalGb: target.totalBytes ? Number((target.totalBytes / 1024 ** 3).toFixed(2)) : null,
+      freeGb: target.freeBytes ? Number((target.freeBytes / 1024 ** 3).toFixed(2)) : null,
+      usedGb: target.usedBytes ? Number((target.usedBytes / 1024 ** 3).toFixed(2)) : null,
+      source: target.source,
+      approximate: Boolean(target.approximate),
+    },
+    probe: probe.available
+      ? {
+        vendor: probe.vendor,
+        count: probe.count,
+        gpus: (probe.gpus || []).map(g => ({
+          index: g.index,
+          name: g.name,
+          vendor: g.vendor,
+          totalMb: g.totalMb ?? null,
+          usedMb: g.usedMb ?? null,
+          freeMb: g.freeMb ?? null,
+          utilizationGpu: g.utilizationGpu ?? null,
+          temperatureC: g.temperatureC ?? null,
+          driverVersion: g.driverVersion ?? null,
+        })),
+      }
+      : null,
+    platform: `${process.platform}/${process.arch}`,
+    checkedAt: new Date().toISOString(),
+  };
+}
+
+const MERGE_KEYS = ['layers', 'queryHeads', 'kvHeads', 'headDim', 'hiddenSize', 'paramCount', 'paramsB', 'contextLength', 'quant'];
+
+
+export function parseEstimateRequest(body = {}, searchParams = null) {
+  const pick = (key) => {
+    if (body[key] !== undefined && body[key] !== null && body[key] !== '') return body[key];
+    if (!searchParams) return undefined;
+    for (const candidate of [key, key.toLowerCase(), key.replace(/([A-Z])/g, m => `_${m.toLowerCase()}`)]) {
+      const raw = searchParams.get(candidate);
+      if (raw !== null) return raw;
+    }
+    return undefined;
+  };
+  const num = (value) => {
+    if (value === undefined || value === null || value === '') return undefined;
+    const parsed = Number(value);
+    return Number.isFinite(parsed) ? parsed : undefined;
+  };
+  return {
+    model: pick('model') || pick('model_name') || undefined,
+    modelPath: pick('model_path') || pick('modelPath') || undefined,
+    quant: pick('quant') || undefined,
+    contextLength: num(pick('contextLength') ?? pick('ctx')),
+    paramsB: num(pick('paramsB') ?? pick('params')),
+    paramCount: num(pick('paramCount')),
+    layers: num(pick('layers')),
+    kvHeads: num(pick('kvHeads') ?? pick('kv_heads')),
+    queryHeads: num(pick('queryHeads') ?? pick('query_heads')),
+    headDim: num(pick('headDim') ?? pick('head_dim')),
+    hiddenSize: num(pick('hiddenSize') ?? pick('hidden_size')),
+    batchSize: num(pick('batchSize')),
+    cacheBits: num(pick('cacheBits')),
+    images: num(pick('images')),
+    hardwareOverrideGb: num(pick('hardwareOverrideGb') ?? pick('availableVramGb')),
+    includeHardware: body.includeHardware !== false,
+  };
+}
+
+function pickFirst(...values) {
+  for (const value of values) {
+    if (value !== undefined && value !== null && value !== '') return value;
+  }
+  return null;
+}
+
+
+
+const LMSTUDIO_PROBE_TTL_MS = 15_000;
+const lmStudioProbeCache = new Map();
+
+export async function probeLmStudioRuntime(baseUrl = 'http://127.0.0.1:1234/v1') {
+  const key = String(baseUrl || 'http://127.0.0.1:1234/v1');
+  const cached = lmStudioProbeCache.get(key);
+  if (cached) {
+    if (cached.inFlight) return cached.inFlight;
+    if (Date.now() - cached.fetchedAt < LMSTUDIO_PROBE_TTL_MS) return cached.data;
+  }
+  const entry = { fetchedAt: Date.now(), data: null, inFlight: probeLmStudioRuntimeUncached(baseUrl) };
+  lmStudioProbeCache.set(key, entry);
+  try {
+    const result = await entry.inFlight;
+    entry.data = result;
+    entry.fetchedAt = Date.now();
+    return result;
+  } catch (err) {
+    lmStudioProbeCache.delete(key);
+    throw err;
+  } finally {
+    entry.inFlight = null;
+  }
+}
+
+async function probeLmStudioRuntimeUncached(baseUrl = 'http://127.0.0.1:1234/v1') {
+  const result = { reachable: false, loadedModel: null, loadedModelType: null, loadedContextLength: null, quantization: null, pendingModel: null };
+  try {
+    const root = String(baseUrl || 'http://127.0.0.1:1234/v1').replace(/\/v1\/?$/, '');
+    const res = await fetch(`${root}/api/v0/models`, { signal: AbortSignal.timeout(2500) });
+    if (!res.ok) return result;
+    const data = await res.json();
+    const models = Array.isArray(data?.data) ? data.data : [];
+    if (models.length === 0) return result;
+    result.reachable = true;
+    const loaded = models.find(m => m.state === 'loaded' && (m.type === 'llm' || m.type === 'vlm'))
+      || models.find(m => m.state === 'loaded');
+    if (loaded) {
+      result.loadedModel = loaded.id || null;
+      result.loadedModelType = loaded.type || null;
+      result.loadedContextLength = loaded.loaded_context_length ?? loaded.max_context_length ?? null;
+      result.quantization = loaded.quantization || null;
+      return result;
+    }
+  
+    const pending = models.find(m => m.state === 'loading' && (m.type === 'llm' || m.type === 'vlm'))
+      || models.find(m => m.state === 'loading');
+    if (pending) result.pendingModel = pending.id || null;
+    return result;
+  } catch {
+    return result;
+  }
+}
+
+export async function computeVramEstimate(request = {}) {
+  let modelName = request.modelName || request.model || request.model_id || null;
+  const modelPath = request.modelPath || request.model_path || null;
+  const includeHardware = request.includeHardware !== false;
+
+  const isAuto = !modelName || String(modelName).trim().toLowerCase() === 'auto';
+  let lmRuntime = null;
+  if (!modelPath) {
+    try {
+      lmRuntime = await probeLmStudioRuntime(request.baseUrl).catch(() => null);
+      if (lmRuntime?.reachable && lmRuntime.loadedModel) {
+        const isTargetLoaded = isAuto || matchModelIdentifier(modelName, lmRuntime.loadedModel);
+        if (isTargetLoaded) {
+          if (isAuto) modelName = lmRuntime.loadedModel;
+          if (!request.contextLength && lmRuntime.loadedContextLength) {
+            request.contextLength = lmRuntime.loadedContextLength;
+          }
+          if (!request.quant && lmRuntime.quantization) {
+            request.quant = lmRuntime.quantization;
+          }
+        }
+      } else if (isAuto && lmRuntime?.reachable && lmRuntime.pendingModel) {
+        modelName = lmRuntime.pendingModel;
+      }
+    } catch {
+      /* ignore probe error */
+    }
+  }
+
+  if (isAuto && (!modelName || String(modelName).trim().toLowerCase() === 'auto')) {
+    const installed = listInstalledGgufs();
+    if (installed.length > 0) {
+      modelName = installed[0].path || installed[0].fileName;
+    } else {
+      modelName = 'google/gemma-4-e4b';
+    }
+  }
+
+  const resolved = await resolveModelConfig({
+    modelPath,
+    modelName,
+    overrides: request,
+    allowNetwork: request.allowNetwork !== false,
+  });
+
+  const overrides = {};
+  for (const key of MERGE_KEYS) {
+    const value = pickFirst(request[key], resolved.config?.[key]);
+    if (value !== null) overrides[key] = value;
+  }
+
+  const swaProfile = (() => {
+    const profile = { ...(knownSwaProfile(resolved.config?.arch) || {}), ...(resolved.config?.swa || {}) };
+    return Number(profile.windowTokens) > 0 ? profile : null;
+  })();
+
+  const vision = {
+    images: Number(request.images) || 0,
+    tokensPerImage: Number(request.tokensPerImage) || resolved.config?.vision?.tokensPerImage || null,
+    encoderParamsB: pickFirst(request.visionEncoderParamsB, resolved.config?.vision?.encoderParamsB),
+  };
+
+ 
+  const weightBytesOverride = (resolved.source === 'gguf' &&
+    resolved.config?.fileSizeBytes &&
+    (!overrides.quant || normalizeQuantName(overrides.quant) === normalizeQuantName(resolved.config.quant)))
+    ? resolved.config.fileSizeBytes
+    : null;
+
+  const estimate = estimateVramFootprint({
+    modelId: modelName || modelPath || resolved.config?.name || null,
+    quant: overrides.quant || 'q4_k_m',
+    paramCount: overrides.paramCount,
+    paramsB: overrides.paramsB,
+    weightBytesOverride,
+    layers: overrides.layers,
+    queryHeads: overrides.queryHeads,
+    kvHeads: overrides.kvHeads,
+    headDim: overrides.headDim,
+    hiddenSize: overrides.hiddenSize,
+    contextLength: overrides.contextLength,
+    batchSize: Number(request.batchSize) || 1,
+    cacheBits: Number(request.cacheBits) || 16,
+    swa: swaProfile,
+    vision,
+    activationGb: request.activationGb,
+    overheadGb: request.overheadGb,
+  });
+
+  const overrideGb = Number(request.hardwareOverrideGb);
+  const hardware = overrideGb > 0
+    ? {
+      available: true,
+      vendor: 'Configured',
+      name: request.hardwareName || 'Configured VRAM budget',
+      totalBytes: Math.round(gbToBytes(overrideGb)),
+      freeBytes: null,
+      usedBytes: null,
+      source: 'request-override',
+      approximate: false,
+      count: 1,
+    }
+    : (includeHardware ? await getHardwareTarget() : null);
+  const predictiveClassification = hardware?.available
+    ? classifyAllocation({ totalBytes: estimate.totalBytes, availableBytes: hardware.totalBytes })
+    : classifyAllocation({});
+
+
+  const isLoadedInLm = Boolean(lmRuntime?.reachable && lmRuntime.loadedModel &&
+    (isAuto || matchModelIdentifier(modelName, lmRuntime.loadedModel)));
+
+  const measured = isLoadedInLm
+    && hardware?.available && hardware.usedBytes > 0
+    ? {
+      ...classifyAllocation({ totalBytes: hardware.usedBytes, availableBytes: hardware.totalBytes }),
+      usedBytes: hardware.usedBytes,
+      totalBytes: hardware.totalBytes,
+      loadedModel: lmRuntime.loadedModel,
+    }
+    : null;
+
+  const classification = measured
+    ? {
+      zone: measured.zone,
+      usagePct: measured.usagePct,
+      headroomBytes: measured.headroomBytes,
+      message: measured.message,
+      requiresForce: measured.requiresForce,
+      basis: 'measured',
+      usedBytes: measured.usedBytes,
+      totalBytes: measured.totalBytes,
+      loadedModel: measured.loadedModel,
+    }
+    : {
+      ...predictiveClassification,
+      basis: predictiveClassification.zone === 'UNKNOWN' ? 'unavailable' : 'predicted',
+    };
+
+  const notes = [];
+  if (measured) {
+    notes.push(`Auto mode: "${measured.loadedModel}" is loaded in LM Studio — the zone reflects measured GPU usage from the hardware probe, while the breakdown stays a predictive worst-case for the shown context.`);
+  }
+  if (swaProfile && estimate.swa) {
+    notes.push(`KV cache modeled as sliding-window hybrid: ${estimate.swa.globalLayers} full-context layers + ${estimate.swa.localLayers} windowed layers (window ${estimate.swa.windowTokens} tokens).`);
+  }
+
+  let remediation = hardware?.available && predictiveClassification.zone !== 'GREEN'
+    ? recommendAdjustments(estimate, hardware.totalBytes)
+    : [];
+  if (remediation.length === 0 && hardware?.available && predictiveClassification.zone !== 'GREEN') {
+    remediation = recommendAdjustments(estimate, hardware.totalBytes, { targetZone: 'YELLOW' });
+  }
+
+  return {
+    ok: true,
+    input: {
+      modelName,
+      modelPath,
+      contextLength: overrides.contextLength,
+      quant: overrides.quant,
+      batchSize: estimate.batchSize,
+      cacheBits: estimate.cacheBits,
+      images: vision.images,
+    },
+    configSource: resolved.source,
+    approximate: Boolean(resolved.approx),
+    configError: resolved.error || null,
+    architecture: {
+      name: resolved.config?.name || modelName || null,
+      arch: resolved.config?.arch || null,
+      layers: estimate.layers,
+      queryHeads: estimate.queryHeads,
+      kvHeads: estimate.kvHeads,
+      headDim: estimate.headDim,
+      hiddenSize: estimate.hiddenSize,
+      paramCount: estimate.paramCount,
+      paramsB: estimate.paramCount ? Number((estimate.paramCount / 1e9).toFixed(2)) : null,
+      expertCount: resolved.config?.expertCount ?? null,
+      registryNotes: resolved.config?.registryNotes ?? null,
+    },
+    estimate,
+    hardware: hardware?.available
+      ? {
+        vendor: hardware.vendor,
+        name: hardware.name,
+        totalBytes: hardware.totalBytes,
+        freeBytes: hardware.freeBytes,
+        usedBytes: hardware.usedBytes ?? null,
+        source: hardware.source,
+        approximate: hardware.approximate,
+      }
+      : null,
+    classification,
+    predictiveClassification,
+    notes,
+    remediation,
+  };
+}
+
+
+export async function runVramPreflight({
+  model = null,
+  modelPath = null,
+  quant = null,
+  contextLength = null,
+  force = false,
+  allowNetwork = true,
+} = {}) {
+  const result = await computeVramEstimate({
+    model,
+    modelPath,
+    quant,
+    contextLength,
+    includeHardware: true,
+    allowNetwork,
+  });
+
+ 
+  const zone = result.predictiveClassification?.zone || result.classification?.zone || 'UNKNOWN';
+  const blocked = zone === 'RED' && !force;
+
+  return {
+    ...result,
+    verdict: zone,
+    blocked,
+    force,
+    message: zone === 'UNKNOWN'
+      ? 'Hardware VRAM unavailable — running without a feasibility guarantee.'
+      : result.predictiveClassification?.message || result.classification.message,
+  };
+}
+
+
+export async function resolveSystemHardwareTier(options = {}) {
+  const hardware = await getHardwareTarget();
+  if (!hardware.available) {
+    const tier0 = classifyHardwareTier({ totalMb: 4096, name: 'Fallback Constrained' });
+    return {
+      available: false,
+      hardware,
+      tier: tier0,
+      profile: null,
+      budget: calculateDynamicVramBudget({ totalVramMb: 4096, kvQuant: 'Q4_K_M' }),
+    };
+  }
+
+  const totalMb = hardware.totalBytes ? hardware.totalBytes / (1024 * 1024) : 6144;
+  const profile = await techPowerUpClient.resolveHardwareProfile({
+    name: hardware.name,
+    totalMb,
+    totalGb: (totalMb / 1024).toFixed(1),
+  }, options);
+
+  const tier = classifyHardwareTier({
+    totalMb,
+    name: profile.name || hardware.name,
+    vendor: hardware.vendor,
+  });
+
+  const budget = calculateDynamicVramBudget({
+    totalVramMb: totalMb,
+    modelWeightsMb: options.modelWeightsMb || 3900,
+    osSafetyMb: options.osSafetyMb || 700,
+    kvQuant: tier.effectiveKvQuant,
+  });
+
+  return {
+    available: true,
+    hardware,
+    profile,
+    tier,
+    budget,
+  };
+}