badgr-cli 1.0.48 → 1.1.0

package/src/gpuDoctor/doctor.js

@@ -0,0 +1,451 @@
+import fs from 'fs';
+import { detectGpus } from './gpuInfo.js';
+import { detectTorch } from './torchInfo.js';
+import { detectDisk } from './diskInfo.js';
+import { estimateModelFit } from './modelFit.js';
+import { classifyLog } from './logClassifier.js';
+import { diagnoseWorkflow, parseWorkflowFile } from './workflowDoctor.js';
+import { checkHealth } from './healthCheck.js';
+import { redactLine } from './redact.js';
+import { findCheapest, findByCanonical, GPU_CATALOG } from '../router.js';
+import { readProbeCache, writeProbeCache } from './probeCache.js';
+
+function shellQuote(value) {
+  return /^[a-z0-9_./:@-]+$/i.test(value) ? value : JSON.stringify(value);
+}
+
+function resolveNamedGpuVramGb(gpuName) {
+  if (!gpuName) return undefined;
+  const canonical = gpuName.toUpperCase().replace(/[\s-]+/g, '_');
+  const byCanonical = findByCanonical(canonical);
+  if (byCanonical) return byCanonical.vramGb;
+  const byName = GPU_CATALOG.find((g) => g.name.toUpperCase().replace(/[\s-]+/g, '_') === canonical);
+  return byName ? byName.vramGb : undefined;
+}
+
+function suggestGpuCanonical(minVramGb) {
+  const match = findCheapest({ minVramGb: minVramGb ?? 0, tag: 'inference' });
+  return match ? match.canonical : null;
+}
+
+function buildServeCommand(model, minVramGb, gpuCount = 1) {
+  const gpu = suggestGpuCanonical(minVramGb);
+  const parts = ['badgr serve', shellQuote(model || 'your-model')];
+  if (gpu) parts.push(`--gpu ${gpu}`);
+  if (gpuCount > 1) parts.push(`--count ${gpuCount}`);
+  parts.push('--max-cost 10');
+  return parts.join(' ');
+}
+
+/**
+ * Read-only, local system probe. Never installs, restarts, kills, or
+ * deletes anything — every check here is inspection only.
+ *
+ * The real nvidia-smi / python-torch probes are cached for a few seconds
+ * (see probeCache.js) so repeated `badgr doctor` runs don't re-pay that
+ * cost. Caching only applies to the real probes: a caller-injected
+ * deps.detectGpus/detectTorch (e.g. in tests) always runs directly and is
+ * never read from or written to the shared cache. Pass deps.noCache to
+ * force fresh probes (e.g. right after a driver change).
+ */
+export function collectEnvironment(deps = {}) {
+  const cacheOpts = { ttlMs: deps.cacheTtlMs, cachePath: deps.cachePath, fsImpl: deps.cacheFsImpl };
+  const useCache = !deps.noCache;
+
+  let gpu;
+  if (deps.detectGpus) {
+    gpu = deps.detectGpus(deps.execImpl);
+  } else {
+    gpu = useCache ? readProbeCache('gpu', cacheOpts) : undefined;
+    if (!gpu) {
+      gpu = detectGpus(deps.execImpl);
+      if (useCache) writeProbeCache('gpu', gpu, cacheOpts);
+    }
+  }
+
+  let torch;
+  if (deps.detectTorch) {
+    torch = deps.detectTorch(deps.execImpl);
+  } else {
+    torch = useCache ? readProbeCache('torch', cacheOpts) : undefined;
+    if (!torch) {
+      torch = detectTorch(deps.execImpl);
+      if (useCache) writeProbeCache('torch', torch, cacheOpts);
+    }
+  }
+
+  const disk = (deps.detectDisk || detectDisk)(deps.fsImpl);
+  return { gpu, torch, disk };
+}
+
+function localAvailableVramGb(env, gpuCount = 1) {
+  const gpus = env.gpu.gpus || [];
+  if (gpus.length === 0) return undefined;
+  if (gpuCount <= 1) {
+    return Math.max(...gpus.map((g) => g.vramFreeGb ?? g.vramTotalGb ?? 0));
+  }
+  // Tensor-parallel sizing: the group is bottlenecked by whichever of the
+  // requested GPUs has the least free VRAM.
+  const sorted = [...gpus].sort((a, b) => (b.vramFreeGb ?? b.vramTotalGb ?? 0) - (a.vramFreeGb ?? a.vramTotalGb ?? 0));
+  const usable = sorted.slice(0, gpuCount);
+  return Math.min(...usable.map((g) => g.vramFreeGb ?? g.vramTotalGb ?? 0));
+}
+
+function baselineVerdict(env) {
+  const evidence = [];
+  if (!env.gpu.available || env.gpu.gpus.length === 0) {
+    evidence.push('nvidia-smi: no NVIDIA GPU detected');
+    return {
+      verdict: 'no NVIDIA GPU detected',
+      verdictSlug: 'no_gpu_detected',
+      category: 'gpu_unavailable',
+      likelyCause: 'GPU unavailable to this process, or not passed into the container.',
+      evidence,
+      fixes: [
+        'Check that the host actually has an NVIDIA GPU',
+        'If running in Docker, pass --gpus all',
+        'Verify the NVIDIA Container Toolkit is installed',
+        'Or run on Badgr: badgr serve <model>',
+      ],
+      badgrCommand: 'badgr serve <model> --max-cost 10',
+    };
+  }
+
+  for (const g of env.gpu.gpus) {
+    evidence.push(`GPU ${g.index}: ${g.name}, ${g.vramTotalGb}GB VRAM (free: ${g.vramFreeGb}GB)`);
+  }
+  if (env.gpu.driverVersion) evidence.push(`Driver version: ${env.gpu.driverVersion}`);
+  if (env.gpu.cudaVersion) evidence.push(`CUDA version (nvidia-smi): ${env.gpu.cudaVersion}`);
+
+  if (!env.torch.torchInstalled) {
+    evidence.push('PyTorch: not installed');
+    return {
+      verdict: 'GPU visible but PyTorch not installed',
+      verdictSlug: 'torch_not_installed',
+      category: 'cuda_pytorch_mismatch',
+      likelyCause: 'The GPU is visible to the system, but no PyTorch install was found to confirm it is usable for GPU workloads.',
+      evidence,
+      fixes: ['Install a CUDA-enabled torch build matching the driver', 'Verify the correct Python environment is active'],
+      badgrCommand: 'badgr serve <model> --max-cost 10',
+    };
+  }
+
+  if (!env.torch.cudaAvailable) {
+    evidence.push(`PyTorch installed (${env.torch.pythonBinary}); torch.cuda.is_available() = False`);
+    return {
+      verdict: 'CUDA visible to system but not to PyTorch',
+      verdictSlug: 'cuda_not_visible_to_pytorch',
+      category: 'cuda_pytorch_mismatch',
+      likelyCause: 'CPU-only torch wheel, a mismatched CUDA runtime, or the container is missing GPU access.',
+      evidence,
+      fixes: [
+        'Reinstall torch with a CUDA-enabled build matching the driver',
+        'If containerized, verify --gpus all / GPU passthrough is set',
+        'Check that CUDA_VISIBLE_DEVICES is not empty or misconfigured',
+      ],
+      badgrCommand: 'badgr serve <model> --max-cost 10',
+    };
+  }
+
+  evidence.push(`PyTorch CUDA OK: ${env.torch.deviceName || 'GPU'} via torch ${env.torch.cudaVersion || ''}`.trim());
+  return {
+    verdict: 'environment looks healthy',
+    verdictSlug: 'environment_ok',
+    category: 'environment_ok',
+    likelyCause: 'GPU is visible to both the system and PyTorch. No environment-level blocker detected.',
+    evidence,
+    fixes: [],
+    badgrCommand: null,
+  };
+}
+
+function diskEvidence(disk) {
+  const lines = [`Model cache: ${disk.cachePath} (${disk.cacheSource})`];
+  lines.push(`Cache path exists: ${disk.cacheExists ? 'yes' : 'no'}; writable: ${disk.writable ? 'yes' : 'no'}`);
+  if (disk.freeGb != null) lines.push(`Free disk: ${disk.freeGb}GB`);
+  return lines;
+}
+
+function modelVerdict(model, env, options) {
+  const gpuCount = Math.max(1, Math.round(options.gpuCount) || 1);
+  const estimate = estimateModelFit(model, { contextLen: options.contextLen, gpuCount });
+  const namedGpuVramGb = resolveNamedGpuVramGb(options.gpu);
+  const gpuNameUnrecognized = Boolean(options.gpu) && namedGpuVramGb === undefined;
+  const availableVramGb = options.vramGb ?? namedGpuVramGb ?? localAvailableVramGb(env, gpuCount);
+  const perGpuLabel = gpuCount > 1 ? ' per GPU' : '';
+  const evidence = [`Model estimate: ${estimate.perGpuVramMinGb ?? '?'}–${estimate.perGpuVramMaxGb ?? '?'}GB${perGpuLabel} at current settings (${estimate.sizeClass})`];
+  if (gpuCount > 1) {
+    evidence.push(`GPU count: ${gpuCount} (tensor-parallel), ${estimate.vramMinGb ?? '?'}–${estimate.vramMaxGb ?? '?'}GB total across all GPUs`);
+    const physicalGpuCount = (env.gpu.gpus || []).length;
+    if (physicalGpuCount > 0 && physicalGpuCount < gpuCount) {
+      evidence.push(`Only ${physicalGpuCount} local GPU(s) detected — fewer than the ${gpuCount} requested`);
+    }
+  }
+  if (gpuNameUnrecognized) evidence.push(`GPU name "${options.gpu}" not recognized in the catalog`);
+  if (availableVramGb !== undefined) evidence.push(`Available VRAM: ${availableVramGb}GB${perGpuLabel}`);
+
+  if (estimate.vramMaxGb == null) {
+    return {
+      verdict: 'insufficient data — model size unknown',
+      verdictSlug: 'insufficient_data',
+      category: 'insufficient_data',
+      likelyCause: 'Could not infer a parameter count from the model name — no reliable estimate is possible.',
+      evidence,
+      fixes: ['Pass --vram-gb to size manually', 'Check the model card for parameter count'],
+      badgrCommand: buildServeCommand(model, null),
+      estimate,
+    };
+  }
+
+  if (gpuNameUnrecognized && availableVramGb === undefined) {
+    return {
+      verdict: 'insufficient data — GPU not recognized',
+      verdictSlug: 'insufficient_data',
+      category: 'insufficient_data',
+      likelyCause: `GPU name "${options.gpu}" was not recognized and no local GPU was detected — there isn't enough data for a reliable verdict.`,
+      evidence,
+      fixes: ['Pass --vram-gb <n> to size manually', `Recommended GPU (unverified estimate): ${estimate.recommendedGpuClass}`],
+      badgrCommand: buildServeCommand(model, estimate.perGpuVramMaxGb, gpuCount),
+      estimate,
+    };
+  }
+
+  if (availableVramGb === undefined) {
+    return {
+      verdict: 'no local GPU detected — showing model estimate only',
+      verdictSlug: 'no_gpu_detected',
+      category: 'gpu_unavailable',
+      likelyCause: 'No local GPU was detected to compare against the model estimate.',
+      evidence,
+      fixes: [estimate.quantSuggestion, `Recommended GPU: ${estimate.recommendedGpuClass}`],
+      badgrCommand: buildServeCommand(model, estimate.perGpuVramMaxGb, gpuCount),
+      estimate,
+    };
+  }
+
+  if (availableVramGb < estimate.perGpuVramMinGb) {
+    return {
+      verdict: 'likely fail',
+      verdictSlug: 'likely_fail',
+      category: 'model_too_large',
+      likelyCause: gpuCount > 1
+        ? 'Model is likely too large for the available VRAM at current settings, even split across the requested GPUs.'
+        : 'Model is likely too large for the available VRAM at current settings.',
+      evidence,
+      fixes: [
+        'Lower max model length / context',
+        'Reduce concurrency',
+        estimate.quantSuggestion,
+        gpuCount > 1 ? `Add more GPUs, or use a larger GPU class: ${estimate.recommendedGpuClass}` : `Recommended GPU: ${estimate.recommendedGpuClass}`,
+      ],
+      badgrCommand: buildServeCommand(model, estimate.perGpuVramMaxGb, gpuCount),
+      estimate,
+    };
+  }
+
+  if (availableVramGb < estimate.perGpuVramMaxGb) {
+    return {
+      verdict: 'likely okay with conservative settings',
+      verdictSlug: 'likely_ok',
+      category: 'model_tight_fit',
+      likelyCause: 'VRAM / KV cache is a close fit for this model and context length.',
+      evidence,
+      fixes: ['Lower max model length if you see OOM errors', 'Reduce concurrency for headroom'],
+      badgrCommand: buildServeCommand(model, estimate.perGpuVramMaxGb, gpuCount),
+      estimate,
+    };
+  }
+
+  return {
+    verdict: 'likely okay with conservative settings',
+    verdictSlug: 'likely_ok',
+    category: 'environment_ok',
+    likelyCause: 'Estimated model VRAM fits comfortably within the available VRAM.',
+    evidence,
+    fixes: ['Lower max model length if you see OOM errors at runtime'],
+    badgrCommand: buildServeCommand(model, estimate.perGpuVramMaxGb, gpuCount),
+    estimate,
+  };
+}
+
+function logVerdict(logText) {
+  const result = classifyLog(logText);
+  const evidence = result.evidenceLines.map((l) => `Logs contain: ${l}`);
+  if (!result.matched) {
+    return {
+      verdict: 'inconclusive — could not classify this log',
+      verdictSlug: 'unknown',
+      category: 'unknown',
+      likelyCause: 'No recognized failure signature was found in the provided log.',
+      evidence,
+      fixes: result.fixes,
+      badgrCommand: null,
+      logResult: result,
+    };
+  }
+  return {
+    verdict: 'likely fail',
+    verdictSlug: 'likely_fail',
+    category: result.category,
+    likelyCause: result.label,
+    evidence,
+    fixes: result.fixes,
+    badgrCommand: null,
+    logResult: result,
+  };
+}
+
+function workflowVerdict(workflow) {
+  const result = diagnoseWorkflow(workflow);
+  const evidence = [
+    `Detected workflow: likely ${result.likelyClass}`,
+    `Models referenced: ${result.models.length ? result.models.join(', ') : 'none found'}`,
+    `Possible custom nodes: ${result.customNodes.length ? result.customNodes.join(', ') : 'none found'}`,
+    `Likely VRAM: ${result.vramBucket}`,
+  ];
+  return {
+    verdict: result.nodeCount === 0 ? 'inconclusive — could not parse workflow nodes' : 'likely okay with conservative settings',
+    verdictSlug: result.nodeCount === 0 ? 'unknown' : 'likely_ok',
+    category: 'comfy_workflow',
+    likelyCause: `ComfyUI workflow classified as ${result.likelyClass}; verify referenced models and custom nodes are installed.`,
+    evidence,
+    fixes: [
+      result.models.length ? 'Verify each referenced model file exists locally' : 'No model filenames detected — confirm nodes reference valid checkpoints',
+      result.customNodes.length ? 'Verify custom nodes are installed before running' : 'No non-core nodes detected',
+      `Recommended GPU class: ${result.vramBucket}`,
+    ],
+    badgrCommand: 'badgr comfyui batch --workflow <workflow.json> --max-cost 5',
+    workflowResult: result,
+  };
+}
+
+function healthVerdict(healthResult, url) {
+  const evidence = [`URL: ${redactLine(url)}`, `Latency: ${healthResult.latencyMs}ms`];
+  if (healthResult.status != null) evidence.push(`HTTP status: ${healthResult.status}`);
+  if (healthResult.modelsCount != null) evidence.push(`/v1/models reports ${healthResult.modelsCount} model(s)`);
+  if (healthResult.detail) evidence.push(healthResult.detail);
+
+  if (!healthResult.reachable) {
+    const causeMap = {
+      'connection refused': 'Server not started, or the port is not exposed.',
+      timeout: 'Server did not respond in time — health check may be too early, or the endpoint is hung.',
+    };
+    return {
+      verdict: 'endpoint not reachable',
+      verdictSlug: 'endpoint_not_reachable',
+      category: 'endpoint_health',
+      likelyCause: causeMap[healthResult.classification] || 'Server not started, port not exposed, or health check too early.',
+      evidence: [...evidence, `Classification: ${healthResult.classification}`],
+      fixes: ['Confirm the process is running', 'Confirm the port is exposed / forwarded', 'Retry after allowing more startup time'],
+      badgrCommand: null,
+    };
+  }
+
+  if (!healthResult.ready) {
+    return {
+      verdict: 'endpoint unhealthy',
+      verdictSlug: 'endpoint_unhealthy',
+      category: 'endpoint_health',
+      likelyCause: `Process is responding but returned HTTP ${healthResult.status}.`,
+      evidence,
+      fixes: ['Check server logs for the underlying error', 'Confirm the model finished loading'],
+      badgrCommand: null,
+    };
+  }
+
+  return {
+    verdict: 'endpoint ready',
+    verdictSlug: 'endpoint_ready',
+    category: 'endpoint_health',
+    likelyCause: 'Endpoint responded successfully.',
+    evidence,
+    fixes: [],
+    badgrCommand: null,
+  };
+}
+
+// Priority when multiple sources are given: logs > url > workflow > model.
+// Listed in that order so provided[0] (if any) is always the selected source.
+function describeSources(options) {
+  const provided = [];
+  if (options.logsPath) provided.push(`--logs ${options.logsPath}`);
+  if (options.url) provided.push(`--url ${redactLine(options.url)}`);
+  if (options.workflowPath) provided.push(`--workflow ${options.workflowPath}`);
+  if (options.model) provided.push(`--model ${options.model}`);
+
+  if (provided.length === 0) {
+    return ['Diagnosing from: baseline environment check (no --logs/--url/--workflow/--model given)'];
+  }
+  const notes = [`Diagnosing from: ${provided[0]}`];
+  if (provided.length > 1) {
+    notes.push(`Ignored (lower priority than the source above): ${provided.slice(1).join(', ')}`);
+  }
+  return notes;
+}
+
+/**
+ * Orchestrates the full read-only diagnosis. Priority when multiple
+ * sources are given: logs > url > workflow > model > baseline environment.
+ */
+export async function runGpuDoctor(options = {}, deps = {}) {
+  const env = collectEnvironment(deps);
+  const baseline = baselineVerdict(env);
+  const evidence = [...describeSources(options), ...baseline.evidence, ...diskEvidence(env.disk)];
+
+  let outcome = null;
+
+  if (options.logsPath) {
+    const readFile = deps.readFileSync || fs.readFileSync;
+    let logText;
+    try {
+      logText = readFile(options.logsPath, 'utf8');
+    } catch (err) {
+      const wrapped = new Error(`Could not read log file: ${options.logsPath} (${err.message})`);
+      wrapped.isToolError = true;
+      throw wrapped;
+    }
+    outcome = logVerdict(logText);
+  } else if (options.url) {
+    const fetchImpl = deps.fetchImpl;
+    const healthResult = await checkHealth(options.url, fetchImpl ? { fetchImpl } : {});
+    outcome = healthVerdict(healthResult, options.url);
+  } else if (options.workflowPath) {
+    const readFile = deps.readFileSync || fs.readFileSync;
+    let raw;
+    try {
+      raw = readFile(options.workflowPath, 'utf8');
+    } catch (err) {
+      const wrapped = new Error(`Could not read workflow file: ${options.workflowPath} (${err.message})`);
+      wrapped.isToolError = true;
+      throw wrapped;
+    }
+    let workflow;
+    try {
+      workflow = parseWorkflowFile(raw);
+    } catch (err) {
+      const wrapped = new Error(`Could not parse workflow JSON: ${options.workflowPath} (${err.message})`);
+      wrapped.isToolError = true;
+      throw wrapped;
+    }
+    outcome = workflowVerdict(workflow);
+  } else if (options.model) {
+    outcome = modelVerdict(options.model, env, options);
+  } else {
+    outcome = baseline;
+  }
+
+  const combinedEvidence = outcome === baseline ? evidence : [...evidence, ...outcome.evidence];
+
+  const report = {
+    verdict: outcome.verdict,
+    verdictSlug: outcome.verdictSlug,
+    category: outcome.category,
+    likelyCause: outcome.likelyCause,
+    evidence: combinedEvidence,
+    suggestedFixes: (outcome.fixes || []).filter(Boolean),
+    badgrCommand: outcome.badgrCommand || null,
+    environment: env,
+  };
+
+  return report;
+}

package/src/gpuDoctor/gpuInfo.js

@@ -0,0 +1,70 @@
+import { execFileSync } from 'child_process';
+
+function round1(n) {
+  return Math.round(n * 10) / 10;
+}
+
+/**
+ * Read-only nvidia-smi probe. Never mutates the machine.
+ */
+export function detectGpus(execImpl = execFileSync) {
+  const result = {
+    available: false,
+    error: null,
+    driverVersion: null,
+    cudaVersion: null,
+    gpus: [],
+    processes: [],
+  };
+
+  let smiOut;
+  try {
+    smiOut = execImpl('nvidia-smi', [], { encoding: 'utf8', timeout: 5000 });
+  } catch (err) {
+    result.error = err && err.code === 'ENOENT' ? 'nvidia-smi not found' : (err && err.message) || 'nvidia-smi failed';
+    return result;
+  }
+
+  result.available = true;
+
+  const driverMatch = smiOut.match(/Driver Version:\s*([\d.]+)/);
+  if (driverMatch) result.driverVersion = driverMatch[1];
+  const cudaMatch = smiOut.match(/CUDA Version:\s*([\d.]+)/);
+  if (cudaMatch) result.cudaVersion = cudaMatch[1];
+
+  try {
+    const csv = execImpl('nvidia-smi', [
+      '--query-gpu=index,name,memory.total,memory.free,memory.used,temperature.gpu,power.draw',
+      '--format=csv,noheader,nounits',
+    ], { encoding: 'utf8', timeout: 5000 });
+    result.gpus = csv.trim().split(/\r?\n/).filter(Boolean).map((line) => {
+      const [index, name, total, free, used, temp, power] = line.split(',').map((s) => s.trim());
+      return {
+        index: Number(index),
+        name,
+        vramTotalGb: round1(Number(total) / 1024),
+        vramFreeGb: round1(Number(free) / 1024),
+        vramUsedGb: round1(Number(used) / 1024),
+        temperatureC: temp && temp !== '[N/A]' ? Number(temp) : null,
+        powerDrawW: power && power !== '[N/A]' ? Number(power) : null,
+      };
+    });
+  } catch {
+    // csv query unsupported on this driver — plain-text probe above still stands
+  }
+
+  try {
+    const procCsv = execImpl('nvidia-smi', [
+      '--query-compute-apps=pid,process_name,used_memory',
+      '--format=csv,noheader,nounits',
+    ], { encoding: 'utf8', timeout: 5000 });
+    result.processes = procCsv.trim().split(/\r?\n/).filter(Boolean).map((line) => {
+      const [pid, name, mem] = line.split(',').map((s) => s.trim());
+      return { pid: Number(pid), name, memoryMb: Number(mem) };
+    });
+  } catch {
+    // no compute-apps support or no processes — leave empty
+  }
+
+  return result;
+}

package/src/gpuDoctor/healthCheck.js

@@ -0,0 +1,63 @@
+// Read-only HTTP health probe. A single GET request — never mutates,
+// restarts, or otherwise touches the remote process.
+
+export async function checkHealth(url, { fetchImpl = fetch, timeoutMs = 5000 } = {}) {
+  const start = Date.now();
+  let res;
+  try {
+    res = await fetchImpl(url, { signal: AbortSignal.timeout(timeoutMs) });
+  } catch (err) {
+    const latencyMs = Date.now() - start;
+    if (err && (err.name === 'TimeoutError' || err.name === 'AbortError')) {
+      return { reachable: false, ready: false, status: null, latencyMs, error: 'timeout', classification: 'timeout' };
+    }
+    const message = (err && err.message) || String(err);
+    if (/ECONNREFUSED/.test(message) || (err && err.cause && err.cause.code === 'ECONNREFUSED')) {
+      return { reachable: false, ready: false, status: null, latencyMs, error: 'connection refused', classification: 'connection refused' };
+    }
+    return { reachable: false, ready: false, status: null, latencyMs, error: message, classification: 'connection error' };
+  }
+
+  const latencyMs = Date.now() - start;
+  let modelsCount = null;
+  let serviceKind = null;
+  let detail = null;
+  try {
+    const body = await res.clone().json();
+    if (body && Array.isArray(body.data)) {
+      // OpenAI-compatible /v1/models
+      modelsCount = body.data.length;
+      serviceKind = 'openai';
+    } else if (body && Array.isArray(body.devices)) {
+      // ComfyUI /system_stats
+      serviceKind = 'comfyui';
+      detail = `ComfyUI system stats — ${body.devices.length} device(s) reported`;
+    } else if (body && typeof body.status === 'string') {
+      // llama.cpp /health, and similar {status: "..."} health endpoints
+      serviceKind = 'llamacpp';
+      detail = `status: ${body.status}`;
+    } else if (body && typeof body.healthy === 'boolean') {
+      serviceKind = 'generic';
+      detail = `healthy: ${body.healthy}`;
+    } else if (body && typeof body.ok === 'boolean') {
+      serviceKind = 'generic';
+      detail = `ok: ${body.ok}`;
+    }
+  } catch {
+    // non-JSON body — check for a bare text status before giving up
+    try {
+      const text = (await res.clone().text()).trim();
+      if (text && /^(ok|healthy|ready|up)$/i.test(text)) {
+        serviceKind = 'generic';
+        detail = `response: ${text}`;
+      }
+    } catch {
+      // empty or unreadable body — fine, this is a best-effort read
+    }
+  }
+
+  if (res.ok) {
+    return { reachable: true, ready: true, status: res.status, latencyMs, modelsCount, serviceKind, detail, classification: 'endpoint ready' };
+  }
+  return { reachable: true, ready: false, status: res.status, latencyMs, modelsCount, serviceKind, detail, classification: 'endpoint unhealthy' };
+}