llm-tuner 0.1.0

package/src/core/engines/ollama.js

@@ -0,0 +1,335 @@
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { run, which, exists, fetchJson, freePort, spawnLogged, killProcess, sleep, PLATFORM, homeDir, timestamp } from '../util.js';
+import { modelMetaFromFile } from '../gguf.js';
+import { runApiBenchmark } from '../benchmark.js';
+import { TunerError, benchError } from '../errors.js';
+
+const DEFAULT_HOST = '127.0.0.1:11434';
+
+function baseUrl() {
+  let h = process.env.OLLAMA_HOST || DEFAULT_HOST;
+  if (!/^https?:\/\//.test(h)) h = `http://${h}`;
+  return h.replace('0.0.0.0', '127.0.0.1').replace(/\/$/, '');
+}
+
+async function ollamaBin() {
+  const extra = PLATFORM === 'win32'
+    ? [path.join(process.env.LOCALAPPDATA || '', 'Programs', 'Ollama', 'ollama.exe')]
+    : PLATFORM === 'darwin' ? ['/Applications/Ollama.app/Contents/Resources/ollama', '/usr/local/bin/ollama', '/opt/homebrew/bin/ollama'] : ['/usr/local/bin/ollama', '/usr/bin/ollama'];
+  return which('ollama', extra);
+}
+
+async function hasSystemdService() {
+  if (PLATFORM !== 'linux') return false;
+  const r = await run('systemctl', ['cat', 'ollama.service']);
+  return r.code === 0;
+}
+
+/** Where the models live (needed to start a private benchmark server). */
+async function modelsDir() {
+  if (process.env.OLLAMA_MODELS) return process.env.OLLAMA_MODELS;
+  const candidates = [path.join(homeDir(), '.ollama', 'models')];
+  if (PLATFORM === 'linux') candidates.unshift('/usr/share/ollama/.ollama/models', '/var/lib/ollama/.ollama/models');
+  for (const c of candidates) {
+    try { await fs.access(path.join(c, 'manifests'), fs.constants.R_OK); return c; } catch { /* next */ }
+  }
+  return null;
+}
+
+// Synthesize estimator metadata from /api/show when the GGUF blob isn't readable.
+function metaFromModelInfo(info, sizeBytes) {
+  const arch = info['general.architecture'];
+  const g = (k) => info[`${arch}.${k}`];
+  const nLayers = g('block_count');
+  const nEmbd = g('embedding_length');
+  const nHead = g('attention.head_count');
+  const nHeadKv = g('attention.head_count_kv') ?? nHead;
+  const keyLen = g('attention.key_length') ?? nEmbd / nHead;
+  const vocab = info['tokenizer.ggml.tokens']?.length ?? 32000;
+  const outputBytes = Math.round(sizeBytes * 0.03);
+  const per = (sizeBytes - outputBytes) / nLayers;
+  const nExperts = g('expert_count') || 0;
+  return {
+    arch, nLayers, nEmbd, nHead, nHeadKv, keyLen, valLen: g('attention.value_length') ?? keyLen, vocab,
+    trainContext: g('context_length'), fileBytes: sizeBytes, layerBytes: Array(nLayers).fill(per),
+    layerExpertBytes: Array(nLayers).fill(nExperts ? per * 0.9 : 0), outputBytes, embdBytes: outputBytes,
+    kvHeadsPerLayer: Array(nLayers).fill(Array.isArray(nHeadKv) ? Math.max(...nHeadKv) : nHeadKv),
+    slidingWindow: g('attention.sliding_window') || null, swaLayers: [],
+    nExperts, nExpertsUsed: g('expert_used_count') || 0, isMoE: nExperts > 1, approximate: true,
+  };
+}
+
+/** Tag of an Ollama model name: the text after the first colon, `latest` when there is none. */
+const tagOf = (name) => {
+  const i = name.indexOf(':');
+  return i === -1 ? 'latest' : name.slice(i + 1);
+};
+
+/** Tag of a model name without its quantization level, lower-cased: what tells models apart. */
+export function tagStem(name, quant) {
+  const parts = tagOf(name).toLowerCase().split('-');
+  if (quant) {
+    const target = String(quant).toLowerCase();
+    const lastIdx = parts.lastIndexOf(target);
+    if (lastIdx !== -1) {
+      parts.splice(lastIdx, 1);
+    }
+  }
+  return parts.join('-');
+}
+
+/** Tags of the same model as `key` in an /api/tags list: [{ key, quant, sizeBytes, selected }]. */
+export function ollamaVariants(models, key) {
+  const isTuned = (name) => tagOf(name).includes('-tuned-');
+  // A tuned copy is not a base model: callers must pass the base key.
+  if (isTuned(key)) return [];
+  const named = models.filter((m) => typeof m?.name === 'string');
+  const self = named.find((m) => m.name === key);
+  if (!self) return [];
+  const toVariant = (m) => ({
+    key: m.name,
+    quant: m.details?.quantization_level ?? null,
+    sizeBytes: m.size,
+    selected: m.name === key,
+  });
+  const size = self.details?.parameter_size;
+  if (!size) return [toVariant(self)];
+  const base = key.split(':')[0];
+  const family = self.details?.family;
+  const stem = tagStem(key, self.details?.quantization_level);
+  // Deliberately conservative: two tags are the same model only when base name, parameter size,
+  // family and tag stem all match. A default tag such as `14b` is not grouped with
+  // `14b-instruct-q8_0`, because the names cannot prove they are the same model.
+  const group = named.filter((m) =>
+    m.name.split(':')[0] === base &&
+    !isTuned(m.name) &&
+    m.details?.parameter_size === size &&
+    m.details?.family === family &&
+    tagStem(m.name, m.details?.quantization_level) === stem
+  );
+  // Alias tags share a digest: keep one of them, preferring the requested key.
+  return group.filter((m, i) =>
+    !m.digest ||
+    m.name === key ||
+    (m.digest !== self.digest && group.findIndex((x) => x.digest === m.digest) === i)
+  ).map(toVariant);
+}
+
+export const ollama = {
+  id: 'ollama',
+  name: 'Ollama',
+  // KV type and flash attention are server-wide settings in Ollama; GPU order via CUDA_VISIBLE_DEVICES.
+  capabilities: { kvTypes: ['f16', 'q8_0', 'q4_0'], perModelKv: false, gpuOrder: true, cpuMoe: false, exactBenchmark: true, variantSelect: true },
+
+  async detect() {
+    const bin = await ollamaBin();
+    const url = baseUrl();
+    const v = await fetchJson(`${url}/api/version`, { timeout: 1500 });
+    let version = v?.json?.version || null;
+    if (!version && bin) version = (await run(bin, ['--version'])).stdout.match(/\d+\.\d+\.\d+/)?.[0] || null;
+    return { installed: !!bin || !!v?.ok, bin, version, serverRunning: !!v?.ok, apiUrl: url, systemd: await hasSystemdService(), modelsDir: await modelsDir() };
+  },
+
+  async ensureServer(ctx) {
+    if ((await fetchJson(`${ctx.detection.apiUrl}/api/version`, { timeout: 1500 }))?.ok) return true;
+    if (!ctx.detection.bin) return false;
+    const { spawn } = await import('node:child_process');
+    spawn(ctx.detection.bin, ['serve'], { detached: true, stdio: 'ignore' }).unref();
+    for (let i = 0; i < 20; i++) {
+      await sleep(500);
+      if ((await fetchJson(`${ctx.detection.apiUrl}/api/version`, { timeout: 1000 }))?.ok) return true;
+    }
+    return false;
+  },
+
+  async listModels(ctx) {
+    await this.ensureServer(ctx);
+    const r = await fetchJson(`${ctx.detection.apiUrl}/api/tags`);
+    return (r?.json?.models || [])
+      .filter((m) => !/embed/i.test(m.name) && !/bert/i.test(m.details?.family || ''))
+      .map((m) => ({ key: m.name, name: m.name, sizeBytes: m.size, arch: m.details?.family, quant: m.details?.quantization_level, maxContext: null }));
+  },
+
+  /** Downloaded variants (tags) of a model: [{ key, quant, sizeBytes, selected }]. */
+  async listVariants(ctx, key) {
+    await this.ensureServer(ctx);
+    const r = await fetchJson(`${ctx.detection.apiUrl}/api/tags`);
+    return ollamaVariants(r?.json?.models || [], key);
+  },
+
+  async modelMeta(ctx, key) {
+    const r = await fetchJson(`${ctx.detection.apiUrl}/api/show`, { method: 'POST', body: { model: key, verbose: true }, timeout: 30000 });
+    if (!r?.ok) throw new TunerError('errors.ollamaUnknownModel', { key });
+    const from = r.json.modelfile?.match(/^FROM\s+(.+)$/m)?.[1]?.trim();
+    if (from && (await exists(from))) {
+      try { return { file: from, ...(await modelMetaFromFile(from)) }; } catch { /* unreadable blob */ }
+    }
+    const list = await this.listModels(ctx);
+    const size = list.find((m) => m.key === key)?.sizeBytes || 0;
+    return { file: null, ...metaFromModelInfo(r.json.model_info || {}, size) };
+  },
+
+  async unloadAll(url) {
+    const ps = await fetchJson(`${url}/api/ps`);
+    for (const m of ps?.json?.models || []) {
+      await fetchJson(`${url}/api/generate`, { method: 'POST', body: { model: m.name, keep_alive: 0 }, timeout: 30000 });
+    }
+  },
+
+  async prepare(ctx) {
+    await this.unloadAll(ctx.detection.apiUrl);
+    await sleep(1500);
+  },
+
+  serverEnv(c) {
+    const env = {
+      OLLAMA_FLASH_ATTENTION: c.flashAttention ? '1' : '0',
+      OLLAMA_KV_CACHE_TYPE: c.kvType,
+      OLLAMA_NUM_PARALLEL: String(c.parallel),
+    };
+    if (c.gpuOrder?.length) {
+      env.CUDA_DEVICE_ORDER = 'PCI_BUS_ID';
+      env.CUDA_VISIBLE_DEVICES = c.gpuOrder.join(',');
+    }
+    return env;
+  },
+
+  requestOptions(c) {
+    return { num_ctx: c.ctx, num_gpu: c.fullOffload ? 999 : c.gpuLayers, num_thread: c.threads, num_batch: c.ubatch };
+  },
+
+  /**
+   * Benchmark in a private `ollama serve` on another port so KV type / flash
+   * attention can change per candidate without touching the user's server.
+   */
+  async benchmark(ctx, model, candidate, opts) {
+    const dir = ctx.detection.modelsDir;
+    if (!dir || !ctx.detection.bin) {
+      // Can't read the models: measure on the main server (its KV type/FA stay as configured).
+      const res = await runApiBenchmark({ kind: 'ollama', baseUrl: ctx.detection.apiUrl, model: model.key, ollamaOptions: this.requestOptions(candidate), ...opts });
+      await this.unloadAll(ctx.detection.apiUrl);
+      return { ...res, noteCode: 'bench.mainServerNote' };
+    }
+    const port = await freePort();
+    const url = `http://127.0.0.1:${port}`;
+    const env = { ...process.env, ...this.serverEnv(candidate), OLLAMA_HOST: `127.0.0.1:${port}`, OLLAMA_MODELS: dir, OLLAMA_MAX_LOADED_MODELS: '1', OLLAMA_KEEP_ALIVE: '10m' };
+    const proc = spawnLogged(ctx.detection.bin, ['serve'], { env });
+    try {
+      let up = false;
+      for (let i = 0; i < 40; i++) {
+        if ((await fetchJson(`${url}/api/version`, { timeout: 1000 }))?.ok) { up = true; break; }
+        await sleep(500);
+      }
+      if (!up) return benchError('bench.ollamaTestServerFailed', { detail: proc.lines.slice(-2).join(' | ') });
+      const res = await runApiBenchmark({ kind: 'ollama', baseUrl: url, model: model.key, ollamaOptions: this.requestOptions(candidate), ...opts });
+      // How much really landed on GPU (Ollama may silently fall back to CPU).
+      const ps = await fetchJson(`${url}/api/ps`);
+      const loaded = ps?.json?.models?.[0];
+      if (loaded) res.offload = { sizeBytes: loaded.size, vramBytes: loaded.size_vram, full: loaded.size_vram >= loaded.size * 0.99 };
+      if (/out of memory|cudaMalloc failed/i.test(proc.lines.join('\n'))) Object.assign(res, benchError('bench.oom'), { oom: true });
+      return res;
+    } finally {
+      await killProcess(proc.child);
+      await sleep(1000);
+    }
+  },
+
+  tunedName(key, c) {
+    const [base, tag = 'latest'] = key.split(':');
+    return `${base}:${tag}-tuned-${Math.round(c.ctx / 1024)}k`;
+  },
+
+  modelfile(key, c) {
+    const o = this.requestOptions(c);
+    return [`FROM ${key}`, ...Object.entries(o).map(([k, v]) => `PARAMETER ${k} ${v}`), ''].join('\n');
+  },
+
+  /**
+   * 1) Create a tuned model variant with per-model parameters (num_ctx, num_gpu...).
+   * 2) Set server-wide env (KV type, flash attention, GPU order) for the Ollama service.
+   */
+  async apply(ctx, model, c, { dryRun = false, useSudo = false, skipIfApplied = false } = {}) {
+    const name = this.tunedName(model.key, c);
+    const env = this.serverEnv(c);
+    const result = { tunedModel: name, modelfile: this.modelfile(model.key, c), env, changes: [], pendingCommands: [] };
+    if (dryRun) return { dryRun: true, ...result };
+
+    const dir = path.join(os.tmpdir(), `llm-tuner-${timestamp()}`);
+    await fs.mkdir(dir, { recursive: true });
+    if (skipIfApplied && (await this.isApplied(ctx, model, c))) {
+      result.changes.push({ code: 'changes.ollamaExists', params: { name } });
+      return result;
+    }
+    const mf = path.join(dir, 'Modelfile');
+    await fs.writeFile(mf, result.modelfile);
+    const r = await run(ctx.detection.bin, ['create', name, '-f', mf], { timeout: 300_000 });
+    if (r.code !== 0) throw new Error(`ollama create failed: ${(r.stderr || r.stdout).slice(-300)}`);
+    result.changes.push({ code: 'changes.ollamaCreated', params: { name } });
+
+    if (PLATFORM === 'win32') {
+      for (const [k, v] of Object.entries(env)) await run('setx', [k, v]);
+      result.changes.push({ code: 'changes.ollamaSetx' });
+      await run('taskkill', ['/F', '/IM', 'ollama app.exe']);
+      await run('taskkill', ['/F', '/IM', 'ollama.exe']);
+      const app = path.join(process.env.LOCALAPPDATA || '', 'Programs', 'Ollama', 'ollama app.exe');
+      if (await exists(app)) { const { spawn } = await import('node:child_process'); spawn(app, [], { detached: true, stdio: 'ignore' }).unref(); }
+    } else if (PLATFORM === 'darwin') {
+      for (const [k, v] of Object.entries(env)) await run('launchctl', ['setenv', k, v]);
+      result.changes.push({ code: 'changes.ollamaLaunchctl' });
+      await run('osascript', ['-e', 'quit app "Ollama"']);
+      await sleep(2000);
+      await run('open', ['-a', 'Ollama']);
+    } else if (ctx.detection.systemd) {
+      const conf = ['[Service]', ...Object.entries(env).map(([k, v]) => `Environment="${k}=${v}"`), ''].join('\n');
+      const confFile = path.join(dir, 'llm-tuner.conf');
+      await fs.writeFile(confFile, conf);
+      const cmds = [
+        'sudo mkdir -p /etc/systemd/system/ollama.service.d',
+        `sudo cp ${confFile} /etc/systemd/system/ollama.service.d/llm-tuner.conf`,
+        'sudo systemctl daemon-reload',
+        'sudo systemctl restart ollama',
+      ];
+      if (useSudo) {
+        const ok = (await run('sudo', ['-n', 'true'])).code === 0;
+        if (ok) {
+          for (const cmd of cmds) { const [, ...rest] = cmd.split(' '); await run('sudo', ['-n', ...rest]); }
+          result.changes.push({ code: 'changes.ollamaSystemd' });
+        } else result.pendingCommands.push(...cmds);
+      } else result.pendingCommands.push(...cmds);
+    } else {
+      result.pendingCommands.push(...Object.entries(env).map(([k, v]) => `export ${k}=${v}`), 'ollama serve');
+    }
+    return result;
+  },
+
+  async isApplied(ctx, model, c) {
+    const r = await fetchJson(`${ctx.detection.apiUrl}/api/show`, { method: 'POST', body: { model: this.tunedName(model.key, c) }, timeout: 10000 });
+    return !!r?.ok;
+  },
+
+  /** Load the tuned variant (kept in memory 30 min) and optionally measure it. */
+  async load(ctx, model, { candidate, depthTokens = 0, genTokens = 200, onProgress } = {}) {
+    await this.ensureServer(ctx);
+    await this.unloadAll(ctx.detection.apiUrl);
+    const name = this.tunedName(model.key, candidate);
+    onProgress?.({ phase: 'load' });
+    const warm = await fetchJson(`${ctx.detection.apiUrl}/api/generate`, { method: 'POST', body: { model: name, keep_alive: '30m' }, timeout: 600_000 });
+    if (!warm?.ok) return warm?.json?.error ? { ok: false, error: warm.json.error } : benchError('bench.loadFailed');
+    const res = await runApiBenchmark({ kind: 'ollama', baseUrl: ctx.detection.apiUrl, model: name, depthTokens, genTokens, onProgress, ollamaOptions: {} });
+    return { ...res, model: name };
+  },
+
+  async chat(ctx, model, messages) {
+    const url = ctx.detection?.apiUrl || baseUrl();
+    const r = await fetchJson(`${url}/api/chat`, {
+      method: 'POST',
+      body: { model, messages, stream: false },
+      timeout: 120_000,
+    });
+    if (!r?.ok) throw new Error(r?.json?.error || 'Failed to chat with Ollama');
+    return { message: r.json.message };
+  },
+};
+

package/src/core/errors.js

@@ -0,0 +1,16 @@
+import { t } from '../i18n/index.js';
+
+/** A user-facing error: interfaces translate `code`; `message` is English for logs. */
+export class TunerError extends Error {
+  constructor(code, params = {}) {
+    super(t('en', code, params));
+    this.name = 'TunerError';
+    this.code = code;
+    this.params = params;
+  }
+}
+
+/** Failed benchmark/load result with a translatable code. */
+export function benchError(code, params = {}) {
+  return { ok: false, errorCode: code, errorParams: params, error: t('en', code, params) };
+}

package/src/core/estimator.js

@@ -0,0 +1,237 @@
+import { MiB } from './util.js';
+
+// Bytes per element of each KV cache type (ggml block sizes).
+export const KV_TYPES = {
+  // readCost: relative cost of reading the cache during decode. Q4_0 dequantization
+  // in flash attention is much slower in practice (measured ~12x on RTX 50xx).
+  f16: { bytes: 2, quality: 1.0, readCost: 1, label: 'F16 (maximum quality)' },
+  q8_0: { bytes: 34 / 32, quality: 0.99, readCost: 1, label: 'Q8_0 (almost lossless)' },
+  q4_0: { bytes: 18 / 32, quality: 0.93, readCost: 12, label: 'Q4_0 (saves memory, loses precision)' },
+};
+
+// Calibrated on RTX 5070 x2 / Qwen2.5-Coder-32B: CUDA context + scratch per GPU.
+// nvidia-smi "free" already excludes the driver's own reserve, so only a small margin.
+const PER_GPU_OVERHEAD = 350 * MiB;
+const PER_GPU_RESERVE = 64 * MiB;
+// Estimates within this distance of fitting are still tried as a full offload.
+const PROBE_MARGIN = 1024 * MiB;
+const COMPUTE_BYTES_PER_CTX_TOKEN = 3.5 * 1024;
+// Fraction of theoretical bandwidth llama.cpp typically achieves in decode.
+const DECODE_EFFICIENCY = 0.7;
+
+/** KV cache bytes for one layer at a given context length. */
+export function kvLayerBytes(meta, layer, ctx, kvType) {
+  const heads = meta.kvHeadsPerLayer[layer];
+  if (!heads) return 0;
+  const tokens = meta.slidingWindow && meta.swaLayers.includes(layer) ? Math.min(ctx, meta.slidingWindow) : ctx;
+  const b = KV_TYPES[kvType].bytes;
+  return tokens * heads * (meta.keyLen * b + meta.valLen * b);
+}
+
+export function kvTotalBytes(meta, ctx, kvType) {
+  let total = 0;
+  for (let i = 0; i < meta.nLayers; i++) total += kvLayerBytes(meta, i, ctx, kvType);
+  return total;
+}
+
+/** Memory each GPU can give to the model, after what's already in use. */
+export function gpuBudgets(gpus) {
+  return gpus.map((g) => ({
+    index: g.index,
+    name: g.name,
+    budget: Math.max(0, g.freeBytes - PER_GPU_OVERHEAD - PER_GPU_RESERVE),
+    bandwidthGBps: g.bandwidthGBps,
+  }));
+}
+
+/**
+ * Pack layers onto GPUs in priority order (llama.cpp split-mode "layer" keeps
+ * layers contiguous per device; the output layer lands on the last device).
+ * MoE: when `cpuMoeLayers` > 0, the expert tensors of the first N layers stay
+ * in system RAM (llama.cpp --n-cpu-moe), which frees VRAM cheaply.
+ */
+export function placeLayers(meta, gpus, { ctx, kvType, ubatch = 512, cpuMoeLayers = 0 }) {
+  const budgets = gpuBudgets(gpus);
+  const logitsBytes = meta.vocab * ubatch * 4;
+  const computePerGpu = ctx * COMPUTE_BYTES_PER_CTX_TOKEN / Math.max(1, budgets.length);
+  let used = budgets.map(() => computePerGpu);
+  let layersPerGpu = budgets.map(() => 0);
+  const layerCost = (i) => meta.layerBytes[i] - (i < cpuMoeLayers ? meta.layerExpertBytes[i] : 0) + kvLayerBytes(meta, i, ctx, kvType);
+
+  let g = 0;
+  let placed = 0;
+  // Layers that do not fit stay on CPU; llama.cpp offloads the *last* n layers,
+  // so we count how many fit and treat the remaining first layers as CPU layers.
+  const order = [...Array(meta.nLayers).keys()].reverse();
+  const fits = [];
+  for (const i of order) {
+    const cost = layerCost(i);
+    while (g < budgets.length && used[g] + cost > budgets[g].budget) g++;
+    if (g >= budgets.length) break;
+    used[g] += cost;
+    layersPerGpu[g]++;
+    fits.push(i);
+    placed++;
+  }
+  // Output layer (+ logits buffer) goes to the GPU that holds the last layers,
+  // i.e. the first one we filled.
+  let outputOnGpu = false;
+  if (placed === meta.nLayers && budgets.length) {
+    const outCost = meta.outputBytes + logitsBytes;
+    const target = used.findIndex((u, k) => u + outCost <= budgets[k].budget);
+    if (target >= 0) { used[target] += outCost; outputOnGpu = true; }
+
+    // Greedy packing leaves the first GPU at its limit; llama.cpp then fails to
+    // allocate compute buffers there. Spread layers so every GPU keeps the same
+    // fraction of headroom.
+    if (outputOnGpu && budgets.length > 1) {
+      const frac = used.reduce((a, b) => a + b, 0) / budgets.reduce((a, b) => a + b.budget, 0);
+      const u2 = budgets.map(() => computePerGpu);
+      const l2 = budgets.map(() => 0);
+      u2[target] += outCost;
+      let k = 0;
+      for (const i of order) {
+        const cost = layerCost(i);
+        while (k < budgets.length - 1 && u2[k] + cost > budgets[k].budget * frac) k++;
+        u2[k] += cost;
+        l2[k]++;
+      }
+      if (u2.every((u, j) => u <= budgets[j].budget)) { used = u2; layersPerGpu = l2; }
+    }
+  }
+
+  // Rough RAM bytes read per token by the CPU for layers left behind.
+  const cpuLayers = meta.nLayers - placed;
+  const activeFrac = meta.isMoE && meta.nExperts ? meta.nExpertsUsed / meta.nExperts : 1;
+  let cpuBytesPerToken = 0;
+  for (let i = 0; i < meta.nLayers; i++) {
+    const onCpu = !fits.includes(i);
+    const expert = meta.layerExpertBytes[i];
+    const dense = meta.layerBytes[i] - expert;
+    if (onCpu) cpuBytesPerToken += dense + expert * activeFrac;
+    else if (i < cpuMoeLayers) cpuBytesPerToken += expert * activeFrac;
+  }
+  if (!outputOnGpu) cpuBytesPerToken += meta.outputBytes;
+
+  return {
+    gpuLayers: placed + (outputOnGpu ? 1 : 0), // llama.cpp counts output as one extra layer
+    totalLayers: meta.nLayers + 1,
+    cpuLayers,
+    fullOffload: placed === meta.nLayers && outputOnGpu,
+    kvType,
+    deficit: budgets.length ? Math.max(0, totalNeed(meta, ctx, kvType, ubatch, cpuMoeLayers, budgets.length) - budgets.reduce((a, b) => a + b.budget, 0)) : Infinity,
+    layersPerGpu: budgets.map((b, k) => ({ index: b.index, layers: layersPerGpu[k], bytes: used[k] + PER_GPU_OVERHEAD, budget: b.budget + PER_GPU_OVERHEAD })),
+    kvBytes: kvTotalBytes(meta, ctx, kvType),
+    cpuBytesPerToken,
+    cpuMoeLayers,
+  };
+}
+
+/** Total bytes a full offload needs across all GPUs. */
+function totalNeed(meta, ctx, kvType, ubatch, cpuMoeLayers, nGpus) {
+  let t = meta.outputBytes + meta.vocab * ubatch * 4 + ctx * COMPUTE_BYTES_PER_CTX_TOKEN * (nGpus ? 1 : 0);
+  for (let i = 0; i < meta.nLayers; i++) t += meta.layerBytes[i] - (i < cpuMoeLayers ? meta.layerExpertBytes[i] : 0) + kvLayerBytes(meta, i, ctx, kvType);
+  return t;
+}
+
+/** Predicted decode tokens/s from memory bandwidth (for ranking only). */
+export function predictTps(meta, placement, gpus, hw, ctxDepth) {
+  const activeFrac = meta.isMoE && meta.nExperts ? meta.nExpertsUsed / meta.nExperts : 1;
+  let seconds = 0;
+  const byIndex = new Map(gpus.map((g) => [g.index, g]));
+  let layerCursor = meta.nLayers - 1;
+  for (const lp of placement.layersPerGpu) {
+    const gpu = byIndex.get(lp.index);
+    let bytes = 0;
+    for (let n = 0; n < lp.layers; n++, layerCursor--) {
+      const i = layerCursor;
+      const expert = i < placement.cpuMoeLayers ? 0 : meta.layerExpertBytes[i];
+      bytes += (meta.layerBytes[i] - meta.layerExpertBytes[i]) + expert * activeFrac;
+    }
+    bytes += placement.kvBytes * (lp.layers / meta.nLayers) * (ctxDepth / Math.max(1, placement.ctx || ctxDepth)) * KV_TYPES[placement.kvType].readCost;
+    seconds += bytes / (gpu.bandwidthGBps * 1e9 * DECODE_EFFICIENCY);
+  }
+  if (placement.fullOffload) seconds += meta.outputBytes / (gpus[0].bandwidthGBps * 1e9 * DECODE_EFFICIENCY);
+  seconds += placement.cpuBytesPerToken / (hw.ramBandwidthGBps * 1e9 * 0.6);
+  return seconds > 0 ? 1 / seconds : 0;
+}
+
+/**
+ * Build ranked candidate configurations for a target context length.
+ * Each candidate is engine-agnostic; engines translate it to their own flags.
+ */
+export function planCandidates(meta, hw, { ctx, kvTypes = Object.keys(KV_TYPES), allowCpuMoe = true } = {}) {
+  const gpus = hw.gpus;
+  const out = [];
+
+  for (const kvType of kvTypes) {
+    const base = { ctx, kvType, flashAttention: true, ubatch: 512, batch: 512, parallel: 1, gpuOrder: gpus.map((g) => g.index) };
+    let placement = placeLayers(meta, gpus, base);
+
+    // MoE: if dense layers don't all fit, try keeping experts of the first N layers in RAM.
+    if (!placement.fullOffload && meta.isMoE && allowCpuMoe) {
+      for (let n = 1; n <= meta.nLayers; n++) {
+        const p = placeLayers(meta, gpus, { ...base, cpuMoeLayers: n });
+        if (p.fullOffload) { placement = p; break; }
+      }
+    }
+
+    placement.ctx = ctx;
+    // Close to fitting: also try the full offload for real (a failed load is cheap).
+    if (!placement.fullOffload && placement.deficit <= PROBE_MARGIN && gpus.length) {
+      // Spread all layers proportionally to each GPU's budget.
+      const total = placement.layersPerGpu.reduce((a, l) => a + l.budget, 0);
+      let left = meta.nLayers;
+      const layersPerGpu = placement.layersPerGpu.map((l, k, arr) => {
+        const n = k === arr.length - 1 ? left : Math.round(meta.nLayers * l.budget / total);
+        left -= n;
+        return { ...l, layers: n };
+      });
+      const probe = { ...placement, layersPerGpu, fullOffload: true, gpuLayers: placement.totalLayers, cpuLayers: 0, cpuBytesPerToken: 0, probe: true };
+      out.push(makeCandidate(meta, hw, base, probe));
+    }
+    out.push(makeCandidate(meta, hw, base, placement));
+  }
+
+  if (!gpus.length) {
+    return out.slice(0, 1).map((c) => ({ ...c, gpuLayers: 0, fullOffload: false }));
+  }
+  return out.sort((a, b) => Number(b.fullOffload && !b.probe) - Number(a.fullOffload && !a.probe) || b.score - a.score);
+}
+
+function makeCandidate(meta, hw, base, placement) {
+  const gpus = hw.gpus;
+  const physical = hw.cpu.physicalCores;
+  const { ctx, kvType } = base;
+  const onGpuOnly = placement.fullOffload && placement.cpuMoeLayers === 0;
+  // Fully on GPU the CPU only feeds tokens; with CPU layers use all physical cores.
+  const threads = onGpuOnly ? Math.min(4, physical) : Math.max(1, physical - (physical > 4 ? 1 : 0));
+  const tpsShort = predictTps(meta, placement, gpus, hw, 512);
+  const tpsDeep = predictTps(meta, placement, gpus, hw, ctx * 0.6);
+
+  return {
+    id: `${kvType}-${placement.gpuLayers}${placement.cpuMoeLayers ? `-moe${placement.cpuMoeLayers}` : ''}${placement.probe ? '-probe' : ''}`,
+    ...base,
+    threads,
+    gpuLayers: placement.gpuLayers,
+    totalLayers: placement.totalLayers,
+    fullOffload: placement.fullOffload,
+    cpuMoeLayers: placement.cpuMoeLayers,
+    tensorSplit: placement.layersPerGpu.map((l) => l.layers),
+    placement,
+    predicted: { tpsShort: +tpsShort.toFixed(1), tpsDeep: +tpsDeep.toFixed(1) },
+    probe: !!placement.probe,
+    score: tpsDeep * KV_TYPES[kvType].quality,
+  };
+}
+
+/** Largest context (multiple of 1024) that fits fully on GPU for a KV type. */
+export function maxFullOffloadContext(meta, hw, kvType, limit = meta.trainContext || 131072) {
+  let lo = 0, hi = Math.floor(limit / 1024);
+  while (lo < hi) {
+    const mid = Math.ceil((lo + hi) / 2);
+    const p = placeLayers(meta, hw.gpus, { ctx: mid * 1024, kvType });
+    if (p.fullOffload) lo = mid; else hi = mid - 1;
+  }
+  return lo * 1024;
+}