offgrid-ai 0.16.0 → 0.17.0

package/src/harness-pi.mjs

@@ -8,10 +8,8 @@ import pc from "picocolors";
 // ── Pi model id ─────────────────────────────────────────────────────────────
 
 /**
- * The model id Pi must send in requests. mlx-vlm registers the loaded model
- * with the full --model path as its API id (verified via /v1/models); sending
- * the repo-id label instead makes mlx-vlm unload the local model and re-fetch
- * the repo from HuggingFace. Other backends use the friendly modelAlias.
+ * The model id Pi must send in requests. All backends use the friendly
+ * modelAlias as the API model id.
  */
 export function piApiModelId(profile) {
   return profile.modelAlias;
@@ -87,7 +85,7 @@ async function activeProviderProfiles(currentProfile) {
   const byAlias = new Map();
   for (const item of [...allProfiles, currentProfile]) {
     if (item.providerId !== currentProfile.providerId) continue;
-    if (item.backend !== "llama-cpp" && item.backend !== "llama-cpp-mtp") {
+    if (item.backend !== "llama-cpp") {
       byAlias.set(item.modelAlias, item);
       continue;
     }

package/src/managed.mjs

@@ -1,6 +1,6 @@
 import { existsSync } from "node:fs";
 import { BACKENDS } from "./backends.mjs";
-import { commandExists } from "./exec.mjs";
+import { hasOmlx } from "./omlx-runtime.mjs";
 
 export const MANAGED_BACKEND_IDS = ["omlx"];
 
@@ -22,6 +22,6 @@ export function hasLmStudioInstalled() {
   return existsSync("/Applications/LM Studio.app");
 }
 
-export function hasOmlxInstalled() {
-  return commandExists("omlx");
+export async function hasOmlxInstalled() {
+  return await hasOmlx();
 }

package/src/mlx-discovery.mjs

@@ -2,7 +2,7 @@
 // to compute sizes and publishers. The oMLX API doesn't return these, so we
 // read them from disk.
 
-import { readdir, stat } from "node:fs/promises";
+import { readdir, stat, readFile } from "node:fs/promises";
 import { existsSync } from "node:fs";
 import { join } from "node:path";
 import { homedir } from "node:os";
@@ -65,6 +65,99 @@ export async function scanOmlxModelSizes() {
   return infoByBasename;
 }
 
+// ── MTP capability detection ─────────────────────────────────────────────
+// oMLX supports native MTP for Qwen 3.5/3.6 (dense + MoE) and DeepSeek-V4
+// models that ship MTP heads in their weights. The check mirrors oMLX's own
+// _mtp_compat_for_model: config must declare mtp_num_hidden_layers, model_type
+// must be on the whitelist, and the safetensors index must contain mtp.* keys.
+
+const MTP_MODEL_TYPES = ["qwen3_5", "qwen3_5_moe", "qwen3_6", "qwen3_6_moe", "deepseek_v4"];
+
+/**
+ * Check if an oMLX model directory supports native MTP.
+ * Returns { compatible: boolean, reason: string }.
+ */
+export async function detectOmlxMtpCapability(modelDir) {
+  const configPath = join(modelDir, "config.json");
+  if (!existsSync(configPath)) return { compatible: false, reason: "config.json not found" };
+
+  let config;
+  try {
+    config = JSON.parse(await readFile(configPath, "utf8"));
+  } catch {
+    return { compatible: false, reason: "failed to read config.json" };
+  }
+
+  const mtpLayers = config.mtp_num_hidden_layers;
+  if (!mtpLayers || mtpLayers <= 0) {
+    return { compatible: false, reason: "model has no MTP heads in config" };
+  }
+
+  const modelType = config.model_type;
+  if (!MTP_MODEL_TYPES.some((t) => modelType === t || modelType?.startsWith(t))) {
+    return { compatible: false, reason: `model_type=${modelType} is not on the MTP whitelist (supported: ${MTP_MODEL_TYPES.join(", ")})` };
+  }
+
+  // Check for mtp.* weight tensors in the safetensors index
+  const hasWeights = await modelHasMtpWeights(modelDir);
+  if (!hasWeights) {
+    return { compatible: false, reason: "Config declares MTP layers but the converted weights are missing mtp.* tensors. Re-convert from HF with a converter that preserves MTP weights." };
+  }
+
+  return { compatible: true, reason: "" };
+}
+
+async function modelHasMtpWeights(modelDir) {
+  const indexPath = join(modelDir, "model.safetensors.index.json");
+  if (existsSync(indexPath)) {
+    try {
+      const index = JSON.parse(await readFile(indexPath, "utf8"));
+      const weightMap = index.weight_map ?? {};
+      return Object.keys(weightMap).some((key) => key.includes("mtp."));
+    } catch {
+      return false;
+    }
+  }
+  // Single-shard fallback: can't easily read safetensors keys in Node without
+  // the safetensors library. Check for an mtp-specific safetensors file.
+  try {
+    const entries = await readdir(modelDir);
+    return entries.some((f) => f.endsWith(".safetensors") && /mtp/i.test(f));
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Find the model directory for a given oMLX model ID.
+ * Searches ~/.omlx/models/ recursively for a directory matching the model ID.
+ */
+export async function findOmlxModelDir(modelId) {
+  if (!existsSync(OMLX_MODELS_DIR)) return null;
+  const basename = modelId.includes("/") ? modelId.slice(modelId.lastIndexOf("/") + 1)
+    : modelId.includes("--") ? modelId.slice(modelId.lastIndexOf("--") + 2)
+    : modelId;
+
+  async function walk(dir) {
+    let entries;
+    try {
+      entries = await readdir(dir, { withFileTypes: true });
+    } catch {
+      return null;
+    }
+    for (const entry of entries) {
+      if (!entry.isDirectory()) continue;
+      const fullPath = join(dir, entry.name);
+      if (entry.name === basename && await isMlxModelDir(fullPath)) return fullPath;
+      const found = await walk(fullPath);
+      if (found) return found;
+    }
+    return null;
+  }
+
+  return await walk(OMLX_MODELS_DIR);
+}
+
 /**
  * Look up a model's info by its oMLX API id. Tries exact match, then the
  * segment after `--` (oMLX org--name format), then after `/` (HF format).

package/src/model-name.mjs

@@ -4,9 +4,9 @@
 // No other function should format, title-case, or dissect a model name.
 //
 // The returned `id` is always the raw identifier (untouched) and is used for
-// API calls, profile IDs, Pi config matching, and benchmark directory slugs.
+// API calls, profile IDs, and Pi config matching.
 // The returned `display` is the human-readable string shown in pickers, details,
-// and benchmark metadata.
+
 
 // ── Known model families ────────────────────────────────────────────────
 //

package/src/model-presenters.mjs

@@ -7,9 +7,8 @@ import { pc, formatBytes, renderSectionRows } from "./ui.mjs";
 import { capabilitySummary, ggufDetailParts, isProfileFileMissing, profileDetailParts } from "./model-summary.mjs";
 import { itemKey } from "./model-catalog.mjs";
 import { DATA_DIR } from "./config.mjs";
-import { findBenchmarkRepo } from "./benchmark.mjs";
 
-const OPTION_SEPARATOR = pc.dim("  │  ");
+const OPTION_SEPARATOR = "  ";
 const OPTION_STATUS_WIDTH = 12;
 const OPTION_BACKEND_WIDTH = 14;
 const OPTION_SOURCE_WIDTH = 14;
@@ -53,7 +52,6 @@ function optionBackendTag(backendId) {
   const label = backend?.label ?? backendId ?? "unknown";
   const colors = {
     "llama-cpp": pc.cyan,
-    "llama-cpp-mtp": pc.blue,
     omlx: pc.magenta,
   };
   return optionPad(label, colors[backendId] ?? pc.dim, OPTION_BACKEND_WIDTH);
@@ -219,15 +217,6 @@ export function printWorkspaceHeader(normalized, runningProfilesNow, modelMissin
   console.log(pc.dim("   ─────────────────────────────────────────────────────────"));
 }
 
-export async function printBenchmarkLine() {
-  const repoPath = await findBenchmarkRepo();
-  if (repoPath) {
-    console.log(pc.green("   ✓") + " local-llm-visual-benchmark linked");
-  } else {
-    console.log(pc.yellow("   ○") + " to run benchmarks, pair with " + pc.cyan("local-llm-visual-benchmark"));
-  }
-}
-
 export async function printProfileDetails(profile) {
   const backend = backendFor(profile.backend);
   const isManaged = backend.type === "managed-server";
@@ -267,8 +256,9 @@ export async function printProfileDetails(profile) {
       const scriptPath = join(profileDir(profile.id), "start.sh");
       console.log("\n" + renderSectionRows("Server command", [
         ["Run manually", pc.cyan(`bash ${scriptPath}`)],
-        ["Command", pc.dim(script)],
-      ], { columns: Math.min(process.stdout.columns ?? 120, 140) }));
+      ]));
+      console.log("");
+      console.log(pc.dim(script));
     }
   }
 }

package/src/omlx-runtime.mjs

@@ -0,0 +1,232 @@
+// oMLX runtime management — discovery, version checking, update prompts,
+// and installation. Mirrors the llama.cpp runtime pattern in runtime.mjs.
+//
+// oMLX is installed either via the macOS app (DMG from GitHub Releases, which
+// installs a ~/.omlx/bin/omlx CLI shim with in-app auto-update) or via Homebrew
+// (brew tap jundot/omlx && brew install omlx). offgrid-ai does NOT download
+// binaries directly — it uses brew for automated installs, and prompts for the
+// DMG when brew is unavailable.
+
+import { execFile } from "node:child_process";
+import { existsSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { promisify } from "node:util";
+import { compareVersions } from "./updates.mjs";
+import { hasHomebrew, ensureHomebrewFor } from "./config.mjs";
+import { commandExists } from "./exec.mjs";
+import { pc, renderCard, renderRows } from "./ui.mjs";
+
+const execFileAsync = promisify(execFile);
+
+const OMLX_CLI_SHIM = join(homedir(), ".omlx", "bin", "omlx");
+const RELEASE_API = "https://api.github.com/repos/jundot/omlx/releases/latest";
+
+// ── Discovery ──────────────────────────────────────────────────────────────
+
+/**
+ * Find the oMLX CLI binary. Checks the app-installed shim first (it shadows
+ * brew on PATH), then falls back to PATH. Returns the path or null.
+ */
+export async function findOmlx() {
+  // 1. macOS app CLI shim (shadows brew on PATH)
+  if (existsSync(OMLX_CLI_SHIM)) return OMLX_CLI_SHIM;
+
+  // 2. PATH (brew install)
+  try {
+    const { stdout } = await execFileAsync("which", ["omlx"]);
+    const path = stdout.trim();
+    if (path && existsSync(path)) return path;
+  } catch { /* not on PATH */ }
+
+  return null;
+}
+
+/**
+ * Check if oMLX is installed (app shim or on PATH).
+ */
+export async function hasOmlx() {
+  return (await findOmlx()) !== null;
+}
+
+/**
+ * Detect how oMLX was installed: "app" (macOS app CLI shim) or "brew" (on PATH
+ * but not the shim) or null (not installed).
+ */
+export async function omlxInstallMethod() {
+  if (existsSync(OMLX_CLI_SHIM)) return "app";
+  if (await commandExists("omlx")) return "brew";
+  return null;
+}
+
+// ── Version checking ───────────────────────────────────────────────────────
+
+/**
+ * Get the installed oMLX version by running `omlx --version`.
+ * Returns null if oMLX is not installed or the version can't be parsed.
+ */
+export async function installedOmlxVersion() {
+  const bin = await findOmlx();
+  if (!bin) return null;
+  try {
+    const { stdout } = await execFileAsync(bin, ["--version"], { timeout: 5000 });
+    const match = stdout.trim().match(/(\d+\.\d+\.\d+)/u);
+    return match ? match[1] : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Check for the latest oMLX release on GitHub.
+ * Returns { tag, version } or null if the check fails.
+ * Callers must treat null as "check failed, skip prompt".
+ */
+export async function latestOmlxRelease(fetchImpl = globalThis.fetch) {
+  try {
+    const response = await fetchImpl(RELEASE_API, { signal: AbortSignal.timeout(5000) });
+    if (!response.ok) return null;
+    const body = await response.json();
+    const tag = typeof body?.tag_name === "string" ? body.tag_name : null;
+    if (!tag) return null;
+    const version = tag.replace(/^v/u, "");
+    return { tag, version };
+  } catch {
+    return null;
+  }
+}
+
+// ── Update prompt ──────────────────────────────────────────────────────────
+
+/**
+ * Check if an oMLX update is available and offer to install it.
+ * Returns true if an update was installed, false otherwise.
+ *
+ * For brew installs: runs `brew upgrade omlx`.
+ * For app installs: the app has in-app auto-update, so we just inform the user.
+ */
+export async function offerManagedOmlxUpdate(prompt, { fetchImpl = globalThis.fetch } = {}) {
+  const latest = await latestOmlxRelease(fetchImpl);
+  if (!latest) return false;
+
+  const installed = await installedOmlxVersion();
+  if (installed && compareVersions(installed, latest.version) >= 0) return false;
+
+  const method = await omlxInstallMethod();
+
+  console.log("\n" + renderCard("oMLX runtime", renderRows([
+    ["Installed", installed ?? pc.yellow("not installed")],
+    ["Latest", pc.green(latest.version)],
+    ["Source", method === "app" ? "macOS app (in-app auto-update)" : "Homebrew"],
+  ]), { formatBorder: pc.cyan }));
+
+  // App installs have their own auto-update — just inform the user
+  if (method === "app" && installed) {
+    console.log(pc.dim("  Update oMLX via the app's in-app updater, then restart offgrid-ai."));
+    return false;
+  }
+
+  // Not installed, or brew install — offer to install/upgrade
+  if (!installed) {
+    const shouldInstall = await prompt.yesNo("Install oMLX runtime?", false);
+    if (!shouldInstall) return false;
+    return await installOmlx(prompt);
+  }
+
+  // Brew install with an update available
+  const shouldUpdate = await prompt.yesNo("Update oMLX runtime?", false);
+  if (!shouldUpdate) return false;
+
+  try {
+    const { runCommand } = await import("./exec.mjs");
+    console.log(pc.dim("Updating oMLX via Homebrew..."));
+    await runCommand("brew", ["update"], { label: "brew update" });
+    await runCommand("brew", ["upgrade", "omlx"], { label: "brew upgrade omlx" });
+    console.log(pc.green(`✓ Updated oMLX to latest`));
+    return true;
+  } catch (err) {
+    console.log(pc.red(`✗ Update failed: ${err.message}`));
+    console.log(pc.dim("Update manually: brew update && brew upgrade omlx"));
+    return false;
+  }
+}
+
+// ── Installation ───────────────────────────────────────────────────────────
+
+/**
+ * Install oMLX. Uses Homebrew if available (automating tap + install).
+ * If Homebrew is not available, prompts to download the DMG from GitHub
+ * Releases or install Homebrew first.
+ *
+ * @param {object} prompt - UI prompt interface (yesNo, choice)
+ * @param {function} [run] - runCommand function for verbose command execution
+ * @returns {Promise<boolean>} true if installation succeeded
+ */
+export async function installOmlx(prompt, run) {
+  const hasBrew = await hasHomebrew();
+
+  if (!hasBrew) {
+    if (!(await ensureHomebrewFor(prompt, run || (async (cmd, args, label) => {
+      const { runCommand } = await import("./exec.mjs");
+      return runCommand(cmd, args, { label });
+    }), "oMLX"))) {
+      console.log(pc.dim("Install oMLX manually:"));
+      console.log(pc.dim("  brew tap jundot/omlx && brew install omlx"));
+      console.log(pc.dim("  — or download the macOS app from https://github.com/jundot/omlx/releases"));
+      return false;
+    }
+  }
+
+  // Install oMLX via Homebrew
+  const runner = run || (async (cmd, args, label) => {
+    const { runCommand } = await import("./exec.mjs");
+    return runCommand(cmd, args, { label });
+  });
+
+  console.log(pc.cyan("Installing oMLX via Homebrew..."));
+  try {
+    await runner("brew", ["tap", "jundot/omlx", "https://github.com/jundot/omlx"], "oMLX tap");
+    await runner("brew", ["install", "omlx"], "oMLX");
+    console.log(pc.green("✓ oMLX installed"));
+    return true;
+  } catch (err) {
+    console.log(pc.red(`✗ oMLX installation failed: ${err.message}`));
+    console.log(pc.dim("Install manually: brew tap jundot/omlx && brew install omlx"));
+    console.log(pc.dim("  — or download the macOS app from https://github.com/jundot/omlx/releases"));
+    return false;
+  }
+}
+
+/**
+ * Ensure oMLX runtime is available. For onboarding: if not installed, offer
+ * to install it. This is the oMLX equivalent of ensureLlamaRuntime().
+ *
+ * @param {object} prompt - UI prompt interface
+ * @param {function} [run] - runCommand function for verbose output
+ * @returns {Promise<boolean>} true if oMLX is available (installed or pre-existing)
+ */
+export async function ensureOmlxRuntime(prompt, run) {
+  let omlxBin = await findOmlx();
+  if (!omlxBin) {
+    console.log(renderCard("oMLX runtime", renderRows([
+      ["Status", pc.yellow("not installed")],
+      ["Used for", "local MLX models (Apple Silicon optimized)"],
+      ["Install", "managed by offgrid-ai via Homebrew"],
+    ]), { formatBorder: pc.cyan }));
+
+    const shouldInstall = await prompt.yesNo("Install oMLX runtime?", true);
+    if (shouldInstall) {
+      await installOmlx(prompt, run);
+      omlxBin = await findOmlx();
+    }
+
+    if (!omlxBin) {
+      console.log(pc.yellow("Skipping oMLX for now. You can still use llama.cpp, or run offgrid-ai again to install."));
+      return false;
+    }
+  }
+
+  const version = await installedOmlxVersion();
+  console.log(pc.green(`✓ oMLX: ${omlxBin}${version ? ` (v${version})` : ""}`));
+  return true;
+}

package/src/process.mjs

@@ -6,6 +6,7 @@ import { basename, join } from "node:path";
 import { LOG_DIR } from "./config.mjs";
 import { writeState, readState, profileDir } from "./profiles.mjs";
 import { backendFor, backendBinaryFor } from "./backends.mjs";
+import { pc } from "./ui.mjs";
 
 const execFileAsync = promisify(execFile);
 
@@ -21,7 +22,7 @@ export async function computeServerCommand(profile) {
   const binary = await backendBinaryFor(profile.backend);
   if (!binary) throw new Error("Server binary not found. Run offgrid-ai interactively to install.");
 
-  // llama-cpp / llama-cpp-mtp
+  // llama-cpp
   const { computeFlags } = await import("./autodetect.mjs");
   const result = computeFlags(
     profile.capabilities ?? {},
@@ -118,6 +119,10 @@ async function startLocalServer(profile) {
 
 async function startManagedServer(profile, backend) {
   if (await serverReady(profile.baseUrl)) {
+    // Apply per-model settings (MTP) even when server is already running.
+    if (backend.id === "omlx" && profile.capabilities?.mtp) {
+      await ensureOmlxMtpSetting(profile);
+    }
     return writeManagedState(profile, backend);
   }
 
@@ -126,9 +131,15 @@ async function startManagedServer(profile, backend) {
     try {
       const { execFile } = await import("node:child_process");
       const { promisify } = await import("node:util");
-      await promisify(execFile)("omlx", ["start"], { timeout: 10000 });
-    } catch {
-      throw new Error(`${backend.label} is not running and could not be auto-started. Install oMLX or run \`omlx start\` manually.`);
+      const { findOmlx } = await import("./omlx-runtime.mjs");
+      const omlxBin = await findOmlx();
+      if (!omlxBin) {
+        throw new Error(`${backend.label} is not installed. Run offgrid-ai to install it, or install manually: brew tap jundot/omlx && brew install omlx`);
+      }
+      await promisify(execFile)(omlxBin, ["start"], { timeout: 10000 });
+    } catch (err) {
+      if (err.message.includes("not installed")) throw err;
+      throw new Error(`${backend.label} could not be auto-started: ${err.message}. Run \`omlx start\` manually.`, { cause: err });
     }
   }
 
@@ -141,9 +152,48 @@ async function startManagedServer(profile, backend) {
   if (!(await serverReady(profile.baseUrl))) {
     throw new Error(`${backend.label} is not responding at ${profile.baseUrl}. Start it and try again.`);
   }
+
+  // Apply per-model settings (MTP) before the model is loaded.
+  // oMLX applies MTP patches at load time, so the setting must be in
+  // model_settings.json before any request triggers a load.
+  if (backend.id === "omlx" && profile.capabilities?.mtp) {
+    await ensureOmlxMtpSetting(profile);
+  }
+
   return writeManagedState(profile, backend);
 }
 
+/**
+ * Enable MTP on an oMLX model via the admin API before loading.
+ * oMLX applies MTP patches at model load time, so the setting must be
+ * persisted to model_settings.json before any request triggers a load.
+ * If the model is already loaded, oMLX will use the setting on next reload.
+ */
+async function ensureOmlxMtpSetting(profile) {
+  const baseUrl = profile.baseUrl?.replace(/\/v1\/?$/u, "") || "";
+  const modelId = profile.omlxModel ?? profile.modelAlias ?? profile.id;
+  try {
+    const response = await fetch(`${baseUrl}/admin/api/models/${encodeURIComponent(modelId)}/settings`, {
+      method: "PUT",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ mtp_enabled: true }),
+      signal: AbortSignal.timeout(5000),
+    });
+    if (response.ok) {
+      console.log(pc.green(`[mtp] Enabled MTP speculative decoding for ${modelId}`));
+    } else if (response.status === 401 || response.status === 403) {
+      console.log(pc.yellow(`[mtp] Could not enable MTP: oMLX admin authentication required. Enable skip_api_key_verification in oMLX settings, or enable MTP manually from the admin panel.`));
+    } else if (response.status === 404) {
+      console.log(pc.yellow(`[mtp] Model ${modelId} not found on oMLX server. MTP setting not applied.`));
+    } else {
+      const detail = await response.text().catch(() => "");
+      console.log(pc.yellow(`[mtp] Could not enable MTP: HTTP ${response.status} ${detail}`));
+    }
+  } catch (err) {
+    console.log(pc.yellow(`[mtp] Could not enable MTP: ${err.message}`));
+  }
+}
+
 async function writeManagedState(profile, backend) {
   const state = {
     pid: null,
@@ -223,7 +273,7 @@ async function processGone(pid) {
 export async function unloadModelFromServer(profile) {
   const backend = backendFor(profile.backend);
 
-  if (backend.id === "llama-cpp" || backend.id === "llama-cpp-mtp") {
+  if (backend.id === "llama-cpp") {
     return { unloaded: false, backend: backend.id, reason: "stop server to unload" };
   }