offgrid-ai 0.15.9 → 0.16.3

package/src/profiles.mjs

@@ -2,10 +2,8 @@ import { existsSync } from "node:fs";
 import { mkdir, readdir, rm, unlink, writeFile, readFile } from "node:fs/promises";
 import { join } from "node:path";
 import { PROFILE_DIR, RUN_DIR, LOG_DIR } from "./config.mjs";
-import { backendFor, baseUrlForFlags, defaultFlagsForBackend } from "./backends.mjs";
+import { backendFor, baseUrlForFlags, defaultFlagsForBackend, BACKENDS } from "./backends.mjs";
 import { computeFlags } from "./autodetect.mjs";
-import { detectMlxCapabilities, defaultMlxContextLength } from "./mlx-discovery.mjs";
-import { detectHardware } from "./hardware.mjs";
 import { readJson, writeJson } from "./json.mjs";
 
 // ── Path helpers ───────────────────────────────────────────────────────────
@@ -42,7 +40,15 @@ export async function loadProfiles() {
     .filter((e) => e.isDirectory() && existsSync(profileJsonPath(e.name)))
     .map((e) => e.name)
     .sort();
-  return Promise.all(ids.map((id) => readProfile(id)));
+  return (await Promise.all(ids.map((id) => readProfile(id))))
+    .map((p) => {
+      // Migrate legacy llama-cpp-mtp backend → llama-cpp with mtp capability
+      if (p.backend === "llama-cpp-mtp") {
+        return { ...p, backend: "llama-cpp", providerId: "llama-cpp", capabilities: { ...(p.capabilities ?? {}), mtp: true } };
+      }
+      return p;
+    })
+    .filter((p) => BACKENDS[p.backend]);
 }
 
 export async function readProfile(id) {
@@ -51,7 +57,7 @@ export async function readProfile(id) {
   return JSON.parse(await readFile(path, "utf8"));
 }
 
-export async function saveProfile(profile, options = {}) {
+export async function saveProfile(profile) {
   const id = sanitizeProfileId(profile.id);
   const dir = profileDir(id);
   await mkdir(dir, { recursive: true });
@@ -128,7 +134,7 @@ export async function createProfileFromModel(model, backendId, drafterPath) {
   const caps = detectCapabilities(model.path, model.mmprojPath);
   // If a drafter is provided, this model supports MTP regardless of filename
   const hasMtp = caps.mtp || Boolean(drafterPath);
-  const backend = backendId ?? (hasMtp ? "llama-cpp-mtp" : "llama-cpp");
+  const backend = backendId ?? "llama-cpp";
   const { flags } = computeFlags(
     { ...caps, mtp: hasMtp },
     model.path,
@@ -152,28 +158,6 @@ export async function createProfileFromModel(model, backendId, drafterPath) {
   });
 }
 
-// ── Auto-create profile from a discovered MLX model ────────────────────────
-
-export async function createProfileFromMlxModel(model) {
-  const { DEFAULT_PORT } = await import("./mlx-flags.mjs");
-  const caps = await detectMlxCapabilities(model.filePath);
-  const ctxSize = defaultMlxContextLength(caps.contextLength, detectHardware().totalRamBytes / (1024 ** 3));
-  return normalizeProfile({
-    id: slugFromLabel(model.label),
-    label: model.label,
-    backend: "mlx-vlm",
-    providerId: "mlx-vlm",
-    modelAlias: model.label,
-    source: model.source,
-    modelPath: model.filePath,
-    mmprojPath: null,
-    drafterPath: null,
-    modelSizeBytes: model.sizeBytes,
-    capabilities: caps,
-    flags: { host: "127.0.0.1", port: DEFAULT_PORT, ctxSize },
-  });
-}
-
 function summarizeCapabilities(caps) {
   return {
     architecture: caps.architecture,

package/src/ui.mjs

@@ -106,25 +106,7 @@ export function renderCard(title, body, options = {}) {
 }
 
 function wrapVisible(text, width) {
-  const words = String(text).split(/(\s+)/u);
-  const lines = [];
-  let current = "";
-  for (let word of words) {
-    // If a single word exceeds the width, hard-break it
-    while (visibleLen(word) > width) {
-      if (current.trim()) { lines.push(current.trimEnd()); current = ""; }
-      lines.push(word.slice(0, width));
-      word = word.slice(width);
-    }
-    if (visibleLen(current + word) > width && current.trim()) {
-      lines.push(current.trimEnd());
-      current = word.trimStart();
-    } else {
-      current += word;
-    }
-  }
-  if (current.trim()) lines.push(current.trimEnd());
-  return lines.length > 0 ? lines : [text];
+  return wrapText(text, width);
 }
 
 export function renderSectionRows(title, rows, options = {}) {
@@ -221,6 +203,7 @@ export function createPrompt() {
           if (!Number.isFinite(input) || input < min || input > max) {
             return `Enter a number from ${min} to ${max}.`;
           }
+          return true;
         },
       });
       return Number(value);

package/resources/mlxvlm-server-wrapper.py

@@ -1,112 +0,0 @@
-#!/usr/bin/env python3
-"""
-mlx-vlm server wrapper with strict=False model loading + APC merge fix.
-
-Two monkey-patches are applied before the server starts:
-
-1. strict=False model loading — needed for architectures with shared-KV weight
-   schemes (e.g. Gemma 4). Most models (Qwen, Llama, Mistral, Phi) load fine
-   with strict=True — strict=False is a no-op for them.
-
-2. BatchRotatingKVCache.merge() shape-mismatch fix — upstream mlx-lm bug
-   (ml-explore/mlx-lm PR #1116, Blaizzy/mlx-vlm Issue #923). The merge() method
-   crashes with `ValueError: [broadcast_shapes] Shapes (1,1,28,256) and
-   (1,1,512,256) cannot be broadcast` when APC merges exact-cache entries with
-   different fill levels. This affects all sliding-window attention models
-   (Gemma 4, Mistral, Mixtral). The fix uses explicit slicing instead of
-   negative indexing to guarantee exactly `l` elements are extracted.
-
-   This patch can be removed once mlx-lm fixes merge() upstream (not fixed in
-   0.31.2 or 0.31.3 — the merge() method is identical in both).
-
-Benchmark finding: mlx-vlm clears Metal cache after every request (GitHub Issue
-#999) unless APC_ENABLED=1 is set. The env var is set by the Electron app at
-spawn time, not in this wrapper.
-
-Usage:
-  python3 mlxvlm-server-wrapper.py --model <path> --host 127.0.0.1 --port <port>
-"""
-import sys
-
-# ── Patch 1: strict=False model loading ──────────────────────────────────────
-
-import mlx_vlm.utils as _utils
-_orig_load_model = _utils.load_model
-
-def _patched_load_model(model_path, lazy=False, strict=True, **kwargs):
-    return _orig_load_model(model_path, lazy=lazy, strict=False, **kwargs)
-
-_utils.load_model = _patched_load_model
-
-# ── Patch 2: BatchRotatingKVCache.merge() shape-mismatch fix ──────────────────
-#
-# Upstream bug: _temporal_order() can return a buffer whose seq dimension differs
-# from c.size(). The negative slice [..., -l:, :] then produces a mismatched shape,
-# crashing with ValueError: [broadcast_shapes].
-#
-# Fix: use explicit slicing to extract exactly `l` elements, right-aligning within
-# the target slice when the buffer is shorter than `l` (left-padded by zeros from
-# the pre-allocated target tensor).
-
-import mlx.core as mx
-from mlx_lm.models import cache as _lm_cache
-
-_orig_merge = _lm_cache.BatchRotatingKVCache.merge
-
-@classmethod
-def _patched_merge(cls, caches):
-    if not all(c.max_size == caches[0].max_size for c in caches):
-        raise ValueError(
-            "BatchRotatingKVCache can only merge caches with the same maximum size"
-        )
-
-    offsets = [c.offset for c in caches]
-    lengths = [c.size() for c in caches]
-    max_length = max(lengths)
-
-    if max_length == 0:
-        return cls(caches[0].max_size, [0] * len(caches))
-
-    padding = [max_length - l for l in lengths]
-    B = len(caches)
-    H = max(c.keys.shape[1] for c in caches if c.keys is not None)
-    Dk = max(c.keys.shape[3] for c in caches if c.keys is not None)
-    Dv = max(c.values.shape[3] for c in caches if c.values is not None)
-    dt = next(iter(c.keys.dtype for c in caches if c.keys is not None))
-
-    keys = mx.zeros((B, H, max_length, Dk), dtype=dt)
-    values = mx.zeros((B, H, max_length, Dv), dtype=dt)
-    for i, (p, l, c) in enumerate(zip(padding, lengths, caches)):
-        if c.keys is None:
-            continue
-        ordered_k = c._temporal_order(c.keys)
-        ordered_v = c._temporal_order(c.values)
-        seq_len = ordered_k.shape[2]
-        if seq_len >= l:
-            # Normal case: extract the last `l` tokens.
-            start = seq_len - l
-            keys[i : i + 1, :, p : p + l] = ordered_k[..., start : start + l, :]
-            values[i : i + 1, :, p : p + l] = ordered_v[..., start : start + l, :]
-        else:
-            # Buffer shorter than l: right-align within the slice (left-padded
-            # by zeros from the pre-allocated target tensor).
-            gap = l - seq_len
-            keys[i : i + 1, :, p + gap : p + l] = ordered_k
-            values[i : i + 1, :, p + gap : p + l] = ordered_v
-
-    cache = cls(caches[0].max_size, padding)
-    cache.keys = keys
-    cache.values = values
-    cache.offset = mx.array(offsets)
-    cache._idx = keys.shape[2]
-    cache._offset = keys.shape[2]
-
-    return cache
-
-_lm_cache.BatchRotatingKVCache.merge = _patched_merge
-
-# ── Run the server ────────────────────────────────────────────────────────────
-# main() parses sys.argv for --model, --host, --port, etc.
-from mlx_vlm.server import main
-main()
-

package/src/mlx-flags.mjs

@@ -1,100 +0,0 @@
-// mlx-vlm server flag computation — pure functions, no side effects.
-// Ported from deprecated-offgrid-desktop/src/main/server-flags.ts (MLX subset).
-//
-// Benchmark-informed decisions (see sidequests/mlx-backend-benchmark/RESULTS.md):
-// - mlx-vlm requires APC_ENABLED=1 env var (86x TTFT improvement) — set at spawn
-//   time in process.mjs, NOT here (this module only computes args).
-// - mlx-vlm uses a strict=False wrapper script for shared-KV architectures
-//   (Gemma 4-class). Safe for all models — strict=False is a no-op for models
-//   that load fine with strict=True.
-// - mlx-vlm uses --enable-thinking for thinking-mode control.
-// - mlx-vlm uses --max-kv-size for the KV cache / context window.
-//
-// Only the mlx-vlm-relevant logic is ported here. offgrid-ai's existing GGUF
-// flag logic (autodetect.mjs / profile-setup.mjs / estimate.mjs) is unchanged.
-
-import { fileURLToPath } from "node:url";
-import { dirname, join } from "node:path";
-
-const MB = 1024 ** 2;
-
-/** Default port for the local model server. Matches the desktop's DEFAULT_PORT. */
-export const DEFAULT_PORT = 18080;
-
-/** Resolved path to the bundled strict=False wrapper script (sibling of src/). */
-export const MLX_VLM_WRAPPER = join(dirname(fileURLToPath(import.meta.url)), "..", "resources", "mlxvlm-server-wrapper.py");
-
-/** Overhead multiplier for mlx-vlm: weights × 1.5 (covers KV cache, activations, APC cache; benchmark-validated). */
-const MLX_VLM_OVERHEAD_MULTIPLIER = 1.5;
-
-/** Server process overhead in MB. */
-const PROCESS_OVERHEAD_MB = 200;
-
-/**
- * Estimate mlx-vlm memory usage (MB): model weights × 1.5 + process overhead.
- *
- * The 1.5 multiplier covers KV cache, activations, and APC cache overhead
- * (benchmark-validated; see sidequests/mlx-backend-benchmark/RESULTS.md).
- * GGUF/llama-server estimation uses the detailed path in estimate.mjs.
- *
- * @param {number} fileSizeBytes - model size on disk (sum of MLX safetensors).
- * @returns {number} estimated memory in MB.
- */
-export function estimateMemoryMb(fileSizeBytes) {
-  return Math.round((fileSizeBytes / MB) * MLX_VLM_OVERHEAD_MULTIPLIER + PROCESS_OVERHEAD_MB);
-}
-
-/**
- * Compute mlx-vlm server arguments.
- *
- * mlx-vlm is the MLX-native server (benchmark-validated best throughput + memory
- * efficiency on Apple Silicon). Invoked via the strict=False wrapper script for
- * compatibility with shared-KV architectures (Gemma 4-class).
- *
- * The APC_ENABLED=1 env var is MANDATORY but is set at spawn time in
- * process.mjs, not in args.
- *
- * The wrapper script (resources/mlxvlm-server-wrapper.py) applies strict=False
- * model loading + the BatchRotatingKVCache.merge() fix, both required for
- * shared-KV architectures (Gemma 4-class). It is resolved to a real path via
- * MLX_VLM_WRAPPER; there is intentionally no raw-mlx_vlm.server path.
- *
- * @param {string} modelPath - path to the MLX model directory.
- * @param {object} [options]
- * @param {number} [options.port] - port (default DEFAULT_PORT).
- * @param {number} [options.ctxSize] - context window (passed as --max-kv-size).
- * @param {boolean} [options.thinkingEnabled=true] - whether to enable thinking.
- * @returns {{ args: string[], port: number }}
- */
-export function computeMlxVlmFlags(modelPath, options = {}) {
-  const port = options.port ?? DEFAULT_PORT;
-  const ctxSize = options.ctxSize;
-  const thinkingEnabled = options.thinkingEnabled ?? true;
-
-  // The binary is "python3" (resolved by backendBinaryFor in backends.mjs); the
-  // wrapper path is the first arg.
-  const args = [
-    MLX_VLM_WRAPPER,
-    "--model", modelPath,
-    "--host", "127.0.0.1",
-    "--port", String(port),
-  ];
-
-  if (thinkingEnabled) {
-    args.push("--enable-thinking");
-  }
-
-  // Context size: mlx-vlm uses --max-kv-size for the KV cache / context window.
-  if (ctxSize && ctxSize > 0) {
-    args.push("--max-kv-size", String(ctxSize));
-  }
-
-  // Default max output tokens — used when the client doesn't specify max_tokens
-  // in the request. Pi's OpenAI completions provider never sends max_tokens
-  // (it doesn't fall back to model.maxTokens like the Anthropic provider does).
-  // llama-server defaults high; mlx-vlm defaults to 2048 which is too low for
-  // coding tasks. Set a generous server-side default.
-  args.push("--max-tokens", "16384");
-
-  return { args, port };
-}