offgrid-ai 0.3.26 → 0.3.28

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "offgrid-ai",
-  "version": "0.3.26",
+  "version": "0.3.28",
   "description": "Privacy-first CLI for running local LLMs — discover, configure, run, benchmark",
   "author": "Eeshan Srivastava (https://eeshans.com)",
   "type": "module",

package/src/autodetect.mjs

@@ -6,6 +6,7 @@ import { readGgufMetadata } from "./gguf.mjs";
 
 export function detectCapabilities(modelPath, mmprojPath) {
   const meta = safeReadGgufMetadata(modelPath);
+  const mmprojMeta = mmprojPath ? safeReadGgufMetadata(mmprojPath) : {};
   const name = basename(modelPath).toLowerCase();
   const pathHints = String(modelPath).toLowerCase();
 
@@ -25,13 +26,14 @@ export function detectCapabilities(modelPath, mmprojPath) {
 
   // Vision — mmproj present
   const vision = Boolean(mmprojPath && existsSync(mmprojPath));
+  const mmprojProjectorType = stringMeta(mmprojMeta, "clip.vision.projector_type") ?? stringMeta(mmprojMeta, "clip.audio.projector_type") ?? null;
 
   // MTP (multi-token prediction) — detect speculative decoding.
   // Do not treat all Qwen models as MTP; require an explicit filename or metadata hint.
   const mtp = /\bmtp\b|draft-mtp|multi-token/i.test(pathHints) || Object.keys(meta).some((key) => /mtp|draft|speculative/i.test(key));
 
   // Quantization
-  const quant = name.match(/(Q\d_K_[A-Z]+|UD-[A-Z0-9_]+)/i)?.[1] ?? null;
+  const quant = name.match(/(Q\d_K_[A-Z]+|Q\d_[01]|UD-[A-Z0-9_]+)/i)?.[1] ?? null;
 
   // Context size from metadata, fallback to name hints
   const metaCtx = architecture
@@ -39,7 +41,7 @@ export function detectCapabilities(modelPath, mmprojPath) {
     : undefined;
   const ctxSize = metaCtx ?? (thinking ? 80000 : 32768);
 
-  return { architecture, thinking, vision, mtp, qat, imatrix, quant, metaCtx, ctxSize, meta };
+  return { architecture, thinking, vision, mtp, qat, imatrix, quant, metaCtx, ctxSize, meta, mmprojProjectorType };
 }
 
 // ── Compute llama-server flags from capabilities ───────────────────────────
@@ -127,4 +129,9 @@ function safeReadGgufMetadata(modelPath) {
 function numberMeta(meta, key) {
   const value = key ? meta[key] : undefined;
   return typeof value === "number" && Number.isFinite(value) ? value : undefined;
+}
+
+function stringMeta(meta, key) {
+  const value = key ? meta[key] : undefined;
+  return typeof value === "string" && value ? value : undefined;
 }

package/src/cli.mjs

@@ -15,6 +15,7 @@ import { removeInstallerPathEntries } from "./shell-path.mjs";
 import { configureLocalProfile } from "./profile-setup.mjs";
 import { buildPrettyCommand } from "./command.mjs";
 import { detectCapabilities } from "./autodetect.mjs";
+import { offerManagedLlamaRuntimeUpdate } from "./runtime.mjs";
 
 // ── Entry point ────────────────────────────────────────────────────────────
 
@@ -66,6 +67,15 @@ export async function run(argv) {
 export async function mainFlow() {
   await ensureDirs();
 
+  if (process.stdin.isTTY) {
+    const runtimePrompt = createPrompt();
+    try {
+      await offerManagedLlamaRuntimeUpdate(runtimePrompt);
+    } finally {
+      runtimePrompt.close();
+    }
+  }
+
   // 1. Check what backends are available
   const llamaBinary = await findLlamaServer();
   const ggufModels = await scanGgufModels();
@@ -456,6 +466,33 @@ function capabilitySummary(caps) {
   return parts.length > 0 ? parts.join(" · ") : "standard GGUF";
 }
 
+function isUnsupportedMmprojError(err, profile) {
+  const message = String(err?.message ?? "");
+  return Boolean(profile.mmprojPath && /unknown projector type|failed to load multimodal model|failed to load CLIP model/i.test(message));
+}
+
+function textOnlyProfile(profile) {
+  return normalizeProfile({
+    ...profile,
+    mmprojPath: null,
+    disabledMmprojPath: profile.disabledMmprojPath ?? profile.mmprojPath,
+    capabilities: { ...(profile.capabilities ?? {}), vision: false, visionDisabledReason: "unsupported-mmproj" },
+    commandArgv: removeCommandOption(profile.commandArgv ?? [], "--mmproj"),
+  });
+}
+
+function removeCommandOption(argv, flag) {
+  const next = [];
+  for (let i = 0; i < argv.length; i++) {
+    if (argv[i] === flag) {
+      if (argv[i + 1] && !argv[i + 1].startsWith("--")) i += 1;
+      continue;
+    }
+    next.push(argv[i]);
+  }
+  return next;
+}
+
 function createManagedProfile(model, backendId) {
   return normalizeProfile({
     id: model.id.replace(/[^a-z0-9._-]+/gi, "-").toLowerCase(),
@@ -510,6 +547,13 @@ async function runProfile(profile, options = {}) {
         if (state?.pid) {
           try { await stopProfile(profile); } catch { /* best effort */ }
         }
+        if (!options.textOnlyRetry && isUnsupportedMmprojError(err, profile)) {
+          console.log(pc.yellow("Vision projector is not supported by this llama.cpp build. Retrying text-only."));
+          console.log(pc.dim("Update llama.cpp later to re-enable vision for this model."));
+          const textOnly = textOnlyProfile(profile);
+          await saveProfile(textOnly);
+          return await runProfile(textOnly, { ...options, textOnlyRetry: true });
+        }
         throw err;
       }
     }

package/src/config.mjs

@@ -10,6 +10,8 @@ export const DATA_DIR = process.env.OFFGRID_DIR || join(homedir(), ".offgrid-ai"
 export const PROFILE_DIR = join(DATA_DIR, "profiles");
 export const LOG_DIR = join(DATA_DIR, "logs");
 export const RUN_DIR = join(DATA_DIR, "run");
+export const RUNTIME_DIR = join(DATA_DIR, "runtime");
+export const MANAGED_LLAMA_SERVER = join(RUNTIME_DIR, "bin", "llama-server");
 
 // ── Default scan directories ──────────────────────────────────────────────
 
@@ -74,14 +76,17 @@ export async function findLlamaServer() {
     return process.env.LLAMA_SERVER_BINARY;
   }
 
-  // 2. which/where
+  // 2. offgrid-ai managed runtime
+  if (existsSync(MANAGED_LLAMA_SERVER)) return MANAGED_LLAMA_SERVER;
+
+  // 3. which/where
   try {
     const { stdout } = await execFileAsync("which", ["llama-server"]);
     const path = stdout.trim();
     if (path && existsSync(path)) return path;
   } catch { /* not on PATH */ }
 
-  // 3. Homebrew
+  // 4. Homebrew
   try {
     const { stdout } = await execFileAsync("brew", ["--prefix", "llama.cpp"]);
     const prefix = stdout.trim();

package/src/profile-setup.mjs

@@ -50,10 +50,22 @@ export async function configureLocalProfile(prompt, profile) {
     console.log("");
     console.log(renderSection("QAT detected", renderRows([
       ["Meaning", "quantization-aware trained"],
-      ["Runtime flags", "none required"],
+      ["Runtime flags", "none QAT-specific"],
     ])));
   }
 
+  if (caps.vision && profile.mmprojPath) {
+    console.log("");
+    const gemma4Unified = isGemma4UnifiedProjector(caps.mmprojProjectorType);
+    console.log(renderSection("Vision projector detected", renderRows([
+      ["Projector", caps.mmprojProjectorType ?? "unknown"],
+      ["Flag", `--mmproj ${profile.mmprojPath}`],
+      ...(gemma4Unified ? [["Note", pc.yellow("Gemma 4 unified projectors need newer llama.cpp than current Homebrew stable.")]] : []),
+    ])));
+    const useVision = await prompt.yesNo("Enable vision with --mmproj?", !gemma4Unified);
+    configured = useVision ? applyVisionDefaults(configured) : removeVisionDefaults(configured, gemma4Unified ? "gemma4-unified-unsupported" : "user-disabled");
+  }
+
   if (caps.thinking) {
     console.log("");
     console.log(renderSection("Thinking model detected", renderRows([
@@ -106,6 +118,23 @@ function removeMtpDefaults(profile) {
   });
 }
 
+function applyVisionDefaults(profile) {
+  if (!profile.mmprojPath) return profile;
+  return applyProfileFlags({
+    ...profile,
+    capabilities: { ...(profile.capabilities ?? {}), vision: true, visionDisabledReason: undefined },
+  }, profile.flags, { values: { "--mmproj": profile.mmprojPath } });
+}
+
+function removeVisionDefaults(profile, reason) {
+  return applyProfileFlags({
+    ...profile,
+    disabledMmprojPath: profile.mmprojPath,
+    mmprojPath: null,
+    capabilities: { ...(profile.capabilities ?? {}), vision: false, visionDisabledReason: reason },
+  }, profile.flags, { remove: ["--mmproj"] });
+}
+
 function applyThinkingDefaults(profile) {
   const flags = { ...profile.flags, ...THINKING_DEFAULTS };
   return applyProfileFlags(profile, flags);
@@ -179,6 +208,10 @@ function renderMemoryEstimate(profile) {
   }
 }
 
+function isGemma4UnifiedProjector(projectorType) {
+  return /gemma4u[va]/i.test(String(projectorType ?? ""));
+}
+
 function detectionSummary(caps) {
   const parts = [];
   if (caps.architecture) parts.push(caps.architecture);

package/src/profiles.mjs

@@ -168,6 +168,7 @@ function summarizeCapabilities(caps) {
     imatrix: caps.imatrix,
     quant: caps.quant,
     metaCtx: caps.metaCtx,
+    mmprojProjectorType: caps.mmprojProjectorType,
     ctxSize: caps.ctxSize,
   };
 }

package/src/runtime.mjs

@@ -0,0 +1,130 @@
+import { createHash } from "node:crypto";
+import { execFile } from "node:child_process";
+import { existsSync } from "node:fs";
+import { chmod, mkdir, mkdtemp, readFile, rm, symlink, unlink, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { basename, join } from "node:path";
+import { promisify } from "node:util";
+import { MANAGED_LLAMA_SERVER, RUNTIME_DIR } from "./config.mjs";
+import { compareVersions } from "./updates.mjs";
+import { pc, renderCard, renderRows } from "./ui.mjs";
+
+const execFileAsync = promisify(execFile);
+const RELEASE_API = "https://api.github.com/repos/ggml-org/llama.cpp/releases/latest";
+const VERSION_PATH = join(RUNTIME_DIR, "llama.cpp", "VERSION.json");
+
+export async function offerManagedLlamaRuntimeUpdate(prompt, { fetchImpl = globalThis.fetch } = {}) {
+  const latest = await latestLlamaRelease(fetchImpl);
+  if (!latest) return false;
+
+  const installed = await installedRuntime();
+  if (installed?.tag && compareBuildTags(installed.tag, latest.tag) >= 0 && existsSync(MANAGED_LLAMA_SERVER)) return false;
+
+  console.log("\n" + renderCard("llama.cpp runtime", renderRows([
+    ["Installed", installed?.tag ?? "none"],
+    ["Latest", pc.green(latest.tag)],
+    ["Source", "official GitHub release binary"],
+  ]), { formatBorder: pc.cyan }));
+
+  const shouldInstall = await prompt.yesNo(installed ? "Update llama.cpp runtime?" : "Install llama.cpp runtime?", true);
+  if (!shouldInstall) return false;
+
+  await installLlamaRelease(latest);
+  return true;
+}
+
+export async function latestLlamaRelease(fetchImpl = globalThis.fetch) {
+  try {
+    const response = await fetchImpl(RELEASE_API, { signal: AbortSignal.timeout(5000) });
+    if (!response.ok) return null;
+    const body = await response.json();
+    const tag = typeof body?.tag_name === "string" ? body.tag_name : null;
+    const asset = selectAsset(body?.assets ?? [], process.platform, process.arch);
+    if (!tag || !asset) return null;
+    return { tag, asset };
+  } catch {
+    return null;
+  }
+}
+
+export async function installedRuntime() {
+  try {
+    return JSON.parse(await readFile(VERSION_PATH, "utf8"));
+  } catch {
+    return null;
+  }
+}
+
+export async function installLlamaRelease(release) {
+  const tmp = await mkdtemp(join(tmpdir(), "offgrid-llama-"));
+  const archive = join(tmp, release.asset.name);
+  const releaseDir = join(RUNTIME_DIR, "llama.cpp", "releases", release.tag);
+  const binDir = join(RUNTIME_DIR, "bin");
+
+  try {
+    console.log(pc.dim(`Downloading ${release.asset.name}...`));
+    const response = await fetch(release.asset.url);
+    if (!response.ok) throw new Error(`Download failed: HTTP ${response.status}`);
+    const bytes = Buffer.from(await response.arrayBuffer());
+    verifyDigest(bytes, release.asset.digest);
+    await writeFile(archive, bytes);
+
+    await rm(releaseDir, { recursive: true, force: true });
+    await mkdir(releaseDir, { recursive: true });
+    await execFileAsync("tar", ["-xzf", archive, "--strip-components", "1", "-C", releaseDir]);
+
+    const llamaServer = join(releaseDir, "llama-server");
+    if (!existsSync(llamaServer)) throw new Error("Release archive did not contain llama-server");
+    await chmod(llamaServer, 0o755);
+
+    await mkdir(binDir, { recursive: true });
+    await unlink(MANAGED_LLAMA_SERVER).catch(() => {});
+    await symlink(llamaServer, MANAGED_LLAMA_SERVER);
+
+    await mkdir(join(RUNTIME_DIR, "llama.cpp"), { recursive: true });
+    await writeFile(VERSION_PATH, JSON.stringify({
+      runtime: "llama.cpp",
+      tag: release.tag,
+      asset: release.asset.name,
+      url: release.asset.url,
+      installedAt: new Date().toISOString(),
+      llamaServer,
+    }, null, 2) + "\n", "utf8");
+
+    console.log(pc.green(`✓ Installed llama.cpp ${release.tag}`));
+  } finally {
+    await rm(tmp, { recursive: true, force: true }).catch(() => {});
+  }
+}
+
+function selectAsset(assets, platform, arch) {
+  const suffix = assetSuffix(platform, arch);
+  if (!suffix) return null;
+  const asset = assets.find((item) => item?.name?.endsWith(suffix));
+  if (!asset?.browser_download_url) return null;
+  return {
+    name: asset.name,
+    url: asset.browser_download_url,
+    digest: asset.digest,
+    size: asset.size,
+  };
+}
+
+function assetSuffix(platform, arch) {
+  const cpu = arch === "arm64" ? "arm64" : arch === "x64" ? "x64" : null;
+  if (!cpu) return null;
+  if (platform === "darwin") return `bin-macos-${cpu}.tar.gz`;
+  if (platform === "linux") return `bin-ubuntu-${cpu}.tar.gz`;
+  return null;
+}
+
+function compareBuildTags(a, b) {
+  return compareVersions(String(a).replace(/^b/u, ""), String(b).replace(/^b/u, ""));
+}
+
+function verifyDigest(bytes, digest) {
+  if (!digest?.startsWith("sha256:")) return;
+  const expected = digest.slice("sha256:".length);
+  const actual = createHash("sha256").update(bytes).digest("hex");
+  if (actual !== expected) throw new Error(`${basename("llama.cpp")}: checksum mismatch`);
+}

package/src/scan.mjs

@@ -74,5 +74,5 @@ function aliasFromName(name) {
 }
 
 function quantFromName(name) {
-  return name.match(/(Q\d_K_[A-Z]+|UD-[A-Z0-9_]+)/)?.[1];
+  return name.match(/(Q\d_K_[A-Z]+|Q\d_[01]|UD-[A-Z0-9_]+)/)?.[1];
 }