offgrid-ai 0.1.2

package/README.md

@@ -0,0 +1,107 @@
+<div align="center">
+
+# offgrid-ai
+
+**Privacy-first CLI for running local LLMs. Your AI, your machine, nothing leaves.**
+
+[![node](https://img.shields.io/badge/node-20%2B-3c873a)](package.json)
+[![platform](https://img.shields.io/badge/platform-macOS%20%7C%20Linux-blue)]()
+
+Install • Run • Done.
+
+```bash
+curl -fsSL https://raw.githubusercontent.com/eeshansrivastava89/offgrid-ai/main/install.sh | bash
+```
+
+</div>
+
+## What it does
+
+You run `offgrid-ai`. It finds your local models, auto-configures everything, starts the server, and launches Pi. Zero configuration. No parameter tuning. No presets.
+
+**First run** walks you through installing anything missing (Homebrew, llama-server). After that, just run it.
+
+```bash
+offgrid-ai          # pick a model and run it
+offgrid-ai status   # show running servers (from another terminal)
+offgrid-ai stop     # stop a running server
+```
+
+## Install
+
+One command. Installs Node.js if you don't have it, then installs offgrid-ai.
+
+```bash
+curl -fsSL https://raw.githubusercontent.com/eeshansrivastava89/offgrid-ai/main/install.sh | bash
+```
+
+Or if you already have Node.js:
+
+```bash
+npm install -g offgrid-ai
+```
+
+Or review the install script first:
+
+```bash
+curl -fsSL https://raw.githubusercontent.com/eeshansrivastava89/offgrid-ai/main/install.sh | less
+```
+
+## How it works
+
+1. **Auto-detect everything.** Scans for GGUF models in LM Studio, HuggingFace, and Ollama directories. Reads model metadata (quantization, context size, vision, thinking mode) directly from the GGUF file. No presets, no manual configuration.
+
+2. **One command to run.** `offgrid-ai` → pick a model → it figures out the flags, starts llama-server, syncs Pi config, and launches Pi.
+
+3. **One model at a time.** Laptops have limited RAM. One server, one model, no confusion.
+
+## Supported backends
+
+| Backend | Type | Auto-detected |
+|---|---|---|
+| **llama.cpp** | Local server | ✓ GGUF models in `~/.lmstudio/models/` |
+| **llama.cpp MTP** | Local server (speculative decoding) | ✓ MTP detected from model metadata |
+| **Ollama** | Managed server | ✓ via `localhost:11434` |
+| **oMLX** | Managed server | ✓ via `127.0.0.1:8000` |
+
+## First run onboarding
+
+When you run `offgrid-ai` for the first time on a fresh machine:
+
+1. **Homebrew** — Required. Offered to install if missing.
+2. **llama-server** — Required for GGUF models. Offered to install via Homebrew.
+3. **Model backend** — At least one is needed: LM Studio, Ollama, or oMLX.
+4. **Models** — If no models found, tells you where to get them.
+
+Subsequent runs skip everything that's already installed.
+
+## Data directory
+
+```
+~/.offgrid-ai/
+  config.json          # auto-detected paths, editable for overrides
+  profiles/            # one per model, auto-created on first run
+    <id>/
+      profile.json     # model metadata + auto-detected settings
+      command.json     # llama-server flags (auto-generated, hand-editable)
+      notes.md         # scratch notes
+  logs/
+  run/                 # PID state files
+```
+
+## Benchmark (coming soon)
+
+"Benchmark" is always shown as an option in the CLI. If the [local-llm-visual-benchmark](https://github.com/eeshansrivastava89/local-llm-visual-benchmark) repo is found locally, it works. If not, it offers to clone it. Model management works standalone; benchmarking is the upsell.
+
+## Development
+
+```bash
+git clone https://github.com/eeshansrivastava89/offgrid-ai.git
+cd offgrid-ai
+npm install
+node bin/offgrid-ai.mjs
+```
+
+## License
+
+Personal project by [Eeshan Srivastava](https://eeshans.com).

package/bin/offgrid-ai.mjs

@@ -0,0 +1,10 @@
+#!/usr/bin/env node
+import pc from "picocolors";
+import { run } from "../src/cli.mjs";
+
+try {
+  await run(process.argv.slice(2));
+} catch (error) {
+  console.error(pc.red("error:"), error instanceof Error ? error.message : String(error));
+  process.exit(1);
+}

package/install.sh

@@ -0,0 +1,162 @@
+#!/usr/bin/env bash
+# offgrid-ai installer
+#
+# Install:
+#   curl -fsSL https://raw.githubusercontent.com/eeshansrivastava89/offgrid-ai/main/install.sh | bash
+#
+# Or review first:
+#   curl -fsSL https://raw.githubusercontent.com/eeshansrivastava89/offgrid-ai/main/install.sh | less
+#
+# What this does:
+#   1. Checks for Node.js
+#   2. If not found, installs it via nvm (no sudo needed)
+#   3. Installs offgrid-ai globally via npm
+#   4. Runs offgrid-ai
+#
+# Flags:
+#   --dry-run    Show what would happen without making changes
+#   --no-run     Install but don't launch offgrid-ai after
+#   --help       Show this help
+#
+# This script never uses sudo. Everything installs to user-writable directories.
+
+set -euo pipefail
+
+# ── Flags ───────────────────────────────────────────────────────────────────
+
+DRY_RUN=false
+SKIP_RUN=false
+
+for arg in "$@"; do
+  case "$arg" in
+    --dry-run)  DRY_RUN=true; echo "[dry-run] No changes will be made." ;;
+    --no-run)   SKIP_RUN=true ;;
+    --help|-h)   echo "Usage: curl -fsSL <url> | bash -s -- [--dry-run] [--no-run]"; exit 0 ;;
+  esac
+done
+
+dry() { if $DRY_RUN; then printf "[dry-run] %s\n" "$*"; return 0; else "$@"; fi; }
+
+# ── Output helpers ──────────────────────────────────────────────────────────
+
+BOLD='\033[1m' RESET='\033[0m' GREEN='\033[32m' YELLOW='\033[33m' BLUE='\033[34m' RED='\033[31m'
+info()  { printf "${BLUE}→${RESET} %s\n" "$*"; }
+ok()     { printf "${GREEN}✓${RESET} %s\n" "$*"; }
+warn()   { printf "${YELLOW}!${RESET} %s\n" "$*"; }
+fail()   { printf "${RED}✗${RESET} %s\n" "$*"; exit 1; }
+
+# ── Detect OS ───────────────────────────────────────────────────────────────
+
+OS="$(uname -s)"
+ARCH="$(uname -m)"
+case "$OS" in
+  Darwin) OS="macos" ;;
+  Linux)  OS="linux" ;;
+  *)      fail "Unsupported OS: $OS. offgrid-ai requires macOS or Linux." ;;
+esac
+case "$ARCH" in
+  x86_64|amd64) ARCH="x64" ;;
+  arm64|aarch64) ARCH="arm64" ;;
+  *)             fail "Unsupported architecture: $ARCH" ;;
+esac
+info "Detected: ${OS}-${ARCH}"
+
+# ── Check for Node.js ───────────────────────────────────────────────────────
+
+if command -v node &>/dev/null; then
+  NODE_VERSION="$(node --version 2>/dev/null || echo "unknown")"
+  ok "Node.js ${NODE_VERSION} found at $(command -v node)"
+else
+  echo ""
+  printf "${BOLD}offgrid-ai needs Node.js.${RESET}\n"
+  printf "It will be installed now via nvm (Node Version Manager).\n"
+  printf "This installs to your home directory — no sudo needed.\n"
+  echo ""
+
+  # Install nvm
+  export NVM_DIR="${NVM_DIR:-$HOME/.nvm}"
+  info "Installing nvm..."
+  dry curl -fsSL https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh -o /tmp/nvm-install.sh
+  dry bash /tmp/nvm-install.sh
+
+  # Source nvm
+  if ! $DRY_RUN; then
+    [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
+  fi
+
+  # Install Node.js LTS
+  info "Installing Node.js LTS..."
+  dry nvm install --lts
+
+  # Verify
+  if $DRY_RUN || command -v node &>/dev/null; then
+    ok "Node.js $(node --version 2>/dev/null || echo 'installed') installed via nvm."
+  else
+    # nvm added to shell profile but not active in this session
+    ok "Node.js installed via nvm."
+    echo ""
+    warn "Node.js was installed but your shell doesn't see it yet."
+    echo "  Restart your terminal, or run: source ~/.nvm/nvm.sh"
+    echo "  Then re-run this installer."
+    exit 0
+  fi
+fi
+
+# ── Install offgrid-ai ──────────────────────────────────────────────────────
+
+echo ""
+printf "${BOLD}Installing offgrid-ai...${RESET}\n"
+dry npm install -g offgrid-ai
+
+# ── Verify ───────────────────────────────────────────────────────────────────
+
+if $DRY_RUN; then
+  ok "offgrid-ai installed (dry-run)"
+  echo ""
+  printf "${BOLD}${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}\n"
+  printf "${BOLD}${GREEN}  offgrid-ai is ready! (dry-run)${RESET}\n"
+  printf "${BOLD}${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}\n"
+  echo ""
+  echo "  Run: offgrid-ai"
+  echo ""
+  exit 0
+fi
+
+if command -v offgrid-ai &>/dev/null; then
+  ok "offgrid-ai installed at $(command -v offgrid-ai)"
+else
+  echo ""
+  warn "offgrid-ai was installed but isn't on your PATH yet."
+  echo "  Restart your terminal and run: offgrid-ai"
+  echo "  Or run: source ~/.nvm/nvm.sh && offgrid-ai"
+  echo ""
+  printf "${BOLD}${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}\n"
+  printf "${BOLD}${GREEN}  offgrid-ai is ready!${RESET}\n"
+  printf "${BOLD}${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}\n"
+  echo ""
+  echo "  Run: offgrid-ai"
+  echo ""
+  exit 0
+fi
+
+# ── Done ─────────────────────────────────────────────────────────────────────
+
+echo ""
+printf "${BOLD}${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}\n"
+printf "${BOLD}${GREEN}  offgrid-ai is ready!${RESET}\n"
+printf "${BOLD}${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}\n"
+echo ""
+echo "  First run will walk you through setting up everything you need"
+echo "  (llama-server, model backends, Pi)."
+echo ""
+echo "  Run: offgrid-ai"
+echo ""
+
+if [[ -t 0 ]] && ! $SKIP_RUN; then
+  printf "${BOLD}Run offgrid-ai now? [Y/n]${RESET} "
+  read -r response
+  response="${response:-Y}"
+  if [[ "$response" =~ ^[Yy]$ ]]; then
+    exec offgrid-ai
+  fi
+fi

package/package.json

@@ -0,0 +1,51 @@
+{
+  "name": "offgrid-ai",
+  "version": "0.1.2",
+  "description": "Privacy-first CLI for running local LLMs — discover, configure, run, benchmark",
+  "author": "Eeshan Srivastava (https://eeshans.com)",
+  "type": "module",
+  "bin": {
+    "offgrid-ai": "./bin/offgrid-ai.mjs"
+  },
+  "files": [
+    "bin/*.mjs",
+    "src/*.mjs",
+    "install.sh"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/eeshansrivastava89/offgrid-ai.git"
+  },
+  "homepage": "https://github.com/eeshansrivastava89/offgrid-ai#readme",
+  "bugs": {
+    "url": "https://github.com/eeshansrivastava89/offgrid-ai/issues"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "scripts": {
+    "start": "node bin/offgrid-ai.mjs",
+    "test": "node --test test/*.mjs",
+    "check:privacy": "node scripts/privacy-gate.mjs",
+    "release:check": "bash scripts/release-check.sh",
+    "release:check:fast": "bash scripts/release-check.sh --skip-install --skip-manual",
+    "prepack": "npm run check:privacy"
+  },
+  "dependencies": {
+    "@clack/prompts": "^1.4.0",
+    "picocolors": "^1.1.0"
+  },
+  "keywords": [
+    "local-llm",
+    "llama-cpp",
+    "ollama",
+    "cli",
+    "privacy",
+    "llm",
+    "ai"
+  ],
+  "license": "MIT"
+}

package/src/autodetect.mjs

@@ -0,0 +1,113 @@
+import { basename } from "node:path";
+import { existsSync } from "node:fs";
+import { readGgufMetadata } from "./gguf.mjs";
+
+// ── Detect model capabilities from GGUF metadata ──────────────────────────
+
+export function detectCapabilities(modelPath, mmprojPath) {
+  const meta = existsSync(modelPath) ? readGgufMetadata(modelPath) : {};
+  const name = basename(modelPath).toLowerCase();
+
+  // Architecture
+  const architecture = meta["general.architecture"] ?? null;
+
+  // Thinking / reasoning mode
+  const hasThinkingKwargs = meta["chat_template_kwargs"] !== undefined;
+  const nameHintsThinking = /qwen3|gemma-4|gemma4|deepseek-r[12]/i.test(name);
+  const thinking = hasThinkingKwargs || nameHintsThinking;
+
+  // Vision — mmproj present
+  const vision = mmprojPath && existsSync(mmprojPath);
+
+  // MTP (multi-token prediction) — detect speculative decoding
+  const mtp = /mtp/i.test(name) || architecture === "qwen3";
+
+  // Quantization
+  const quant = name.match(/(Q\d_K_[A-Z]+|UD-[A-Z0-9_]+)/i)?.[1] ?? null;
+
+  // Context size from metadata, fallback to name hints
+  const metaCtx = architecture
+    ? numberMeta(meta, `${architecture}.context_length`)
+    : undefined;
+  const ctxSize = metaCtx ?? (thinking ? 80000 : 32768);
+
+  return { architecture, thinking, vision, mtp, quant, metaCtx, ctxSize, meta };
+}
+
+// ── Compute llama-server flags from capabilities ───────────────────────────
+
+export function computeFlags(capabilities, modelPath, mmprojPath, draftModelPath) {
+  const { thinking, mtp, quant } = capabilities;
+  const isLowMem = quant && /[Qq]4[_0]/i.test(quant);
+
+  const flags = {
+    host: "127.0.0.1",
+    port: 8080,
+    ctxSize: capabilities.ctxSize,
+    flashAttention: "on",
+    cacheTypeK: isLowMem ? "f16" : "bf16",
+    cacheTypeV: isLowMem ? "f16" : "bf16",
+    jinja: true,
+    temperature: 0.6,
+    topP: 0.95,
+    topK: thinking ? 64 : 20,
+    minP: 0,
+    presencePenalty: thinking ? 0 : 1.5,
+    repeatPenalty: thinking ? 1.1 : 1.0,
+    parallel: 1,
+    batchSize: 512,
+  };
+
+  // Thinking mode
+  if (thinking) {
+    flags.chatTemplateKwargs = { enable_thinking: true };
+  }
+
+  // Build argv
+  const argv = [
+    "--model", modelPath,
+  ];
+
+  if (mmprojPath) argv.push("--mmproj", mmprojPath);
+  if (draftModelPath) argv.push("--spec-draft-model", draftModelPath);
+
+  argv.push(
+    "--host", String(flags.host),
+    "--port", String(flags.port),
+    "--ctx-size", String(flags.ctxSize),
+    "--flash-attn", flags.flashAttention,
+    "--cache-type-k", flags.cacheTypeK,
+    "--cache-type-v", flags.cacheTypeV,
+  );
+
+  if (flags.jinja) argv.push("--jinja");
+
+  argv.push(
+    "--temp", String(flags.temperature),
+    "--top-p", String(flags.topP),
+    "--top-k", String(flags.topK),
+    "--min-p", flags.minP.toFixed(2),
+    "--presence-penalty", String(flags.presencePenalty),
+    "--repeat-penalty", String(flags.repeatPenalty),
+    "--batch-size", String(flags.batchSize),
+    "--parallel", String(flags.parallel),
+  );
+
+  if (flags.chatTemplateKwargs) {
+    argv.push("--chat-template-kwargs", JSON.stringify(flags.chatTemplateKwargs));
+  }
+
+  // MTP flags
+  if (mtp) {
+    argv.push("--spec-type", "draft-mtp", "--spec-draft-n-max", "2");
+  }
+
+  return { flags, argv };
+}
+
+// ── Internal helper ─────────────────────────────────────────────────────
+
+function numberMeta(meta, key) {
+  const value = key ? meta[key] : undefined;
+  return typeof value === "number" && Number.isFinite(value) ? value : undefined;
+}

package/src/backends.mjs

@@ -0,0 +1,135 @@
+import { findLlamaServer } from "./config.mjs";
+import { scanGgufModels } from "./scan.mjs";
+
+// ── Backend definitions ────────────────────────────────────────────────────
+
+export const BACKENDS = {
+  "llama-cpp": {
+    id: "llama-cpp",
+    label: "llama.cpp",
+    type: "local-server",
+    providerId: "llama-cpp",
+    defaultPort: 8080,
+    defaultBaseUrl: "http://127.0.0.1:8080/v1",
+    needsCommandFile: true,
+    needsModelFile: true,
+    scanModels: () => scanGgufModels(),
+  },
+  "llama-cpp-mtp": {
+    id: "llama-cpp-mtp",
+    label: "llama.cpp MTP",
+    type: "local-server",
+    providerId: "llama-cpp-mtp",
+    defaultPort: 8081,
+    defaultBaseUrl: "http://127.0.0.1:8081/v1",
+    needsCommandFile: true,
+    needsModelFile: true,
+    extraArgs: ["--spec-type", "draft-mtp", "--spec-draft-n-max", "2"],
+    scanModels: () => scanGgufModels(),
+  },
+  "ollama": {
+    id: "ollama",
+    label: "Ollama",
+    type: "managed-server",
+    providerId: "ollama",
+    defaultPort: 11434,
+    defaultBaseUrl: "http://localhost:11434/v1",
+    needsCommandFile: false,
+    needsModelFile: false,
+    scanModels: () => scanOllamaModels(),
+  },
+  "omlx": {
+    id: "omlx",
+    label: "oMLX",
+    type: "managed-server",
+    providerId: "omlx",
+    defaultPort: 8000,
+    defaultBaseUrl: "http://127.0.0.1:8000/v1",
+    needsCommandFile: false,
+    needsModelFile: false,
+    scanModels: () => scanOmlxModels(),
+  },
+};
+
+export function backendFor(backendId) {
+  const backend = BACKENDS[backendId ?? "llama-cpp"];
+  if (!backend) throw new Error(`Unknown backend: ${backendId}`);
+  return backend;
+}
+
+export function inferBackendId(modelOrProfile) {
+  const haystack = [
+    modelOrProfile?.path,
+    modelOrProfile?.modelPath,
+    modelOrProfile?.label,
+    modelOrProfile?.modelAlias,
+    modelOrProfile?.id,
+    modelOrProfile?.providerId,
+    modelOrProfile?.backend,
+  ].filter(Boolean).join(" ").toLowerCase();
+  if (haystack.includes("mtp")) return "llama-cpp-mtp";
+  return "llama-cpp";
+}
+
+export async function backendBinaryFor(backendId) {
+  const backend = BACKENDS[backendId ?? "llama-cpp"];
+  if (backend.type === "managed-server") return null;
+  const discovered = await findLlamaServer();
+  return discovered; // null means "not found — trigger onboarding"
+}
+
+// ── Ollama model discovery ──────────────────────────────────────────────
+
+async function scanOllamaModels() {
+  try {
+    const response = await fetch("http://localhost:11434/api/tags", { signal: AbortSignal.timeout(3000) });
+    if (!response.ok) return [];
+    const body = await response.json();
+    if (!Array.isArray(body?.models)) return [];
+    return body.models.map((model) => ({
+      id: model.name,
+      label: ollamaLabel(model.name),
+      aliasSuggestion: model.name,
+      sizeBytes: model.size ?? 0,
+      quant: model.details?.quantization_level,
+      family: model.details?.family,
+      backend: "ollama",
+      source: "ollama",
+    })).sort((a, b) => a.label.localeCompare(b.label));
+  } catch {
+    return [];
+  }
+}
+
+// ── oMLX model discovery ───────────────────────────────────────────────
+
+async function scanOmlxModels() {
+  try {
+    const response = await fetch("http://127.0.0.1:8000/v1/models", { signal: AbortSignal.timeout(3000) });
+    if (!response.ok) return [];
+    const body = await response.json();
+    if (!Array.isArray(body?.data)) return [];
+    return body.data.map((model) => ({
+      id: model.id,
+      label: omlxLabel(model.id),
+      aliasSuggestion: model.id,
+      sizeBytes: 0,
+      quant: null,
+      family: null,
+      backend: "omlx",
+      source: "omlx",
+    })).sort((a, b) => a.label.localeCompare(b.label));
+  } catch {
+    return [];
+  }
+}
+
+// ── Labels ──────────────────────────────────────────────────────────────
+
+function ollamaLabel(name) {
+  return name.replace(/[-_]/g, " ").replace(/^gemma\b/i, "Gemma").replace(/^qwen/i, "Qwen");
+}
+
+function omlxLabel(id) {
+  return id.replace(/[-_]/g, " ").replace(/^gemma-4/i, "Gemma 4").replace(/^qwen/i, "Qwen");
+}