offgrid-ai 0.9.3 → 0.9.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "offgrid-ai",
-  "version": "0.9.3",
+  "version": "0.9.5",
   "description": "Privacy-first CLI for running local LLMs — discover, configure, run, benchmark",
   "author": "Eeshan Srivastava (https://eeshans.com)",
   "type": "module",

package/src/benchmark/finalize.mjs

@@ -1,7 +1,7 @@
 // ── Unload model from server memory after benchmark ────────────────────────────
 
 import { backendFor } from "../backends.mjs";
-import { apiRootUrl } from "../process.mjs";
+import { apiRootUrl, serverModelIds } from "../process.mjs";
 import { existsSync } from "node:fs";
 import { readFile, writeFile } from "node:fs/promises";
 import { join } from "node:path";
@@ -33,14 +33,67 @@ export async function unloadModelFromServer(profile) {
   }
 
   if (backend.id === "omlx") {
-    // oMLX does not expose a model-unload endpoint. The model stays resident
-    // until the oMLX server process is stopped.
-    return { unloaded: false, backend: backend.id, reason: "no unload API available" };
+    return await unloadOmlxModel(profile);
   }
 
   return { unloaded: false, backend: backend.id, reason: "unsupported backend" };
 }
 
+async function unloadOmlxModel(profile) {
+  const baseUrl = profile.baseUrl?.replace(/\/v1\/?$/u, "") || "";
+  const adminUrl = `${baseUrl}/admin/api/models`;
+  const modelId = profile.modelAlias || profile.omlxModel || profile.id;
+
+  try {
+    const ids = await serverModelIds(profile.baseUrl);
+    const match = ids.find((id) => id.toLowerCase() === modelId.toLowerCase());
+    const targetId = match ?? modelId;
+
+    const response = await fetch(`${adminUrl}/${encodeURIComponent(targetId)}/unload`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      signal: AbortSignal.timeout(30000),
+    });
+
+    if (response.ok) {
+      return { unloaded: true, backend: "omlx", modelId: targetId };
+    }
+
+    const detail = await responseErrorDetail(response);
+
+    if (response.status === 400 && /not loaded/i.test(detail)) {
+      return { unloaded: true, backend: "omlx", modelId: targetId, reason: "model was not loaded" };
+    }
+
+    if (response.status === 401 || response.status === 403) {
+      return {
+        unloaded: false,
+        backend: "omlx",
+        modelId: targetId,
+        error: "oMLX admin authentication required. Enable skip_api_key_verification in oMLX settings, or unload manually from the admin panel.",
+      };
+    }
+
+    return { unloaded: false, backend: "omlx", modelId: targetId, error: `HTTP ${response.status}: ${detail}` };
+  } catch (err) {
+    if (err?.name === "AbortError" || err?.name === "TimeoutError") {
+      return { unloaded: false, backend: "omlx", modelId, error: "Unload request timed out. The model may still be unloading in the background." };
+    }
+    return { unloaded: false, backend: "omlx", modelId, error: err.message };
+  }
+}
+
+async function responseErrorDetail(response) {
+  const text = await response.text().catch(() => "");
+  if (!text) return "";
+  try {
+    const body = JSON.parse(text);
+    return body?.detail ?? body?.message ?? text;
+  } catch {
+    return text;
+  }
+}
+
 export async function finalizeBenchmarkRun(runDirectory, runResult, speedMetrics) {
   const metadataPath = join(runDirectory, "metadata.json");
   const metadata = JSON.parse(await readFile(metadataPath, "utf8"));

package/src/benchmark/flow.mjs

@@ -4,7 +4,7 @@ import { join } from "node:path";
 import { ensureDirs } from "../config.mjs";
 import { backendFor } from "../backends.mjs";
 import { hasPi, hasPiModel, syncPiConfig } from "../harness-pi.mjs";
-import { serverReady, startServer, waitForReady, stopProfile } from "../process.mjs";
+import { serverReady, startServer, waitForReady, stopProfile, modelAvailableOnServer } from "../process.mjs";
 import { loadProfiles } from "../profiles.mjs";
 import { pc, createPrompt } from "../ui.mjs";
 import { linkBenchmarkRepo } from "./repo.mjs";
@@ -28,9 +28,20 @@ async function chooseBenchmarkAction(prompt, canRun) {
   return await prompt.choice("Action", canRun ? choices : choices.filter((c) => c.value === "prepare"), canRun ? "run" : "prepare");
 }
 
+function managedModelId(profile) {
+  return profile.omlxModel ?? profile.ollamaModel ?? profile.modelAlias ?? profile.label;
+}
+
+async function ensureManagedModelAvailableForBenchmark(profile, backend) {
+  if (backend.type !== "managed-server") return;
+  if (await modelAvailableOnServer(profile)) return;
+  throw new Error(`${managedModelId(profile)} is not available on ${backend.label} at ${profile.baseUrl}.`);
+}
+
 async function ensureServerForBenchmark(profile) {
   const backend = backendFor(profile.backend);
   if (await serverReady(profile.baseUrl)) {
+    await ensureManagedModelAvailableForBenchmark(profile, backend);
     console.log(pc.green(`[ready] ${backend.label} at ${profile.baseUrl}`));
     return { started: false };
   }
@@ -52,6 +63,7 @@ export async function runPreparedBenchmark(profile, runDirectory, options = {})
     options.signal.addEventListener("abort", () => controller.abort(), { once: true });
   }
   let serverStarted = false;
+  let benchmarkStarted = false;
   let metadata = null;
 
   const onSigint = () => {
@@ -72,6 +84,7 @@ export async function runPreparedBenchmark(profile, runDirectory, options = {})
       await syncPiConfig(profile);
     }
 
+    benchmarkStarted = true;
     const runResult = await runBenchmarkInPi(profile, runDirectory, { signal: controller.signal });
 
     let speedMetrics = null;
@@ -110,11 +123,13 @@ export async function runPreparedBenchmark(profile, runDirectory, options = {})
         console.log(result.stopped ? pc.green(`[stop] ${result.message}`) : pc.dim(`[stop] ${result.message}`));
       }
     }
-    const unloadResult = await unloadModelFromServer(profile);
-    if (!unloadResult.unloaded && unloadResult.error) {
-      console.log(pc.yellow(`[unload] ${unloadResult.backend}: ${unloadResult.error}`));
-    } else if (!unloadResult.unloaded && unloadResult.reason) {
-      console.log(pc.dim(`[unload] ${unloadResult.backend}: ${unloadResult.reason}`));
+    if (benchmarkStarted) {
+      const unloadResult = await unloadModelFromServer(profile);
+      if (!unloadResult.unloaded && unloadResult.error) {
+        console.log(pc.yellow(`[unload] ${unloadResult.backend}: ${unloadResult.error}`));
+      } else if (!unloadResult.unloaded && unloadResult.reason) {
+        console.log(pc.dim(`[unload] ${unloadResult.backend}: ${unloadResult.reason}`));
+      }
     }
   }
 

package/src/process.mjs

@@ -209,7 +209,7 @@ export async function waitForReady(profile, pid, rawLogPath) {
 
 // ── Internals ──────────────────────────────────────────────────────────────
 
-async function serverModelIds(baseUrl) {
+export async function serverModelIds(baseUrl) {
   const result = await fetchJson(`${baseUrl.replace(/\/+$/u, "")}/models`);
   if (!result.ok) return [];
   return (Array.isArray(result.data?.data) ? result.data.data : [])