@hawkeyexl/inference 0.1.0 → 0.2.0

package/dist/index.js

@@ -6,6 +6,22 @@ var InferenceError = class extends Error {
   }
 };
 
+// src/runtime.ts
+var MINIMUM_NODE_MAJOR = 24;
+var warnedNodeVersion = false;
+function warnIfUnsupportedNode(version = process.versions.node) {
+  if (warnedNodeVersion) return;
+  const major = Number.parseInt(version, 10);
+  if (!Number.isInteger(major) || major >= MINIMUM_NODE_MAJOR) return;
+  warnedNodeVersion = true;
+  console.warn(
+    `inference: running on Node ${version}, older than the Node ${MINIMUM_NODE_MAJOR} this package requires. npm only warns about that at install time (EBADENGINE), so nothing has stopped you yet \u2014 upgrade Node, or expect failures this library cannot explain.`
+  );
+}
+function resetNodeVersionWarning() {
+  warnedNodeVersion = false;
+}
+
 // src/providers/anthropic.ts
 import Anthropic from "@anthropic-ai/sdk";
 var DEFAULT_TOOL_NAME = "record_result";
@@ -331,7 +347,15 @@ var ClaudeCliProvider = class {
         `Claude CLI exited ${result.code}: ${result.stderr.trim().slice(-300)}`
       );
     }
-    const wrapper = JSON.parse(result.stdout);
+    let wrapper;
+    try {
+      wrapper = JSON.parse(result.stdout);
+    } catch {
+      const excerpt = result.stdout.trim().replace(/\s+/g, " ").slice(0, 200);
+      throw new Error(
+        `Claude CLI printed non-JSON output (is it logged in?): ${excerpt || "(no output)"}`
+      );
+    }
     if (typeof wrapper.result !== "string") {
       throw new Error("Claude CLI returned no result field");
     }
@@ -384,12 +408,671 @@ function mockVerdict(match, confidence, overrides = {}) {
   };
 }
 
+// src/cache.ts
+import { createHash } from "crypto";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
+import { join } from "path";
+function sha256(text) {
+  return createHash("sha256").update(text, "utf8").digest("hex");
+}
+function buildCacheKey(parts) {
+  return sha256(parts.map((p) => `${p.length}:${p}`).join("|"));
+}
+var JsonCache = class {
+  constructor(dir, enabled = true, label = "inference") {
+    this.dir = dir;
+    this.enabled = enabled;
+    this.label = label;
+  }
+  dir;
+  enabled;
+  label;
+  /** Cache-write failures warn once per process, not once per entry. */
+  warned = false;
+  get(key) {
+    if (!this.enabled) return void 0;
+    const path = join(this.dir, `${key}.json`);
+    if (!existsSync(path)) return void 0;
+    try {
+      return JSON.parse(readFileSync(path, "utf8"));
+    } catch {
+      return void 0;
+    }
+  }
+  set(key, value) {
+    if (!this.enabled) return;
+    try {
+      mkdirSync(this.dir, { recursive: true });
+      writeFileSync(
+        join(this.dir, `${key}.json`),
+        JSON.stringify(value, null, 2)
+      );
+    } catch (e) {
+      if (!this.warned) {
+        this.warned = true;
+        console.warn(
+          `${this.label}: could not write the cache at ${this.dir} (${e instanceof Error ? e.message : String(e)}). Continuing without caching.`
+        );
+      }
+    }
+  }
+};
+
+// src/providers/llama-models.ts
+import { readdirSync } from "fs";
+import { homedir } from "os";
+import { join as join2 } from "path";
+function defaultLlamaModelsDirectory() {
+  return process.env["INFERENCE_MODELS_DIR"] || join2(homedir(), ".hawkeyexl-inference", "models");
+}
+var LLAMA_TIERS = ["fast", "balanced", "quality"];
+var LLAMA_SELECTORS = ["auto", ...LLAMA_TIERS];
+var LLAMA_MODELS = deepFreezeEntries({
+  "gemma-4-e2b": {
+    uri: "hf:unsloth/gemma-4-E2B-it-qat-GGUF/gemma-4-E2B-it-qat-UD-Q4_K_XL.gguf",
+    sizeBytes: 2620370976,
+    license: "Apache-2.0",
+    tier: "fast",
+    notes: "IFEval 94.6. Smallest Gemma 4; the floor for this family."
+  },
+  "gemma-4-e4b": {
+    uri: "hf:unsloth/gemma-4-E4B-it-qat-GGUF/gemma-4-E4B-it-qat-UD-Q4_K_XL.gguf",
+    sizeBytes: 4215695776,
+    license: "Apache-2.0",
+    tier: "balanced",
+    notes: "IFEval 96.7. The default for most machines."
+  },
+  "gemma-4-12b": {
+    uri: "hf:unsloth/gemma-4-12B-it-qat-GGUF/gemma-4-12B-it-qat-UD-Q4_K_XL.gguf",
+    sizeBytes: 6716356800,
+    license: "Apache-2.0",
+    tier: "quality",
+    notes: "IFEval 97.2. Dense 12B; wants a GPU or plenty of RAM."
+  },
+  "gemma-4-26b-a4b": {
+    uri: "hf:unsloth/gemma-4-26B-A4B-it-qat-GGUF/gemma-4-26B-A4B-it-qat-UD-Q4_K_XL.gguf",
+    sizeBytes: 14249047104,
+    license: "Apache-2.0",
+    notes: "MoE: 25.2B total, 3.8B active \u2014 infers near E4B speed if it fits in memory."
+  },
+  "gemma-4-e2b-q2": {
+    uri: "hf:unsloth/gemma-4-E2B-it-qat-GGUF/gemma-4-E2B-it-qat-UD-Q2_K_XL.gguf",
+    sizeBytes: 2186186784,
+    license: "Apache-2.0",
+    notes: "Q2 build of the fast tier; smallest download, lowest fidelity."
+  }
+});
+function deepFreezeEntries(catalog) {
+  for (const entry of Object.values(catalog)) Object.freeze(entry);
+  return Object.freeze(catalog);
+}
+var TIER_ALIAS = {
+  fast: "gemma-4-e2b",
+  balanced: "gemma-4-e4b",
+  quality: "gemma-4-12b"
+};
+function isLlamaSelector(model) {
+  return LLAMA_SELECTORS.includes(model);
+}
+var MEMORY_HEADROOM = 3.5;
+function tierForBudget(budgetBytes) {
+  let chosen = "fast";
+  for (const tier of LLAMA_TIERS) {
+    const entry = LLAMA_MODELS[TIER_ALIAS[tier]];
+    if (entry.sizeBytes * MEMORY_HEADROOM <= budgetBytes) chosen = tier;
+  }
+  return chosen;
+}
+function aliasForTier(tier) {
+  return TIER_ALIAS[tier];
+}
+function uriForTier(tier) {
+  return LLAMA_MODELS[TIER_ALIAS[tier]].uri;
+}
+function resolveLlamaModelRef(model) {
+  if (isLlamaSelector(model)) {
+    throw new InferenceError(
+      `llama-cpp model "${model}" is a selector and needs a hardware probe to resolve. Use resolveProviderIdentityAsync/makeProviderAsync, or name a concrete model (e.g. "gemma-4-e4b").`
+    );
+  }
+  const entry = LLAMA_MODELS[model];
+  if (entry) return entry.uri;
+  if (isModelPathOrUri(model)) return model;
+  throw new InferenceError(
+    `Unknown llama-cpp model "${model}". Use a selector (${LLAMA_SELECTORS.join(
+      ", "
+    )}), a curated alias (${Object.keys(LLAMA_MODELS).join(
+      ", "
+    )}), an hf: URI, or a path to a .gguf file.`
+  );
+}
+function blobNameFor(model) {
+  const ref = resolveLlamaModelRef(model).split("#")[0];
+  return ref.split(/[/\\]/).pop();
+}
+function matchesModelBlob(entry, blobName) {
+  const base = entry.replace(/\.ipull$/, "");
+  if (base.endsWith(blobName)) return true;
+  const stem = blobName.replace(/\.gguf$/, "");
+  return new RegExp(`${escapeRegExp(stem)}-\\d{5}-of-\\d{5}\\.gguf$`).test(base);
+}
+function escapeRegExp(text) {
+  return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function isModelDownloaded(model, directory) {
+  const blobName = blobNameFor(model);
+  return listModelDirectory(directory).some(
+    (entry) => !entry.endsWith(".ipull") && matchesModelBlob(entry, blobName)
+  );
+}
+function listModelDirectory(directory) {
+  try {
+    return readdirSync(directory, { withFileTypes: true }).filter((entry) => entry.isFile()).map((entry) => entry.name);
+  } catch {
+    return [];
+  }
+}
+function isModelPathOrUri(model) {
+  return /^(hf|huggingface):/i.test(model) || /^https?:\/\//i.test(model) || /^(hf|huggingface)\.co\//i.test(model) || model.endsWith(".gguf");
+}
+
+// src/providers/llama-install.ts
+import {
+  existsSync as existsSync2,
+  mkdirSync as mkdirSync2,
+  rmSync,
+  statSync,
+  writeFileSync as writeFileSync2
+} from "fs";
+import { homedir as homedir2 } from "os";
+import { join as join3 } from "path";
+import { pathToFileURL } from "url";
+var PACKAGE_SPEC = "node-llama-cpp@^3.19.0";
+var SHIM = "loader.mjs";
+var LOCK = ".install.lock";
+var INSTALL_TIMEOUT_MS = 9e5;
+var LOCK_WAIT_MS = INSTALL_TIMEOUT_MS;
+var LOCK_STALE_MS = INSTALL_TIMEOUT_MS + 6e4;
+function defaultLlamaRuntimeDirectory(env = process.env) {
+  return env["INFERENCE_RUNTIME_DIR"] || join3(homedir2(), ".hawkeyexl-inference", "runtime");
+}
+function isModuleNotFound(e) {
+  const code = e?.code;
+  return code === "ERR_MODULE_NOT_FOUND" || code === "MODULE_NOT_FOUND";
+}
+function describe(e) {
+  return e instanceof Error ? e.message : String(e);
+}
+async function nodeLlamaCppStatus(options = {}) {
+  const env = options.env ?? process.env;
+  const directory = options.directory ?? defaultLlamaRuntimeDirectory(env);
+  const probeImport = options.probeImport ?? (() => import("node-llama-cpp"));
+  try {
+    await probeImport();
+    return { state: "present" };
+  } catch (e) {
+    if (!isModuleNotFound(e)) {
+      return {
+        state: "refused",
+        reason: `node-llama-cpp is installed but failed to load (${describe(e)})`
+      };
+    }
+  }
+  if (existsSync2(join3(directory, SHIM))) return { state: "present" };
+  if ((env["INFERENCE_NO_AUTO_INSTALL"] ?? "") !== "") {
+    return {
+      state: "refused",
+      reason: `node-llama-cpp is not installed and INFERENCE_NO_AUTO_INSTALL is set`
+    };
+  }
+  return { state: "installable", directory };
+}
+var installs = /* @__PURE__ */ new Map();
+var warnedInstall = false;
+function resetRuntimeInstall() {
+  installs.clear();
+  warnedInstall = false;
+}
+function importNodeLlamaCpp(options = {}) {
+  const env = options.env ?? process.env;
+  const directory = options.directory ?? defaultLlamaRuntimeDirectory(env);
+  const existing = installs.get(directory);
+  if (existing) return existing;
+  const pending = fromPrefix(directory, env, options);
+  const guarded = pending.catch((e) => {
+    if (installs.get(directory) === guarded) installs.delete(directory);
+    throw e;
+  });
+  installs.set(directory, guarded);
+  return guarded;
+}
+async function fromPrefix(directory, env, options) {
+  const importShim = options.importShim ?? ((url) => import(url));
+  const shim = join3(directory, SHIM);
+  if (existsSync2(shim)) return importShim(pathToFileURL(shim).href);
+  if ((env["INFERENCE_NO_AUTO_INSTALL"] ?? "") !== "") {
+    throw new InferenceError(
+      `The llama-cpp provider needs node-llama-cpp, and INFERENCE_NO_AUTO_INSTALL is set. Install it yourself (npm i ${PACKAGE_SPEC}), unset INFERENCE_NO_AUTO_INSTALL to allow installing into ${directory}, or name a different provider.`
+    );
+  }
+  mkdirSync2(directory, { recursive: true });
+  await withLock(directory, async () => {
+    if (existsSync2(shim)) return;
+    warnInstalling(directory);
+    await runInstall(directory, env, options);
+    writeFileSync2(shim, `export * from "node-llama-cpp";
+`, "utf8");
+  });
+  return importShim(pathToFileURL(shim).href);
+}
+async function runInstall(directory, env, options) {
+  const manifest = join3(directory, "package.json");
+  if (!existsSync2(manifest)) {
+    writeFileSync2(
+      manifest,
+      `${JSON.stringify(
+        {
+          name: "hawkeyexl-inference-runtime",
+          version: "0.0.0",
+          private: true,
+          description: "Auto-installed runtime for @hawkeyexl/inference. Safe to delete."
+        },
+        null,
+        2
+      )}
+`,
+      "utf8"
+    );
+  }
+  const exec = options.exec ?? realExec;
+  const result = await exec(
+    [
+      "npm",
+      "install",
+      "--prefix",
+      directory,
+      PACKAGE_SPEC,
+      "--no-audit",
+      "--no-fund"
+    ],
+    { timeoutMs: options.timeoutMs ?? INSTALL_TIMEOUT_MS, env }
+  );
+  if (result.spawnError != null) {
+    throw new InferenceError(
+      `Could not run npm to install node-llama-cpp (${result.spawnError}). Install it yourself with: npm i ${PACKAGE_SPEC}, or name a provider that does not need it.`
+    );
+  }
+  if (result.timedOut) {
+    throw new InferenceError(
+      `Installing node-llama-cpp into ${directory} timed out. A source build can take a while \u2014 retry, raise the timeout, or install it yourself with: npm i ${PACKAGE_SPEC}.`
+    );
+  }
+  if (result.code !== 0) {
+    throw new InferenceError(
+      `Installing node-llama-cpp into ${directory} failed (exit ${String(
+        result.code
+      )}).
+${tail(result.stderr || result.stdout)}
+Install it yourself with: npm i ${PACKAGE_SPEC}, or name a provider that does not need it.`
+    );
+  }
+}
+function tail(output, lines = 12) {
+  return output.trimEnd().split(/\r?\n/).slice(-lines).join("\n");
+}
+function warnInstalling(directory) {
+  if (warnedInstall) return;
+  warnedInstall = true;
+  console.warn(
+    `inference: node-llama-cpp is not installed \u2014 fetching it into ${directory} so the local model can run. This is a one-time native install; set INFERENCE_NO_AUTO_INSTALL=1 to refuse it, or name a provider that does not need it.`
+  );
+}
+async function withLock(directory, fn) {
+  const lock = join3(directory, LOCK);
+  const deadline = Date.now() + LOCK_WAIT_MS;
+  for (; ; ) {
+    try {
+      writeFileSync2(lock, String(process.pid), { flag: "wx" });
+      break;
+    } catch (e) {
+      if (e.code !== "EEXIST") throw e;
+      if (ageOf(lock) > LOCK_STALE_MS) {
+        rmSync(lock, { force: true });
+        continue;
+      }
+      if (existsSync2(join3(directory, SHIM))) return;
+      if (Date.now() > deadline) {
+        throw new InferenceError(
+          `Timed out waiting for another process to install node-llama-cpp into ${directory}. If nothing else is running, remove ${lock} and retry.`
+        );
+      }
+      await delay(250);
+    }
+  }
+  try {
+    await fn();
+  } finally {
+    rmSync(lock, { force: true });
+  }
+}
+function ageOf(path) {
+  try {
+    return Date.now() - statSync(path).mtimeMs;
+  } catch {
+    return Number.POSITIVE_INFINITY;
+  }
+}
+function delay(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+// src/providers/llama-cpp.ts
+var loadedModels = /* @__PURE__ */ new Map();
+async function disposeLlamaModels() {
+  const pending = [...loadedModels.values()];
+  loadedModels.clear();
+  await Promise.all(
+    pending.map((p) => p.then((m) => m.dispose()).catch(() => void 0))
+  );
+}
+var LlamaCppProvider = class {
+  constructor(model, options = {}) {
+    this.model = model;
+    if (isLlamaSelector(model)) {
+      throw new InferenceError(
+        `llama-cpp model "${model}" is a selector. Constructing a provider directly needs a concrete model (e.g. "gemma-4-e4b") \u2014 use makeProviderAsync to resolve a selector against this machine.`
+      );
+    }
+    this.uri = resolveLlamaModelRef(model);
+    this.runtime = options.runtime ?? defaultLlamaRuntime();
+    this.thoughtTokens = options.thoughtTokens ?? 0;
+    this.maxTokens = options.maxTokens;
+    this.modelsDirectory = options.modelsDirectory ?? defaultLlamaModelsDirectory();
+    this.cacheKey = buildCacheKey([this.modelsDirectory, this.uri]);
+  }
+  model;
+  uri;
+  runtime;
+  thoughtTokens;
+  maxTokens;
+  modelsDirectory;
+  /**
+   * Loaded-model key: the same URI in two directories is two different files.
+   * Built with `buildCacheKey` so its parts are length-prefixed — a plain join
+   * would let two different (directory, uri) pairs collide and hand a provider
+   * back the wrong weights.
+   */
+  cacheKey;
+  provider() {
+    return "llama-cpp";
+  }
+  modelName() {
+    return this.model;
+  }
+  async completeJSON(req) {
+    const model = await this.load();
+    const session = await model.createSession(systemPromptFor(req));
+    try {
+      const result = await session.prompt(req.user, {
+        schema: req.schema,
+        temperature: req.temperature,
+        thoughtTokens: this.thoughtTokens,
+        ...this.maxTokens != null ? { maxTokens: this.maxTokens } : {}
+      });
+      if (result.stopReason === "maxTokens") {
+        throw new Error(
+          `llama-cpp generation hit the token limit before completing the JSON${this.maxTokens != null ? ` (maxTokens: ${this.maxTokens})` : ""} \u2014 raise llamaCpp.maxTokens, or shorten the prompt if the context is full.`
+        );
+      }
+      return { json: extractJson(result.text), usage: result.usage };
+    } finally {
+      await session.dispose().catch(() => void 0);
+    }
+  }
+  load() {
+    const existing = loadedModels.get(this.cacheKey);
+    if (existing) return existing;
+    const pending = (async () => {
+      const path = await this.runtime.resolveModelFile(
+        this.uri,
+        this.modelsDirectory
+      );
+      return this.runtime.loadModel(path);
+    })();
+    const guarded = pending.catch((e) => {
+      if (loadedModels.get(this.cacheKey) === guarded) {
+        loadedModels.delete(this.cacheKey);
+      }
+      throw e;
+    });
+    loadedModels.set(this.cacheKey, guarded);
+    return guarded;
+  }
+};
+function systemPromptFor(req) {
+  return `${req.system}
+
+Respond with ONLY a JSON object conforming to this JSON Schema:
+${JSON.stringify(
+    req.schema
+  )}`;
+}
+var runtimePromise;
+function defaultLlamaRuntime() {
+  const real = () => (
+    // Drop a failed init so the next call retries. A GPU that failed to
+    // initialise, or a binary still being extracted by a concurrent install,
+    // must not poison the runtime for the rest of the process — the same rule
+    // `load()` applies to weights.
+    runtimePromise ??= loadNodeLlamaCpp().catch((e) => {
+      runtimePromise = void 0;
+      throw e;
+    })
+  );
+  return {
+    resolveModelFile: (uri, directory) => real().then((r) => r.resolveModelFile(uri, directory)),
+    loadModel: (path) => real().then((r) => r.loadModel(path)),
+    getMemoryBudgetBytes: () => real().then((r) => r.getMemoryBudgetBytes())
+  };
+}
+async function loadNodeLlamaCpp() {
+  let mod;
+  try {
+    mod = await import("node-llama-cpp");
+  } catch (e) {
+    if (!isModuleNotFound(e)) {
+      throw new InferenceError(
+        `node-llama-cpp is installed but failed to load (${e instanceof Error ? e.message : String(e)}). This is the copy resolved from your own node_modules, so reinstalling it here will not help \u2014 check the Node version and the platform build.`
+      );
+    }
+    mod = await importNodeLlamaCpp();
+  }
+  const { getLlama, resolveModelFile, LlamaChatSession, TokenMeter } = mod;
+  const llama = await getLlama();
+  return {
+    // `directory` is this library's own, not node-llama-cpp's global default —
+    // owning it is what makes `clearLlamaModels` safe.
+    resolveModelFile: (uri, directory) => resolveModelFile(uri, { directory }),
+    async loadModel(path) {
+      const model = await llama.loadModel({ modelPath: path });
+      return {
+        async createSession(systemPrompt) {
+          const context = await model.createContext();
+          const sequence = context.getSequence();
+          const session = new LlamaChatSession({
+            contextSequence: sequence,
+            systemPrompt
+          });
+          return {
+            async prompt(text, options) {
+              const grammar = await llama.createGrammarForJsonSchema(
+                options.schema
+              );
+              const before = sequence.tokenMeter.getState();
+              const result = await session.promptWithMeta(text, {
+                grammar,
+                temperature: options.temperature,
+                budgets: { thoughtTokens: options.thoughtTokens },
+                ...options.maxTokens != null ? { maxTokens: options.maxTokens } : {}
+              });
+              const diff = TokenMeter.diff(sequence.tokenMeter, before);
+              return {
+                text: result.responseText,
+                stopReason: result.stopReason,
+                usage: {
+                  inputTokens: diff.usedInputTokens,
+                  outputTokens: diff.usedOutputTokens
+                }
+              };
+            },
+            async dispose() {
+              await context.dispose();
+            }
+          };
+        },
+        async dispose() {
+          await model.dispose();
+        }
+      };
+    },
+    async getMemoryBudgetBytes() {
+      const { totalmem } = await import("os");
+      const ramBudget = totalmem() / 2;
+      try {
+        const vram = await llama.getVramState();
+        return Math.max(vram.free, ramBudget);
+      } catch {
+        return ramBudget;
+      }
+    }
+  };
+}
+
+// src/providers/detect.ts
+var DETECTION_ORDER = [
+  "anthropic",
+  "openai",
+  "claude-cli",
+  "llama-cpp"
+];
+var DEFAULT_KEY_ENV = {
+  anthropic: "ANTHROPIC_API_KEY",
+  openai: "OPENAI_API_KEY"
+};
+function hasKey(provider) {
+  return (process.env[DEFAULT_KEY_ENV[provider]] ?? "") !== "";
+}
+var cliProbes = /* @__PURE__ */ new Map();
+function resetClaudeCliProbe() {
+  cliProbes.clear();
+}
+function probeClaudeCli(spec) {
+  const exec = spec.exec ?? realExec;
+  const command = spec.command ?? "claude";
+  const cached = cliProbes.get(command);
+  if (cached) return cached;
+  const probing = exec([command, "--version"], { timeoutMs: 1e4 }).then((r) => r.code === 0 && !r.timedOut && r.spawnError == null).catch(() => false);
+  cliProbes.set(command, probing);
+  return probing;
+}
+async function probeLlamaCpp(spec) {
+  const injected = spec.llamaRuntime ?? spec.llamaCpp?.runtime;
+  if (injected) return budgetProbe(injected);
+  const status = await nodeLlamaCppStatus();
+  if (status.state === "refused") {
+    return { available: false, reason: status.reason };
+  }
+  if (status.state === "installable") {
+    return { available: true };
+  }
+  return budgetProbe(defaultLlamaRuntime());
+}
+function budgetProbe(runtime) {
+  return runtime.getMemoryBudgetBytes().then(
+    () => ({ available: true }),
+    (e) => ({
+      available: false,
+      reason: e instanceof Error && /node-llama-cpp/.test(e.message) ? "node-llama-cpp is not installed (npm i node-llama-cpp)" : `node-llama-cpp could not start (${e instanceof Error ? e.message : String(e)})`
+    })
+  );
+}
+async function probe(provider, spec) {
+  switch (provider) {
+    case "anthropic":
+      return hasKey("anthropic") ? { available: true } : {
+        available: false,
+        reason: `ANTHROPIC_API_KEY is not set`
+      };
+    case "openai":
+      return hasKey("openai") || spec.baseUrl ? { available: true } : {
+        available: false,
+        reason: `OPENAI_API_KEY is not set and no baseUrl was given`
+      };
+    case "claude-cli":
+      return await probeClaudeCli(spec) ? { available: true } : {
+        available: false,
+        reason: `could not run \`${spec.command ?? "claude"}\` (is the Claude CLI installed?)`
+      };
+    case "llama-cpp":
+      return probeLlamaCpp(spec);
+    default:
+      return { available: false, reason: "not auto-selectable" };
+  }
+}
+async function availableProviders(spec = {}) {
+  const probes = await Promise.all(
+    DETECTION_ORDER.map((name) => probe(name, spec))
+  );
+  return DETECTION_ORDER.filter((_, i) => probes[i].available);
+}
+async function detectProvider(spec = {}) {
+  const reasons = [];
+  for (const name of DETECTION_ORDER) {
+    const result = await probe(name, spec);
+    if (result.available) {
+      warnSelected(name, spec.provider === "auto");
+      return name;
+    }
+    reasons.push(`  ${name.padEnd(10)} \u2014 ${result.reason}`);
+  }
+  throw new InferenceError(
+    `No inference provider is available. Tried:
+${reasons.join("\n")}
+Pass an explicit \`provider\`, set one of the keys above, or install node-llama-cpp.`
+  );
+}
+var warnedSelection = false;
+var warnedDownload = false;
+function resetProviderDetectionWarning() {
+  warnedSelection = false;
+  warnedDownload = false;
+}
+function warnSelected(provider, wasExplicitAuto) {
+  if (warnedSelection) return;
+  warnedSelection = true;
+  const because = wasExplicitAuto ? `provider "auto"` : "no provider specified";
+  console.warn(
+    `inference: ${because} \u2014 auto-selected "${provider}". Pass an explicit \`provider\` to pin it.`
+  );
+}
+function warnPendingDownload(model, sizeBytes) {
+  if (warnedDownload) return;
+  warnedDownload = true;
+  console.warn(
+    `inference: "${model}" is not downloaded yet \u2014 the first run will fetch ~${(sizeBytes / 1e9).toFixed(2)} GB. Pre-fetch it, or pass an explicit \`provider\` to avoid the local model entirely.`
+  );
+}
+
 // src/providers/index.ts
 var DEFAULT_MODELS = {
   anthropic: "claude-sonnet-4-5",
   openai: "gpt-4o-mini",
   "claude-cli": "claude-sonnet-4-5",
-  mock: "mock-model"
+  mock: "mock-model",
+  // A selector, not a pinned model: which weights a tier points at is then a
+  // catalog change rather than an API change. Resolving it needs the async
+  // factory — see `resolveProviderIdentityAsync`.
+  "llama-cpp": "auto"
 };
 var DEFAULT_API_KEY_ENV = {
   anthropic: "ANTHROPIC_API_KEY",
@@ -397,12 +1080,51 @@ var DEFAULT_API_KEY_ENV = {
 };
 var DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1";
 function resolveProviderIdentity(spec) {
-  return {
-    provider: spec.provider,
-    model: spec.model ?? DEFAULT_MODELS[spec.provider] ?? "unknown"
-  };
+  if (spec.provider == null || spec.provider === "auto") {
+    throw new InferenceError(
+      `No provider specified. Detecting one probes the environment, the Claude CLI and the local model runtime, which cannot be done synchronously \u2014 use resolveProviderIdentityAsync/makeProviderAsync, or name a provider (${Object.keys(DEFAULT_MODELS).join(", ")}).`
+    );
+  }
+  const model = spec.model ?? DEFAULT_MODELS[spec.provider] ?? "unknown";
+  if (spec.provider === "llama-cpp" && isLlamaSelector(model)) {
+    throw new InferenceError(
+      `llama-cpp model "${model}" is a selector and cannot be resolved synchronously \u2014 picking a tier probes GPU memory. Use resolveProviderIdentityAsync/makeProviderAsync, or name a concrete model (e.g. "gemma-4-e4b").`
+    );
+  }
+  return { provider: spec.provider, model };
+}
+async function resolveProviderIdentityAsync(spec) {
+  if ((spec.provider == null || spec.provider === "auto") && spec.model != null) {
+    throw new InferenceError(
+      `Model "${spec.model}" was given without a provider, and a model name does not say which provider owns it. Name the provider too (${Object.keys(DEFAULT_MODELS).join(", ")}), or drop the model to take the detected provider's default.`
+    );
+  }
+  const provider = spec.provider == null || spec.provider === "auto" ? await detectProvider(spec) : spec.provider;
+  const resolved = { ...spec, provider };
+  const model = spec.model ?? DEFAULT_MODELS[provider] ?? "unknown";
+  if (provider !== "llama-cpp" || !isLlamaSelector(model)) {
+    return resolveProviderIdentity(resolved);
+  }
+  const tier = model === "auto" ? await probeTier(llamaRuntimeFor(spec)) : model;
+  return { provider, model: aliasForTier(tier) };
+}
+function warnIfDownloadPending(spec, model) {
+  const entry = LLAMA_MODELS[model];
+  if (!entry) return;
+  const directory = spec.llamaCpp?.modelsDirectory ?? defaultLlamaModelsDirectory();
+  if (!isModelDownloaded(model, directory)) {
+    warnPendingDownload(model, entry.sizeBytes);
+  }
+}
+function llamaRuntimeFor(spec) {
+  return spec.llamaRuntime ?? spec.llamaCpp?.runtime;
+}
+async function probeTier(runtime) {
+  const source = runtime ?? defaultLlamaRuntime();
+  return tierForBudget(await source.getMemoryBudgetBytes());
 }
 function makeProvider(spec) {
+  warnIfUnsupportedNode();
   const { model } = resolveProviderIdentity(spec);
   switch (spec.provider) {
     case "anthropic":
@@ -428,6 +1150,11 @@ function makeProvider(spec) {
       );
     case "mock":
       return new MockProvider(spec.mockResponses ?? [{ json: {} }], model);
+    case "llama-cpp":
+      return new LlamaCppProvider(model, {
+        ...spec.llamaCpp ?? {},
+        ...spec.llamaRuntime ? { runtime: spec.llamaRuntime } : {}
+      });
     default:
       throw new InferenceError(
         `Unknown provider "${String(spec.provider)}". Available: ${Object.keys(
@@ -436,6 +1163,55 @@ function makeProvider(spec) {
       );
   }
 }
+async function makeProviderAsync(spec) {
+  const { provider, model } = await resolveProviderIdentityAsync(spec);
+  if (provider === "llama-cpp") warnIfDownloadPending(spec, model);
+  return makeProvider({ ...spec, provider, model });
+}
+
+// src/providers/llama-clean.ts
+import { rmSync as rmSync2, statSync as statSync2 } from "fs";
+import { join as join4 } from "path";
+async function clearLlamaModels(options = {}) {
+  const directory = options.directory ?? defaultLlamaModelsDirectory();
+  const dryRun = options.dryRun ?? false;
+  const wanted = options.models?.map(blobNameFor);
+  if (!dryRun) await disposeLlamaModels();
+  const files = [];
+  for (const entry of listModelDirectory(directory)) {
+    if (!isModelBlob(entry)) continue;
+    if (wanted && !wanted.some((name) => matchesModelBlob(entry, name))) {
+      continue;
+    }
+    const path = join4(directory, entry);
+    const sizeBytes = sizeOf(path);
+    if (sizeBytes === void 0) continue;
+    if (!dryRun) {
+      try {
+        rmSync2(path);
+      } catch {
+        continue;
+      }
+    }
+    files.push({ path, sizeBytes });
+  }
+  return {
+    files,
+    freedBytes: files.reduce((total, file) => total + file.sizeBytes, 0),
+    directory,
+    dryRun
+  };
+}
+function isModelBlob(entry) {
+  return entry.endsWith(".gguf") || entry.endsWith(".gguf.ipull");
+}
+function sizeOf(path) {
+  try {
+    return statSync2(path).size;
+  } catch {
+    return void 0;
+  }
+}
 
 // src/complete.ts
 import { Ajv2020 } from "ajv/dist/2020.js";
@@ -448,6 +1224,7 @@ function validatorFor(schema) {
   return compiled;
 }
 async function completeValidatedJSON(options) {
+  warnIfUnsupportedNode();
   const {
     provider,
     system,
@@ -488,56 +1265,6 @@ async function completeValidatedJSON(options) {
   return { ...base, error: lastError, durationMs: Date.now() - start };
 }
 
-// src/cache.ts
-import { createHash } from "crypto";
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
-import { join } from "path";
-function sha256(text) {
-  return createHash("sha256").update(text, "utf8").digest("hex");
-}
-function buildCacheKey(parts) {
-  return sha256(parts.map((p) => `${p.length}:${p}`).join("|"));
-}
-var JsonCache = class {
-  constructor(dir, enabled = true, label = "inference") {
-    this.dir = dir;
-    this.enabled = enabled;
-    this.label = label;
-  }
-  dir;
-  enabled;
-  label;
-  /** Cache-write failures warn once per process, not once per entry. */
-  warned = false;
-  get(key) {
-    if (!this.enabled) return void 0;
-    const path = join(this.dir, `${key}.json`);
-    if (!existsSync(path)) return void 0;
-    try {
-      return JSON.parse(readFileSync(path, "utf8"));
-    } catch {
-      return void 0;
-    }
-  }
-  set(key, value) {
-    if (!this.enabled) return;
-    try {
-      mkdirSync(this.dir, { recursive: true });
-      writeFileSync(
-        join(this.dir, `${key}.json`),
-        JSON.stringify(value, null, 2)
-      );
-    } catch (e) {
-      if (!this.warned) {
-        this.warned = true;
-        console.warn(
-          `${this.label}: could not write the cache at ${this.dir} (${e instanceof Error ? e.message : String(e)}). Continuing without caching.`
-        );
-      }
-    }
-  }
-};
-
 // src/cost.ts
 var PRICE_TABLE = {
   "claude-sonnet-4-5": { inputPerMTok: 3, outputPerMTok: 15 },
@@ -693,29 +1420,56 @@ export {
   DEFAULT_MODELS,
   DEFAULT_OPENAI_BASE_URL,
   DEFAULT_ZONES,
+  DETECTION_ORDER,
   InferenceError,
   JsonCache,
+  LLAMA_MODELS,
+  LLAMA_SELECTORS,
+  LLAMA_TIERS,
+  LlamaCppProvider,
   MockProvider,
   OpenAICompatProvider,
   PRICE_TABLE,
   VERDICT_SCHEMA,
+  aliasForTier,
+  availableProviders,
+  blobNameFor,
   buildCacheKey,
+  clearLlamaModels,
   completeValidatedJSON,
   computeConsensus,
   costOfRuns,
   costOfUsage,
+  defaultLlamaModelsDirectory,
+  defaultLlamaRuntime,
+  defaultLlamaRuntimeDirectory,
+  detectProvider,
+  disposeLlamaModels,
   extractJson,
+  importNodeLlamaCpp,
+  isLlamaSelector,
+  isModelDownloaded,
   judge,
   makeProvider,
+  makeProviderAsync,
   mockVerdict,
+  nodeLlamaCppStatus,
   pricingFor,
   realExec,
+  resetClaudeCliProbe,
+  resetNodeVersionWarning,
+  resetProviderDetectionWarning,
+  resetRuntimeInstall,
   resetTemperatureWarning,
+  resolveLlamaModelRef,
   resolveProviderIdentity,
+  resolveProviderIdentityAsync,
   runEnsemble,
   sha256,
   stripNulls,
+  tierForBudget,
   toStrictSchema,
+  uriForTier,
   validatorFor,
   zoneFor
 };