@hawkeyexl/inference 0.3.1 → 0.4.0

package/dist/index.js

@@ -1,3 +1,8 @@
+import {
+  WORKER_ENV_FLAG,
+  openBackend
+} from "./chunk-UFHBIS5K.js";
+
 // src/types.ts
 var InferenceError = class extends Error {
   constructor(message) {
@@ -596,6 +601,11 @@ function isModelPathOrUri(model) {
   return /^(hf|huggingface):/i.test(model) || /^https?:\/\//i.test(model) || /^(hf|huggingface)\.co\//i.test(model) || model.endsWith(".gguf");
 }
 
+// src/providers/llama-host.ts
+import { fork } from "child_process";
+import { existsSync as existsSync3 } from "fs";
+import { fileURLToPath } from "url";
+
 // src/providers/llama-install.ts
 import {
   existsSync as existsSync2,
@@ -616,6 +626,9 @@ var LOCK_STALE_MS = INSTALL_TIMEOUT_MS + 6e4;
 function defaultLlamaRuntimeDirectory(env = process.env) {
   return env["INFERENCE_RUNTIME_DIR"] || join3(homedir2(), ".hawkeyexl-inference", "runtime");
 }
+function nodeLlamaCppShimUrl(directory = defaultLlamaRuntimeDirectory()) {
+  return pathToFileURL(join3(directory, SHIM)).href;
+}
 function isModuleNotFound(e) {
   const code = e?.code;
   return code === "ERR_MODULE_NOT_FOUND" || code === "MODULE_NOT_FOUND";
@@ -786,6 +799,554 @@ function delay(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+// src/providers/llama-host.ts
+var LLAMA_GPUS = ["auto", "cuda", "vulkan", "metal", false];
+function isLlamaGpu(value) {
+  return LLAMA_GPUS.includes(value);
+}
+var STDERR_TAIL_LINES = 20;
+var SHUTDOWN_GRACE_MS = 5e3;
+var STDERR_DRAIN_MS = 250;
+var GPU_OFF_VALUES = ["false", "off", "none", "disable", "disabled"];
+var LlamaWorkerCrash = class extends Error {
+  constructor(host, code, signal, blameless = false) {
+    super("The local-model worker exited.");
+    this.host = host;
+    this.code = code;
+    this.signal = signal;
+    this.blameless = blameless;
+    this.name = "LlamaWorkerCrash";
+  }
+  host;
+  code;
+  signal;
+  blameless;
+};
+var WorkerHost = class {
+  child;
+  /** The backend that initialised, once it has. */
+  gpu;
+  /** The backend initialising now, so a crash during `init` is attributable. */
+  trying;
+  dead = false;
+  /** How a crash of this host was handled, decided once for every caller. */
+  verdict;
+  exited;
+  closing = false;
+  nextId = 1;
+  pending = /* @__PURE__ */ new Map();
+  stderrTail = [];
+  partialLine = "";
+  constructor(entry) {
+    this.child = fork(entry, [], {
+      // stderr is piped rather than inherited so a crash can name its last
+      // line; it is still forwarded live, so llama.cpp's output is not lost.
+      stdio: ["ignore", "inherit", "pipe", "ipc"],
+      serialization: "json",
+      // Never the parent's flags (an inspector port, a test runner's loader).
+      // From `src/` Node strips the worker's types itself; say nothing about it.
+      execArgv: entry.endsWith(".ts") ? ["--disable-warning=ExperimentalWarning"] : [],
+      env: { ...process.env, [WORKER_ENV_FLAG]: "1" }
+    });
+    const stderr = this.child.stderr;
+    stderr.setEncoding("utf8");
+    stderr.on("data", (chunk) => {
+      process.stderr.write(chunk);
+      this.collect(chunk);
+    });
+    this.child.on("message", (message) => this.onMessage(message));
+    this.exited = new Promise((resolve) => {
+      this.child.once("error", (e) => {
+        this.dead = true;
+        this.rejectAll(
+          new InferenceError(`Could not start the local-model worker (${e.message}).`)
+        );
+        resolve();
+      });
+      this.child.once("exit", (code, signal) => {
+        this.dead = true;
+        void this.drainStderr().then(() => {
+          this.onExit(code, signal);
+          resolve();
+        });
+      });
+    });
+    this.idle();
+  }
+  get pid() {
+    return this.dead ? void 0 : this.child.pid;
+  }
+  get stderr() {
+    return this.partialLine ? [...this.stderrTail, this.partialLine] : this.stderrTail;
+  }
+  request(request) {
+    if (this.dead) {
+      return Promise.reject(new LlamaWorkerCrash(this, null, null, true));
+    }
+    const id = this.nextId++;
+    return new Promise((resolve, reject) => {
+      this.pending.set(id, {
+        resolve,
+        reject
+      });
+      if (this.pending.size === 1) this.busy();
+      this.child.send({ ...request, id }, (e) => {
+        if (e && this.pending.delete(id)) {
+          if (this.pending.size === 0) this.idle();
+          reject(new LlamaWorkerCrash(this, null, null, true));
+        }
+      });
+    });
+  }
+  async shutdown() {
+    if (this.dead) return;
+    this.closing = true;
+    this.busy();
+    try {
+      this.child.send({ id: this.nextId++, op: "shutdown" });
+    } catch {
+    }
+    const timer = setTimeout(() => this.child.kill(), SHUTDOWN_GRACE_MS);
+    await this.exited;
+    clearTimeout(timer);
+  }
+  onMessage(message) {
+    if ("event" in message) {
+      this.trying = message.gpu;
+      return;
+    }
+    const pending = this.pending.get(message.id);
+    if (!pending) return;
+    this.pending.delete(message.id);
+    if (this.pending.size === 0) this.idle();
+    if (message.ok) {
+      pending.resolve(message.value);
+    } else {
+      pending.reject(rebuildError(message.error));
+    }
+  }
+  onExit(code, signal) {
+    if (this.pending.size === 0) return;
+    this.rejectAll(
+      this.closing ? new InferenceError(
+        "The local-model worker was shut down by disposeLlamaModels while a call was still running."
+      ) : new LlamaWorkerCrash(this, code, signal)
+    );
+  }
+  rejectAll(reason) {
+    const pending = [...this.pending.values()];
+    this.pending.clear();
+    this.idle();
+    for (const p of pending) p.reject(reason);
+  }
+  collect(chunk) {
+    const lines = (this.partialLine + chunk).split(/\r?\n/);
+    this.partialLine = lines.pop() ?? "";
+    this.stderrTail.push(...lines);
+    this.stderrTail.splice(0, Math.max(0, this.stderrTail.length - STDERR_TAIL_LINES));
+  }
+  /** `exit` can arrive before the last of stderr; give it a moment. */
+  drainStderr() {
+    const stderr = this.child.stderr;
+    if (stderr.readableEnded || stderr.destroyed) return Promise.resolve();
+    return new Promise((resolve) => {
+      const timer = setTimeout(resolve, STDERR_DRAIN_MS);
+      timer.unref();
+      const done = () => {
+        clearTimeout(timer);
+        resolve();
+      };
+      stderr.once("end", done);
+      stderr.once("close", done);
+    });
+  }
+  /**
+   * Hold the parent open only while it is waiting on the worker. An idle
+   * worker must not keep a consumer's process alive after its work is done —
+   * and when that process exits, the worker sees `disconnect` and exits too.
+   */
+  busy() {
+    this.child.ref();
+    this.child.channel?.ref();
+    this.child.stderr?.ref?.();
+  }
+  idle() {
+    this.child.unref();
+    this.child.channel?.unref();
+    this.child.stderr?.unref?.();
+  }
+};
+function rebuildError(error) {
+  if (error.name === "InferenceError") return new InferenceError(error.message);
+  const rebuilt = new Error(error.message);
+  rebuilt.name = error.name;
+  return rebuilt;
+}
+function backendName(gpu) {
+  switch (gpu) {
+    case "cuda":
+      return "CUDA";
+    case "vulkan":
+      return "Vulkan";
+    case "metal":
+      return "Metal";
+    default:
+      return "CPU";
+  }
+}
+var GGML_ABORT_LINE = /\.(?:cu|cpp|cc|c|h|m|mm):\d+: \S/;
+function describeExit(crash) {
+  const how = crash.signal ? `signal ${crash.signal}` : `exit code ${String(crash.code)}`;
+  const lines = crash.host.stderr.map((l) => l.trim()).filter((l) => l !== "");
+  const line = [...lines].reverse().find((l) => GGML_ABORT_LINE.test(l)) ?? lines.find((l) => /error|abort|assert|fatal/i.test(l)) ?? lines[lines.length - 1];
+  return line ? `${how}: ${line}` : how;
+}
+var failedBackends = /* @__PURE__ */ new Map();
+var Slot = class {
+  constructor(gpu, source, entry) {
+    this.gpu = gpu;
+    this.source = source;
+    this.entry = entry;
+  }
+  gpu;
+  source;
+  entry;
+  host;
+  starting;
+  /** The crash a new worker is replacing, so its start can say what changed. */
+  switchedFrom;
+  get pid() {
+    return this.host?.pid;
+  }
+  /**
+   * Run `fn` against a live worker, retrying on the next backend when the
+   * worker crashes under it. Ordinary errors pass through untouched.
+   */
+  async run(fn) {
+    for (; ; ) {
+      try {
+        return await fn(await this.acquire());
+      } catch (e) {
+        if (!(e instanceof LlamaWorkerCrash)) throw e;
+        const verdict = e.host.verdict ??= this.decide(e);
+        if (verdict instanceof Error) throw verdict;
+      }
+    }
+  }
+  async shutdown() {
+    const host = this.host;
+    this.host = void 0;
+    this.starting = void 0;
+    await host?.shutdown();
+  }
+  acquire() {
+    if (this.starting && !this.host?.dead) return this.starting;
+    const starting = (async () => {
+      const source = await backendSource();
+      const host = new WorkerHost(this.entry);
+      this.host = host;
+      const { gpu } = await host.request({
+        op: "init",
+        moduleUrl: source.moduleUrl,
+        options: this.options()
+      });
+      host.gpu = gpu;
+      if (this.switchedFrom) {
+        warnSwitch(this.switchedFrom, gpu);
+        this.switchedFrom = void 0;
+      }
+      return host;
+    })();
+    this.starting = starting;
+    starting.catch((e) => {
+      if (e instanceof LlamaWorkerCrash || this.starting !== starting) return;
+      const host = this.host;
+      this.starting = void 0;
+      this.host = void 0;
+      void host?.shutdown();
+    });
+    return starting;
+  }
+  options() {
+    if (this.gpu !== "auto") return { gpu: this.gpu };
+    const exclude = [...failedBackends.keys()];
+    return exclude.length > 0 ? { gpu: { type: "auto", exclude }, build: "never" } : { gpu: "auto" };
+  }
+  decide(crash) {
+    if (this.host === crash.host) {
+      this.host = void 0;
+      this.starting = void 0;
+    }
+    if (crash.blameless) return "retry";
+    const gpu = crash.host.gpu ?? crash.host.trying;
+    const exit = describeExit(crash);
+    if (gpu === void 0) {
+      return new InferenceError(
+        `llama.cpp's local-model worker crashed before it chose a backend (${exit}). The request was not answered.`
+      );
+    }
+    const name = backendName(gpu);
+    if (this.gpu !== "auto") {
+      return new InferenceError(
+        `llama.cpp's ${name} backend crashed the local-model worker (${exit}). ${name} was chosen explicitly (${this.source ?? "llamaCpp.gpu"}), so the library did not switch backends. Choose another \u2014 ${alternativesTo(gpu)} \u2014 or unset it to let the library fall back on its own.`
+      );
+    }
+    if (gpu === false) {
+      const all = [...failedBackends].map(([g, e]) => `${backendName(g)} (${e})`);
+      return new InferenceError(
+        `llama.cpp crashed the local-model worker on every backend this machine offers \u2014 ${[...all, `CPU (${exit})`].join(", ")}. The request was not answered.`
+      );
+    }
+    failedBackends.set(gpu, exit);
+    this.switchedFrom = { gpu, exit };
+    return "retry";
+  }
+};
+function alternativesTo(gpu) {
+  switch (gpu) {
+    case "cuda":
+      return "NODE_LLAMA_CPP_GPU=vulkan, or false for the CPU";
+    case "vulkan":
+      return "NODE_LLAMA_CPP_GPU=cuda, or false for the CPU";
+    case "metal":
+      return "NODE_LLAMA_CPP_GPU=false for the CPU";
+    default:
+      return "NODE_LLAMA_CPP_GPU=auto for a GPU backend";
+  }
+}
+function warnSwitch(from, to) {
+  const crashed = `inference: llama.cpp's ${backendName(from.gpu)} backend crashed the local-model worker (${from.exit}).`;
+  if (to === false) {
+    console.warn(
+      `${crashed} Retrying on the CPU, which is much slower \u2014 expect minutes per call \u2014 and staying there for the rest of this process. Pin a backend with NODE_LLAMA_CPP_GPU or llamaCpp.gpu to fail fast instead.`
+    );
+    return;
+  }
+  const name = backendName(to);
+  console.warn(
+    `${crashed} Retrying on ${name}, and staying on ${name} for the rest of this process. Set NODE_LLAMA_CPP_GPU=${to} (or llamaCpp.gpu: "${to}") to start there.`
+  );
+}
+function requestedGpu(option, env = process.env) {
+  if (option !== void 0) {
+    return option === "auto" ? { gpu: "auto" } : { gpu: option, source: `llamaCpp.gpu: ${JSON.stringify(option)}` };
+  }
+  const raw = env["NODE_LLAMA_CPP_GPU"];
+  if (raw == null || raw === "" || raw === "auto") return { gpu: "auto" };
+  if (GPU_OFF_VALUES.includes(raw)) {
+    return { gpu: false, source: `NODE_LLAMA_CPP_GPU=${raw}` };
+  }
+  if (raw === "cuda" || raw === "vulkan" || raw === "metal") {
+    return { gpu: raw, source: `NODE_LLAMA_CPP_GPU=${raw}` };
+  }
+  return { gpu: "auto" };
+}
+var slots = /* @__PURE__ */ new Map();
+function slotFor(gpu, source, entry) {
+  const key = JSON.stringify([gpu, source ?? null]);
+  let slot = slots.get(key);
+  if (!slot) {
+    slot = new Slot(gpu, source, entry);
+    slots.set(key, slot);
+  }
+  return slot;
+}
+var sourceOverride;
+var defaultSource;
+function backendSource() {
+  if (sourceOverride) return Promise.resolve(sourceOverride);
+  return defaultSource ??= nodeLlamaCppSource().catch((e) => {
+    defaultSource = void 0;
+    throw e;
+  });
+}
+async function nodeLlamaCppSource() {
+  let mod;
+  let moduleUrl = "node-llama-cpp";
+  try {
+    mod = await import("node-llama-cpp");
+  } catch (e) {
+    if (!isModuleNotFound(e)) {
+      throw new InferenceError(
+        `node-llama-cpp is installed but failed to load (${e instanceof Error ? e.message : String(e)}). This is the copy resolved from your own node_modules, so reinstalling it here will not help \u2014 check the Node version and the platform build.`
+      );
+    }
+    mod = await importNodeLlamaCpp();
+    moduleUrl = nodeLlamaCppShimUrl();
+  }
+  return {
+    moduleUrl,
+    // `directory` is this library's own, not node-llama-cpp's global default —
+    // owning it is what makes `clearLlamaModels` safe.
+    resolveModelFile: (uri, directory) => mod.resolveModelFile(uri, { directory })
+  };
+}
+function workerEntry() {
+  for (const name of ["llama-worker.js", "llama-worker.ts"]) {
+    const path = fileURLToPath(new URL(`./${name}`, import.meta.url));
+    if (existsSync3(path)) return path;
+  }
+  return void 0;
+}
+var WorkerModelProxy = class {
+  constructor(slot, path) {
+    this.slot = slot;
+    this.path = path;
+  }
+  slot;
+  path;
+  trainContextSize;
+  ids = /* @__PURE__ */ new Map();
+  async open() {
+    const loaded = await this.slot.run((host) => this.on(host));
+    this.trainContextSize = loaded.trainContextSize;
+  }
+  /** This model's id in `host`, loading it there first if a fallback moved us. */
+  on(host) {
+    for (const known of this.ids.keys()) if (known.dead) this.ids.delete(known);
+    let loaded = this.ids.get(host);
+    if (!loaded) {
+      loaded = host.request({ op: "loadModel", path: this.path });
+      this.ids.set(host, loaded);
+      const settled = loaded;
+      settled.catch(() => {
+        if (this.ids.get(host) === settled) this.ids.delete(host);
+      });
+    }
+    return loaded;
+  }
+  countTokens(text) {
+    return this.slot.run(
+      async (host) => host.request({
+        op: "countTokens",
+        modelId: (await this.on(host)).modelId,
+        text
+      })
+    );
+  }
+  async createSession(systemPrompt, contextSize) {
+    const session = new WorkerSessionProxy(this.slot, this, systemPrompt, contextSize);
+    await session.open();
+    return {
+      contextSize: session.contextSize,
+      prompt: (text, options) => session.prompt(text, options),
+      dispose: () => session.dispose()
+    };
+  }
+  async dispose() {
+    await Promise.all(
+      [...this.ids].map(async ([host, loaded]) => {
+        if (host.dead) return;
+        const { modelId } = await loaded;
+        await host.request({ op: "disposeModel", modelId });
+      }).map((p) => p.catch(() => void 0))
+    );
+    this.ids.clear();
+  }
+};
+var WorkerSessionProxy = class {
+  constructor(slot, model, systemPrompt, requestedSize) {
+    this.slot = slot;
+    this.model = model;
+    this.systemPrompt = systemPrompt;
+    this.requestedSize = requestedSize;
+  }
+  slot;
+  model;
+  systemPrompt;
+  requestedSize;
+  contextSize;
+  ids = /* @__PURE__ */ new Map();
+  async open() {
+    await this.slot.run((host) => this.on(host));
+  }
+  /** This session's id in `host`. A retried prompt gets a fresh one there. */
+  on(host) {
+    let id = this.ids.get(host);
+    if (!id) {
+      id = (async () => {
+        const opened = await host.request({
+          op: "createSession",
+          modelId: (await this.model.on(host)).modelId,
+          systemPrompt: this.systemPrompt,
+          ...this.requestedSize != null ? { contextSize: this.requestedSize } : {}
+        });
+        this.contextSize = opened.contextSize;
+        return opened.sessionId;
+      })();
+      this.ids.set(host, id);
+      const settled = id;
+      settled.catch(() => {
+        if (this.ids.get(host) === settled) this.ids.delete(host);
+      });
+    }
+    return id;
+  }
+  prompt(text, options) {
+    return this.slot.run(
+      async (host) => host.request({
+        op: "prompt",
+        sessionId: await this.on(host),
+        text,
+        options
+      })
+    );
+  }
+  async dispose() {
+    await Promise.all(
+      [...this.ids].map(async ([host, id]) => {
+        if (host.dead) return;
+        await host.request({ op: "disposeSession", sessionId: await id });
+      }).map((p) => p.catch(() => void 0))
+    );
+    this.ids.clear();
+  }
+};
+var warnedInProcess = false;
+function inProcessRuntime(gpu) {
+  if (!warnedInProcess) {
+    warnedInProcess = true;
+    console.warn(
+      `inference: the local-model worker (llama-worker.js) is missing beside this library \u2014 was it bundled? Running llama.cpp in-process instead, so a native crash in llama.cpp will end this process.`
+    );
+  }
+  let opened;
+  const backend = () => opened ??= backendSource().then(
+    async (source) => openBackend(await import(source.moduleUrl), { gpu }, { trying: () => void 0 })
+  ).catch((e) => {
+    opened = void 0;
+    throw e;
+  });
+  return {
+    resolveModelFile: (uri, directory) => backendSource().then((source) => source.resolveModelFile(uri, directory)),
+    loadModel: (path) => backend().then((b) => b.loadModel(path)),
+    getMemoryBudgetBytes: () => backend().then((b) => b.memoryBudget())
+  };
+}
+function createWorkerRuntime(options = {}) {
+  const { gpu, source } = requestedGpu(options.gpu);
+  const entry = workerEntry();
+  if (!entry) return inProcessRuntime(gpu);
+  const slot = slotFor(gpu, source, entry);
+  return {
+    resolveModelFile: (uri, directory) => backendSource().then((s) => s.resolveModelFile(uri, directory)),
+    async loadModel(path) {
+      const model = new WorkerModelProxy(slot, path);
+      await model.open();
+      return {
+        trainContextSize: model.trainContextSize,
+        countTokens: (text) => model.countTokens(text),
+        createSession: (systemPrompt, contextSize) => model.createSession(systemPrompt, contextSize),
+        dispose: () => model.dispose()
+      };
+    },
+    getMemoryBudgetBytes: () => slot.run((host) => host.request({ op: "memoryBudget" }))
+  };
+}
+async function shutdownLlamaWorkers() {
+  await Promise.all([...slots.values()].map((slot) => slot.shutdown()));
+}
+
 // src/providers/llama-cpp.ts
 var loadedModels = /* @__PURE__ */ new Map();
 async function disposeLlamaModels() {
@@ -794,7 +1355,11 @@ async function disposeLlamaModels() {
   await Promise.all(
     pending.map((p) => p.then((m) => m.dispose()).catch(() => void 0))
   );
+  await shutdownLlamaWorkers();
 }
+var DEFAULT_CONTEXT_SIZE = 8192;
+var CHAT_TEMPLATE_OVERHEAD_TOKENS = 512;
+var DEFAULT_RESPONSE_RESERVE_TOKENS = 2048;
 var LlamaCppProvider = class {
   constructor(model, options = {}) {
     this.model = model;
@@ -803,8 +1368,19 @@ var LlamaCppProvider = class {
         `llama-cpp model "${model}" is a selector. Constructing a provider directly needs a concrete model (e.g. "${aliasForTier("balanced")}") \u2014 use makeProviderAsync to resolve a selector against this machine.`
       );
     }
+    if (options.contextSize !== void 0 && !(Number.isInteger(options.contextSize) && options.contextSize > 0)) {
+      throw new InferenceError(
+        `llamaCpp.contextSize must be a positive integer number of tokens, got ${String(options.contextSize)}.`
+      );
+    }
+    if (options.gpu !== void 0 && !isLlamaGpu(options.gpu)) {
+      throw new InferenceError(
+        `llamaCpp.gpu must be "auto", "cuda", "vulkan", "metal" or false, got ${JSON.stringify(options.gpu) ?? String(options.gpu)}.`
+      );
+    }
+    this.contextSize = options.contextSize;
     this.uri = resolveLlamaModelRef(model);
-    this.runtime = options.runtime ?? defaultLlamaRuntime();
+    this.runtime = options.runtime ?? defaultLlamaRuntime(options.gpu !== void 0 ? { gpu: options.gpu } : {});
     this.thoughtTokens = options.thoughtTokens ?? 0;
     this.maxTokens = options.maxTokens;
     this.modelsDirectory = options.modelsDirectory ?? defaultLlamaModelsDirectory();
@@ -815,6 +1391,7 @@ var LlamaCppProvider = class {
   runtime;
   thoughtTokens;
   maxTokens;
+  contextSize;
   modelsDirectory;
   /**
    * Loaded-model key: the same URI in two directories is two different files.
@@ -831,14 +1408,24 @@ var LlamaCppProvider = class {
   }
   async completeJSON(req) {
     const model = await this.load();
-    const session = await model.createSession(systemPromptFor(req));
+    const systemPrompt = systemPromptFor(req);
+    const plan = await this.contextFor(model, systemPrompt, req.user);
+    const session = await model.createSession(systemPrompt, plan.contextSize);
+    const contextSize = session.contextSize ?? plan.contextSize;
+    const implicitMaxTokens = this.maxTokens == null && plan.promptTokens != null ? contextSize - plan.promptTokens : void 0;
+    const maxTokens = this.maxTokens ?? implicitMaxTokens;
     try {
       const result = await session.prompt(req.user, {
         schema: req.schema,
         temperature: req.temperature,
         thoughtTokens: this.thoughtTokens,
-        ...this.maxTokens != null ? { maxTokens: this.maxTokens } : {}
+        ...maxTokens != null ? { maxTokens } : {}
       });
+      if (result.stopReason === "maxTokens" && implicitMaxTokens != null) {
+        throw new Error(
+          `llama-cpp generation filled the ${contextSize}-token context before completing the JSON \u2014 raise llamaCpp.contextSize, or set llamaCpp.maxTokens to bound the response.`
+        );
+      }
       if (result.stopReason === "maxTokens") {
         throw new Error(
           `llama-cpp generation hit the token limit before completing the JSON${this.maxTokens != null ? ` (maxTokens: ${this.maxTokens})` : ""} \u2014 raise llamaCpp.maxTokens, or shorten the prompt if the context is full.`
@@ -849,6 +1436,39 @@ var LlamaCppProvider = class {
       await session.dispose().catch(() => void 0);
     }
   }
+  /**
+   * The context this call needs: both prompts in the model's own tokens, the
+   * chat template's overhead, and room for the response. A prompt that does not
+   * fit is refused here, before anything is created. llama.cpp would otherwise
+   * shift the overflow out of the context and answer a prompt nobody sent.
+   */
+  async contextFor(model, systemPrompt, user) {
+    const ceiling = model.trainContextSize;
+    const fallback = this.contextSize ?? (ceiling != null ? Math.min(DEFAULT_CONTEXT_SIZE, ceiling) : DEFAULT_CONTEXT_SIZE);
+    if (!model.countTokens) return { contextSize: fallback };
+    const [system, prompt] = await Promise.all([
+      model.countTokens(systemPrompt),
+      model.countTokens(user)
+    ]);
+    const response = (this.maxTokens ?? DEFAULT_RESPONSE_RESERVE_TOKENS) + this.thoughtTokens;
+    const promptTokens = system + prompt + CHAT_TEMPLATE_OVERHEAD_TOKENS;
+    const needed = promptTokens + response;
+    const counted = `Counted: system ${system} + user ${prompt} + ${CHAT_TEMPLATE_OVERHEAD_TOKENS} chat-template overhead + ${response} for the response.`;
+    if (this.contextSize != null) {
+      if (needed > this.contextSize) {
+        throw new InferenceError(
+          `llama-cpp prompt needs ${needed} tokens of context, more than llamaCpp.contextSize (${this.contextSize}). ${counted} Raise llamaCpp.contextSize, ${this.maxTokens != null ? "lower" : "set"} llamaCpp.maxTokens, or leave contextSize unset so the context is sized to the prompt.`
+        );
+      }
+      return { contextSize: this.contextSize, promptTokens };
+    }
+    if (ceiling != null && needed > ceiling) {
+      throw new InferenceError(
+        `llama-cpp prompt needs ${needed} tokens of context, more than this model's training context of ${ceiling} tokens. ${counted} Shorten the prompt${this.maxTokens != null ? ", or lower llamaCpp.maxTokens" : ""}.`
+      );
+    }
+    return { contextSize: Math.max(fallback, needed), promptTokens };
+  }
   load() {
     const existing = loadedModels.get(this.cacheKey);
     if (existing) return existing;
@@ -893,95 +1513,8 @@ function restoreOpenBrace(text) {
     return text;
   }
 }
-var runtimePromise;
-function defaultLlamaRuntime() {
-  const real = () => (
-    // Drop a failed init so the next call retries. A GPU that failed to
-    // initialise, or a binary still being extracted by a concurrent install,
-    // must not poison the runtime for the rest of the process — the same rule
-    // `load()` applies to weights.
-    runtimePromise ??= loadNodeLlamaCpp().catch((e) => {
-      runtimePromise = void 0;
-      throw e;
-    })
-  );
-  return {
-    resolveModelFile: (uri, directory) => real().then((r) => r.resolveModelFile(uri, directory)),
-    loadModel: (path) => real().then((r) => r.loadModel(path)),
-    getMemoryBudgetBytes: () => real().then((r) => r.getMemoryBudgetBytes())
-  };
-}
-async function loadNodeLlamaCpp() {
-  let mod;
-  try {
-    mod = await import("node-llama-cpp");
-  } catch (e) {
-    if (!isModuleNotFound(e)) {
-      throw new InferenceError(
-        `node-llama-cpp is installed but failed to load (${e instanceof Error ? e.message : String(e)}). This is the copy resolved from your own node_modules, so reinstalling it here will not help \u2014 check the Node version and the platform build.`
-      );
-    }
-    mod = await importNodeLlamaCpp();
-  }
-  const { getLlama, resolveModelFile, LlamaChatSession, TokenMeter } = mod;
-  const llama = await getLlama();
-  return {
-    // `directory` is this library's own, not node-llama-cpp's global default —
-    // owning it is what makes `clearLlamaModels` safe.
-    resolveModelFile: (uri, directory) => resolveModelFile(uri, { directory }),
-    async loadModel(path) {
-      const model = await llama.loadModel({ modelPath: path });
-      return {
-        async createSession(systemPrompt) {
-          const context = await model.createContext();
-          const sequence = context.getSequence();
-          const session = new LlamaChatSession({
-            contextSequence: sequence,
-            systemPrompt
-          });
-          return {
-            async prompt(text, options) {
-              const grammar = await llama.createGrammarForJsonSchema(
-                options.schema
-              );
-              const before = sequence.tokenMeter.getState();
-              const result = await session.promptWithMeta(text, {
-                grammar,
-                temperature: options.temperature,
-                budgets: { thoughtTokens: options.thoughtTokens },
-                ...options.maxTokens != null ? { maxTokens: options.maxTokens } : {}
-              });
-              const diff = TokenMeter.diff(sequence.tokenMeter, before);
-              return {
-                text: result.responseText,
-                stopReason: result.stopReason,
-                usage: {
-                  inputTokens: diff.usedInputTokens,
-                  outputTokens: diff.usedOutputTokens
-                }
-              };
-            },
-            async dispose() {
-              await context.dispose();
-            }
-          };
-        },
-        async dispose() {
-          await model.dispose();
-        }
-      };
-    },
-    async getMemoryBudgetBytes() {
-      const { totalmem } = await import("os");
-      const ramBudget = totalmem() / 2;
-      try {
-        const vram = await llama.getVramState();
-        return Math.max(vram.free, ramBudget);
-      } catch {
-        return ramBudget;
-      }
-    }
-  };
+function defaultLlamaRuntime(options = {}) {
+  return createWorkerRuntime(options);
 }
 
 // src/providers/detect.ts
@@ -1141,7 +1674,7 @@ async function resolveProviderIdentityAsync(spec) {
   if (provider !== "llama-cpp" || !isLlamaSelector(model)) {
     return resolveProviderIdentity(resolved);
   }
-  const tier = model === "auto" ? await probeTier(llamaRuntimeFor(spec)) : model;
+  const tier = model === "auto" ? await probeTier(llamaRuntimeFor(spec), spec.llamaCpp?.gpu) : model;
   return { provider, model: aliasForTier(tier) };
 }
 function warnIfDownloadPending(spec, model) {
@@ -1155,8 +1688,8 @@ function warnIfDownloadPending(spec, model) {
 function llamaRuntimeFor(spec) {
   return spec.llamaRuntime ?? spec.llamaCpp?.runtime;
 }
-async function probeTier(runtime) {
-  const source = runtime ?? defaultLlamaRuntime();
+async function probeTier(runtime, gpu) {
+  const source = runtime ?? defaultLlamaRuntime(gpu !== void 0 ? { gpu } : {});
   return tierForBudget(await source.getMemoryBudgetBytes());
 }
 function makeProvider(spec) {