@hawkeyexl/inference 0.3.1 → 0.4.0

package/dist/llama-cpp-CdVKP63Z.d.ts

@@ -0,0 +1,308 @@
+/**
+ * The provider contract. A provider turns a (system, user, schema) request
+ * into schema-conforming JSON. `provider()` and `modelName()` feed cache keys
+ * and pricing lookups, so two providers/models never share a cached result.
+ *
+ * This is deliberately the narrowest useful surface: no streaming, no
+ * multi-turn, no tool loops. Everything downstream of it — judging,
+ * extraction, classification — is schema-constrained single-shot completion.
+ */
+interface CompleteJSONRequest {
+    system: string;
+    user: string;
+    /** JSON Schema the response must conform to. */
+    schema: Record<string, unknown>;
+    temperature: number;
+}
+interface TokenUsage {
+    inputTokens: number;
+    outputTokens: number;
+}
+interface CompleteJSONResponse {
+    json: unknown;
+    /** Absent when the provider does not report usage (e.g. the Claude CLI). */
+    usage?: TokenUsage;
+}
+interface InferenceProvider {
+    /** Stable provider id — feeds cache keys. */
+    provider(): string;
+    /** Model id — feeds cache keys and pricing. */
+    modelName(): string;
+    completeJSON(req: CompleteJSONRequest): Promise<CompleteJSONResponse>;
+}
+interface ExecResult {
+    code: number | null;
+    stdout: string;
+    stderr: string;
+    timedOut: boolean;
+    /** Set when the process could not be spawned (e.g. binary not found). */
+    spawnError?: string;
+}
+interface ExecOptions {
+    cwd?: string;
+    timeoutMs?: number;
+    /**
+     * Overrides on the ambient environment. A key mapped to `undefined`
+     * *unsets* that variable for the child rather than passing it through —
+     * Node omits undefined-valued keys when it builds the child's environment.
+     * Clearing inherited state (`GIT_*`, say) needs this, so the value type is
+     * deliberately wider than `string`.
+     */
+    env?: Record<string, string | undefined>;
+    /** Text piped to the child's stdin (stdin is closed after writing). */
+    input?: string;
+}
+/** Injectable process-execution seam — subprocess providers take one for tests. */
+type ExecFn = (cmd: string[], opts?: ExecOptions) => Promise<ExecResult>;
+
+/** A llama.cpp backend, as node-llama-cpp names it; `false` is the CPU. */
+type WorkerGpu = "metal" | "cuda" | "vulkan" | false;
+/** What `getLlama` is asked for: one backend, or the best one not excluded. */
+type WorkerGpuRequest = "auto" | WorkerGpu | {
+    type: "auto";
+    exclude: WorkerGpu[];
+};
+interface WorkerBackendOptions {
+    gpu: WorkerGpuRequest;
+    /**
+     * `"never"` on a fallback, so switching backends uses a prebuilt binary or
+     * fails — it never starts a multi-minute CMake build mid-run.
+     */
+    build?: "never";
+}
+interface WorkerBackendHooks {
+    /**
+     * Names the backend about to initialise, before it does. A crash inside
+     * initialisation is then attributable, so the fallback knows what to skip.
+     */
+    trying(gpu: WorkerGpu): void;
+}
+interface WorkerSession {
+    readonly contextSize?: number;
+    prompt(text: string, options: LlamaPromptOptions): Promise<LlamaPromptResult>;
+    dispose(): Promise<void>;
+}
+interface WorkerModel {
+    readonly trainContextSize?: number;
+    countTokens(text: string): number;
+    createSession(systemPrompt: string, contextSize?: number): Promise<WorkerSession>;
+    dispose(): Promise<void>;
+}
+interface WorkerBackend {
+    /** The backend that actually initialised. */
+    readonly gpu: WorkerGpu;
+    /** Memory available for weights, in bytes — see `getMemoryBudgetBytes`. */
+    memoryBudget(): Promise<number>;
+    loadModel(path: string): Promise<WorkerModel>;
+}
+/**
+ * Open a backend from an imported module: node-llama-cpp itself, or — in the
+ * test suite — a module exporting `createWorkerBackend`, which stands in for
+ * the inference while the process, the IPC, and the crash stay real.
+ */
+declare function openBackend(mod: unknown, options: WorkerBackendOptions, hooks: WorkerBackendHooks): Promise<WorkerBackend>;
+/** One request from the parent. Every one gets exactly one reply. */
+type WorkerRequest = {
+    id: number;
+} & ({
+    op: "init";
+    moduleUrl: string;
+    options: WorkerBackendOptions;
+} | {
+    op: "memoryBudget";
+} | {
+    op: "loadModel";
+    path: string;
+} | {
+    op: "countTokens";
+    modelId: number;
+    text: string;
+} | {
+    op: "createSession";
+    modelId: number;
+    systemPrompt: string;
+    contextSize?: number;
+} | {
+    op: "prompt";
+    sessionId: number;
+    text: string;
+    options: LlamaPromptOptions;
+} | {
+    op: "disposeSession";
+    sessionId: number;
+} | {
+    op: "disposeModel";
+    modelId: number;
+} | {
+    op: "shutdown";
+});
+type WorkerMessage = {
+    id: number;
+    ok: true;
+    value: unknown;
+} | {
+    id: number;
+    ok: false;
+    error: {
+        name: string;
+        message: string;
+    };
+} | {
+    event: "trying";
+    gpu: WorkerGpu;
+};
+/** Set by the parent at fork time, so importing this file never serves. */
+declare const WORKER_ENV_FLAG = "INFERENCE_LLAMA_WORKER";
+
+/**
+ * Which llama.cpp backend to run on. `"auto"` picks the best one this machine
+ * has and falls back from one that crashes; anything else pins it.
+ */
+type LlamaGpu = "auto" | WorkerGpu;
+
+interface LlamaPromptOptions {
+    /** JSON Schema converted to a GBNF grammar by the runtime. */
+    schema: Record<string, unknown>;
+    temperature: number;
+    /** Thinking budget; 0 disables it. See the note in `completeJSON`. */
+    thoughtTokens: number;
+    maxTokens?: number;
+}
+interface LlamaPromptResult {
+    text: string;
+    usage?: TokenUsage;
+    /**
+     * Why generation stopped. `"maxTokens"` means the output was cut off, so the
+     * text is almost certainly truncated JSON — see the guard in `completeJSON`.
+     */
+    stopReason?: string;
+}
+interface LlamaSession {
+    prompt(text: string, options: LlamaPromptOptions): Promise<LlamaPromptResult>;
+    dispose(): Promise<void>;
+    /**
+     * Tokens of context the runtime actually created. llama.cpp may round a
+     * requested size up to a multiple of 256. Optional: a fake has no context.
+     */
+    readonly contextSize?: number;
+}
+interface LlamaLoadedModel {
+    /**
+     * Open a single-turn session on a fresh context of `contextSize` tokens.
+     * The provider always passes it, sized from the prompt it is about to send;
+     * the real runtime treats an absent size as the 8192-token default.
+     */
+    createSession(systemPrompt: string, contextSize?: number): Promise<LlamaSession>;
+    dispose(): Promise<void>;
+    /**
+     * The context length the model was trained on, the most a context can
+     * usefully hold. Optional so a runtime written before it existed still
+     * satisfies the seam; without it the provider assumes no ceiling.
+     */
+    readonly trainContextSize?: number;
+    /**
+     * Count `text` in this model's own tokens. Optional for the same reason;
+     * without it the provider uses the default size and cannot check fit. May
+     * be async: the real runtime's tokenizer lives in the worker process.
+     */
+    countTokens?(text: string): number | Promise<number>;
+}
+/**
+ * The whole of `node-llama-cpp` that this provider uses. Kept this narrow so a
+ * test fake is a few lines and so the real adapter is the only place that
+ * knows the upstream API shape.
+ */
+interface LlamaRuntime {
+    /**
+     * Resolve an `hf:` URI or path to a local file inside `directory`,
+     * downloading if needed.
+     */
+    resolveModelFile(uri: string, directory: string): Promise<string>;
+    loadModel(path: string): Promise<LlamaLoadedModel>;
+    /** Memory available for weights, in bytes — VRAM if there is a GPU, else RAM. */
+    getMemoryBudgetBytes(): Promise<number>;
+}
+interface LlamaCppProviderOptions {
+    /** Injected for tests; defaults to the real `node-llama-cpp` adapter. */
+    runtime?: LlamaRuntime;
+    /**
+     * Thinking budget in tokens, default 0.
+     *
+     * Gemma 4 has a thinking mode, but a grammar constrains generation from
+     * token 0 — so an unbudgeted model starts reasoning and gets cut off
+     * mid-thought. Zero is the deterministic choice for judging; raise it if you
+     * want reasoning before the JSON.
+     */
+    thoughtTokens?: number;
+    maxTokens?: number;
+    /**
+     * A fixed context size in tokens, used for every call.
+     *
+     * Unset, the context is sized to the work: 8192 tokens, or more when the
+     * prompt and its response reserve need it, up to the model's training
+     * context. Either way, a prompt that does not fit fails with an
+     * `InferenceError` rather than being truncated. ADR 01011.
+     */
+    contextSize?: number;
+    /**
+     * Where to download and look for weights. Defaults to this library's own
+     * directory — see `defaultLlamaModelsDirectory`.
+     */
+    modelsDirectory?: string;
+    /**
+     * The llama.cpp backend: `"cuda"`, `"vulkan"`, `"metal"`, or `false` for the
+     * CPU. Unset, `NODE_LLAMA_CPP_GPU` decides, and failing that `"auto"`.
+     *
+     * `"auto"` picks the best backend this machine has and, if it crashes the
+     * local-model worker, retries on the next — CUDA, then Vulkan, then the CPU —
+     * for the rest of the process. A named backend is never replaced: a crash on
+     * it is an error. Ignored when `runtime` is injected. ADR 01012.
+     */
+    gpu?: LlamaGpu;
+}
+/**
+ * Free every loaded model.
+ *
+ * A standalone function rather than a `dispose()` on `InferenceProvider`:
+ * adding one to the contract would make all five providers carry a lifecycle
+ * only this one has. Short-lived processes can skip it.
+ */
+declare function disposeLlamaModels(): Promise<void>;
+declare class LlamaCppProvider implements InferenceProvider {
+    private readonly model;
+    private readonly uri;
+    private readonly runtime;
+    private readonly thoughtTokens;
+    private readonly maxTokens;
+    private readonly contextSize;
+    private readonly modelsDirectory;
+    /**
+     * Loaded-model key: the same URI in two directories is two different files.
+     * Built with `buildCacheKey` so its parts are length-prefixed — a plain join
+     * would let two different (directory, uri) pairs collide and hand a provider
+     * back the wrong weights.
+     */
+    private readonly cacheKey;
+    constructor(model: string, options?: LlamaCppProviderOptions);
+    provider(): string;
+    modelName(): string;
+    completeJSON(req: CompleteJSONRequest): Promise<CompleteJSONResponse>;
+    /**
+     * The context this call needs: both prompts in the model's own tokens, the
+     * chat template's overhead, and room for the response. A prompt that does not
+     * fit is refused here, before anything is created. llama.cpp would otherwise
+     * shift the overflow out of the context and answer a prompt nobody sent.
+     */
+    private contextFor;
+    private load;
+}
+/**
+ * The real runtime: node-llama-cpp in a worker process, falling back from a
+ * GPU backend that crashes it. Lazy — constructing a provider for a
+ * fully-cached run starts no process and loads no native binary.
+ */
+declare function defaultLlamaRuntime(options?: {
+    gpu?: LlamaGpu;
+}): LlamaRuntime;
+
+export { type CompleteJSONRequest as C, type ExecFn as E, type InferenceProvider as I, type LlamaCppProviderOptions as L, type TokenUsage as T, WORKER_ENV_FLAG as W, type CompleteJSONResponse as a, type LlamaRuntime as b, type ExecOptions as c, type ExecResult as d, LlamaCppProvider as e, type LlamaGpu as f, type LlamaLoadedModel as g, type LlamaPromptOptions as h, type LlamaPromptResult as i, type LlamaSession as j, defaultLlamaRuntime as k, disposeLlamaModels as l, type WorkerBackend as m, type WorkerBackendHooks as n, type WorkerBackendOptions as o, type WorkerGpu as p, type WorkerGpuRequest as q, type WorkerMessage as r, type WorkerModel as s, type WorkerRequest as t, type WorkerSession as u, openBackend as v };

package/dist/llama-worker.d.ts

@@ -0,0 +1 @@
+export { W as WORKER_ENV_FLAG, m as WorkerBackend, n as WorkerBackendHooks, o as WorkerBackendOptions, p as WorkerGpu, q as WorkerGpuRequest, r as WorkerMessage, s as WorkerModel, t as WorkerRequest, u as WorkerSession, v as openBackend } from './llama-cpp-CdVKP63Z.js';

package/dist/llama-worker.js

@@ -0,0 +1,9 @@
+import {
+  WORKER_ENV_FLAG,
+  openBackend
+} from "./chunk-UFHBIS5K.js";
+export {
+  WORKER_ENV_FLAG,
+  openBackend
+};
+//# sourceMappingURL=llama-worker.js.map

package/dist/llama-worker.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@hawkeyexl/inference",
-  "version": "0.3.1",
-  "description": "Shared TypeScript LLM inference layer: schema-constrained completion across Anthropic, OpenAI-compatible, Claude CLI, and in-process local llama.cpp providers, with caching, cost accounting, and an LLM-as-judge ensemble.",
+  "version": "0.4.0",
+  "description": "Shared TypeScript LLM inference layer: schema-constrained completion across Anthropic, OpenAI-compatible, Claude CLI, and local llama.cpp providers, with caching, cost accounting, and an LLM-as-judge ensemble.",
   "type": "module",
   "main": "dist/index.js",
   "module": "dist/index.js",