@elizaos/ui 2.0.0-alpha.85 → 2.0.0-beta.1

package/dist/services/local-inference/downloader.js

@@ -0,0 +1,389 @@
+/**
+ * Resumable GGUF downloader.
+ *
+ * Streams directly from HuggingFace to a staging file under
+ * `$STATE_DIR/local-inference/downloads/<id>.part`, then atomically moves
+ * it into `models/<id>.gguf` on success. On restart the staging file is
+ * still there; `resumeIfPossible` sends a Range request starting at the
+ * current partial size.
+ *
+ * Concurrency model: at most one download per model id. Callers use
+ * `subscribe()` to receive progress events; the service facade wires that
+ * to SSE.
+ *
+ * The runtime `fetch` follows HuggingFace redirects and still gives us a body
+ * stream that can be piped into a Node WriteStream.
+ */
+import { randomUUID } from "node:crypto";
+import fs from "node:fs";
+import fsp from "node:fs/promises";
+import path from "node:path";
+import { Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import { ensureDefaultAssignment } from "./assignments.js";
+import { buildHuggingFaceResolveUrl, findCatalogModel } from "./catalog.js";
+import { downloadsStagingDir, elizaModelsDir, localInferenceRoot, } from "./paths.js";
+import { upsertElizaModel } from "./registry.js";
+import { hashFile } from "./verify.js";
+const PROGRESS_THROTTLE_MS = 250;
+const TERMINAL_DOWNLOADS_FILENAME = "download-status.json";
+const TERMINAL_DOWNLOAD_LIMIT = 32;
+async function* readFetchBody(body) {
+    const reader = body.getReader();
+    try {
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                return;
+            if (value)
+                yield Buffer.from(value);
+        }
+    }
+    finally {
+        reader.releaseLock();
+    }
+}
+function stagingFilename(modelId) {
+    // Filename is derived deterministically so repeated download attempts
+    // reuse the same partial file and actually resume.
+    const safe = modelId.replace(/[^a-zA-Z0-9._-]/g, "_");
+    return `${safe}.part`;
+}
+function finalFilename(model) {
+    const safe = model.id.replace(/[^a-zA-Z0-9._-]/g, "_");
+    return `${safe}.gguf`;
+}
+async function ensureDirs() {
+    await fsp.mkdir(downloadsStagingDir(), { recursive: true });
+    await fsp.mkdir(elizaModelsDir(), { recursive: true });
+}
+function terminalDownloadsPath() {
+    return path.join(localInferenceRoot(), TERMINAL_DOWNLOADS_FILENAME);
+}
+async function partialSize(stagingPath) {
+    try {
+        const stat = await fsp.stat(stagingPath);
+        return stat.isFile() ? stat.size : 0;
+    }
+    catch {
+        return 0;
+    }
+}
+export class Downloader {
+    active = new Map();
+    terminal = new Map();
+    listeners = new Set();
+    lastEmit = new Map();
+    constructor() {
+        this.loadTerminalDownloads();
+    }
+    subscribe(listener) {
+        this.listeners.add(listener);
+        return () => {
+            this.listeners.delete(listener);
+        };
+    }
+    snapshot() {
+        const active = [...this.active.values()].map((a) => ({ ...a.job }));
+        const activeIds = new Set(active.map((job) => job.modelId));
+        const terminal = [...this.terminal.values()]
+            .filter((job) => !activeIds.has(job.modelId))
+            .sort((left, right) => right.updatedAt.localeCompare(left.updatedAt))
+            .map((job) => ({ ...job }));
+        return [...active, ...terminal];
+    }
+    isActive(modelId) {
+        const current = this.active.get(modelId);
+        return (!!current &&
+            (current.job.state === "queued" || current.job.state === "downloading"));
+    }
+    /**
+     * Start a download for a model. Accepts either a curated catalog id, or
+     * a full `CatalogModel` spec for ad-hoc HF-search results. Idempotent —
+     * returns the existing job if one is already running for the same id.
+     */
+    async start(modelIdOrSpec) {
+        const catalogEntry = typeof modelIdOrSpec === "string"
+            ? findCatalogModel(modelIdOrSpec)
+            : modelIdOrSpec;
+        if (!catalogEntry) {
+            throw new Error(`Unknown model id: ${typeof modelIdOrSpec === "string" ? modelIdOrSpec : "(no id)"}`);
+        }
+        const modelId = catalogEntry.id;
+        this.clearTerminalDownload(modelId);
+        const existing = this.active.get(modelId);
+        if (existing &&
+            (existing.job.state === "queued" || existing.job.state === "downloading")) {
+            return { ...existing.job };
+        }
+        await ensureDirs();
+        const stagingPath = path.join(downloadsStagingDir(), stagingFilename(modelId));
+        const finalPath = path.join(elizaModelsDir(), finalFilename(catalogEntry));
+        const job = {
+            jobId: randomUUID(),
+            modelId,
+            state: "queued",
+            received: await partialSize(stagingPath),
+            total: Math.round(catalogEntry.sizeGb * 1024 ** 3),
+            bytesPerSec: 0,
+            etaMs: null,
+            startedAt: new Date().toISOString(),
+            updatedAt: new Date().toISOString(),
+        };
+        const abortController = new AbortController();
+        const record = {
+            job,
+            abortController,
+            stagingPath,
+            finalPath,
+        };
+        this.active.set(modelId, record);
+        // Fire-and-forget; errors are captured and emitted as a "failed" event.
+        void this.runJob(catalogEntry, record).catch(() => {
+            // `runJob` handles its own failure telemetry; we only need to swallow
+            // the unhandled-rejection here.
+        });
+        this.emit({ type: "progress", job: { ...job } });
+        return { ...job };
+    }
+    cancel(modelId) {
+        const record = this.active.get(modelId);
+        if (!record)
+            return false;
+        if (record.job.state !== "downloading" && record.job.state !== "queued") {
+            return false;
+        }
+        record.abortController.abort();
+        this.updateState(record, "cancelled");
+        this.rememberTerminalDownload(record.job);
+        this.emit({ type: "cancelled", job: { ...record.job } });
+        this.active.delete(modelId);
+        return true;
+    }
+    emit(event) {
+        for (const listener of this.listeners) {
+            try {
+                listener(event);
+            }
+            catch {
+                // A bad listener must not kill the downloader; drop it silently.
+                this.listeners.delete(listener);
+            }
+        }
+    }
+    updateState(record, state) {
+        record.job.state = state;
+        record.job.updatedAt = new Date().toISOString();
+    }
+    loadTerminalDownloads() {
+        try {
+            const raw = fs.readFileSync(terminalDownloadsPath(), "utf8");
+            const parsed = JSON.parse(raw);
+            if (!parsed || parsed.version !== 1 || !Array.isArray(parsed.jobs)) {
+                return;
+            }
+            for (const job of parsed.jobs) {
+                if (job &&
+                    typeof job.modelId === "string" &&
+                    (job.state === "completed" ||
+                        job.state === "failed" ||
+                        job.state === "cancelled")) {
+                    this.terminal.set(job.modelId, { ...job });
+                }
+            }
+        }
+        catch {
+            // Missing or malformed terminal-download state should not block
+            // local inference. New terminal states will rewrite the file.
+        }
+    }
+    persistTerminalDownloads() {
+        try {
+            fs.mkdirSync(localInferenceRoot(), { recursive: true });
+            const jobs = [...this.terminal.values()]
+                .sort((left, right) => right.updatedAt.localeCompare(left.updatedAt))
+                .slice(0, TERMINAL_DOWNLOAD_LIMIT);
+            const payload = { version: 1, jobs };
+            fs.writeFileSync(terminalDownloadsPath(), JSON.stringify(payload, null, 2), "utf8");
+        }
+        catch {
+            // Terminal status is useful for chat/UI telemetry but is not allowed to
+            // fail the download path.
+        }
+    }
+    rememberTerminalDownload(job) {
+        this.terminal.set(job.modelId, { ...job });
+        const ordered = [...this.terminal.values()].sort((left, right) => right.updatedAt.localeCompare(left.updatedAt));
+        this.terminal.clear();
+        for (const terminalJob of ordered.slice(0, TERMINAL_DOWNLOAD_LIMIT)) {
+            this.terminal.set(terminalJob.modelId, terminalJob);
+        }
+        this.persistTerminalDownloads();
+    }
+    clearTerminalDownload(modelId) {
+        if (!this.terminal.delete(modelId))
+            return;
+        this.persistTerminalDownloads();
+    }
+    throttleEmit(record) {
+        const now = Date.now();
+        const last = this.lastEmit.get(record.job.modelId) ?? 0;
+        if (now - last < PROGRESS_THROTTLE_MS)
+            return;
+        this.lastEmit.set(record.job.modelId, now);
+        this.emit({ type: "progress", job: { ...record.job } });
+    }
+    async runJob(catalogEntry, record) {
+        try {
+            this.updateState(record, "downloading");
+            const url = buildHuggingFaceResolveUrl(catalogEntry);
+            const httpClient = await this.loadHttpClient();
+            const startByte = record.job.received;
+            const headers = {
+                "user-agent": "Eliza-LocalInference/1.0",
+            };
+            if (startByte > 0) {
+                headers.range = `bytes=${startByte}-`;
+            }
+            const response = await httpClient.request(url, {
+                method: "GET",
+                headers,
+                signal: record.abortController.signal,
+            });
+            if (response.statusCode >= 400) {
+                throw new Error(`HTTP ${response.statusCode} from HuggingFace for ${catalogEntry.hfRepo}`);
+            }
+            const contentLengthHeader = response.headers["content-length"];
+            const contentLength = Array.isArray(contentLengthHeader)
+                ? Number.parseInt(contentLengthHeader[0] ?? "0", 10)
+                : Number.parseInt(contentLengthHeader ?? "0", 10);
+            if (Number.isFinite(contentLength) && contentLength > 0) {
+                record.job.total = startByte + contentLength;
+            }
+            const writeStream = fs.createWriteStream(record.stagingPath, {
+                flags: startByte > 0 ? "a" : "w",
+            });
+            let lastSampleBytes = record.job.received;
+            let lastSampleAt = Date.now();
+            const bodyStream = Readable.from(response.body);
+            bodyStream.on("data", (chunk) => {
+                record.job.received += chunk.length;
+                const now = Date.now();
+                const elapsed = now - lastSampleAt;
+                if (elapsed >= 1000) {
+                    record.job.bytesPerSec =
+                        ((record.job.received - lastSampleBytes) * 1000) / elapsed;
+                    record.job.etaMs =
+                        record.job.bytesPerSec > 0
+                            ? ((record.job.total - record.job.received) * 1000) /
+                                record.job.bytesPerSec
+                            : null;
+                    lastSampleAt = now;
+                    lastSampleBytes = record.job.received;
+                }
+                this.throttleEmit(record);
+            });
+            await pipeline(bodyStream, writeStream);
+            await fsp.rename(record.stagingPath, record.finalPath);
+            const finalStat = await fsp.stat(record.finalPath);
+            // Compute SHA256 on commit so we have an integrity baseline. The
+            // chunk hasher we maintain during streaming gives the same result
+            // but would also have to handle resume-from-partial correctly; for
+            // a ~1-20 GB file a second disk pass at the end is simpler and
+            // robust. Measured at ~400 MB/s on an NVMe so even the 20 GB
+            // catalog entries finish in well under a minute.
+            const sha256 = await hashFile(record.finalPath);
+            const installed = {
+                id: catalogEntry.id,
+                displayName: catalogEntry.displayName,
+                path: record.finalPath,
+                sizeBytes: finalStat.size,
+                hfRepo: catalogEntry.hfRepo,
+                installedAt: new Date().toISOString(),
+                lastUsedAt: null,
+                source: "eliza-download",
+                sha256,
+                lastVerifiedAt: new Date().toISOString(),
+                ...(catalogEntry.runtimeRole
+                    ? { runtimeRole: catalogEntry.runtimeRole }
+                    : {}),
+                ...(catalogEntry.companionForModelId
+                    ? { companionFor: catalogEntry.companionForModelId }
+                    : {}),
+            };
+            await upsertElizaModel(installed);
+            // First-light convenience: assign the freshly-installed model to any
+            // empty slot so chat works without a Settings detour. Idempotent —
+            // ignores slots the user has already configured. See
+            // assignments.ts#ensureDefaultAssignment for the per-slot policy.
+            if (catalogEntry.runtimeRole !== "dflash-drafter") {
+                await ensureDefaultAssignment(installed.id);
+            }
+            for (const companionId of catalogEntry.companionModelIds ?? []) {
+                if (!this.isActive(companionId)) {
+                    void this.start(companionId).catch((err) => {
+                        const job = {
+                            jobId: randomUUID(),
+                            modelId: companionId,
+                            state: "failed",
+                            received: 0,
+                            total: 0,
+                            bytesPerSec: 0,
+                            etaMs: null,
+                            startedAt: new Date().toISOString(),
+                            updatedAt: new Date().toISOString(),
+                            error: err instanceof Error ? err.message : String(err),
+                        };
+                        this.rememberTerminalDownload(job);
+                        this.emit({
+                            type: "failed",
+                            job,
+                        });
+                    });
+                }
+            }
+            this.updateState(record, "completed");
+            record.job.received = finalStat.size;
+            record.job.total = finalStat.size;
+            this.rememberTerminalDownload(record.job);
+            this.emit({ type: "completed", job: { ...record.job } });
+        }
+        catch (err) {
+            if (record.abortController.signal.aborted) {
+                this.updateState(record, "cancelled");
+                this.rememberTerminalDownload(record.job);
+                this.emit({ type: "cancelled", job: { ...record.job } });
+            }
+            else {
+                this.updateState(record, "failed");
+                record.job.error = err instanceof Error ? err.message : String(err);
+                this.rememberTerminalDownload(record.job);
+                this.emit({ type: "failed", job: { ...record.job } });
+            }
+        }
+        finally {
+            this.active.delete(record.job.modelId);
+        }
+    }
+    async loadHttpClient() {
+        const fetchImpl = globalThis.fetch;
+        return {
+            request: async (url, options) => {
+                const response = await fetchImpl(url, {
+                    method: options.method,
+                    headers: options.headers,
+                    signal: options.signal,
+                    redirect: "follow",
+                });
+                if (!response.body) {
+                    throw new Error(`Empty response body from ${url}`);
+                }
+                return {
+                    statusCode: response.status,
+                    headers: Object.fromEntries(response.headers.entries()),
+                    body: readFetchBody(response.body),
+                };
+            },
+        };
+    }
+}

package/dist/services/local-inference/engine.d.ts

@@ -0,0 +1,54 @@
+/**
+ * Standalone llama.cpp engine.
+ *
+ * Owns one `Llama` binding instance, at most one loaded `LlamaModel`, and
+ * a cached `LlamaChatSession` that wraps it. Model swap is unload-then-load
+ * so we never double-allocate VRAM.
+ *
+ * Two consumption paths:
+ *   1. The Model Hub UI calls `load()` / `unload()` to make "Activate" work.
+ *   2. The agent runtime calls `generate()` via the registered
+ *      `ModelType.TEXT_SMALL` / `TEXT_LARGE` handlers (see
+ *      `ensure-local-inference-handler.ts`).
+ *
+ * Dynamic import keeps the binding optional: if `node-llama-cpp` is not
+ * installed, `available()` returns false and callers surface a clear error
+ * instead of crashing the process.
+ */
+import type { LocalInferenceLoadArgs } from "./active-model";
+export interface GenerateArgs {
+    prompt: string;
+    stopSequences?: string[];
+    /** Upper bound on output tokens; defaults to 2048. */
+    maxTokens?: number;
+    /** 0..1; 0.7 default. */
+    temperature?: number;
+    /** nucleus sampling; defaults to 0.9. */
+    topP?: number;
+}
+export declare class LocalInferenceEngine {
+    private llama;
+    private loadedModel;
+    private loadedContext;
+    private loadedSession;
+    private loadedPath;
+    private bindingChecked;
+    private bindingModule;
+    /** Serialises generate calls so concurrent requests don't corrupt session state. */
+    private generationQueue;
+    available(): Promise<boolean>;
+    currentModelPath(): string | null;
+    hasLoadedModel(): boolean;
+    unload(): Promise<void>;
+    load(modelPath: string, resolved?: LocalInferenceLoadArgs): Promise<void>;
+    /**
+     * Generate text from the loaded model. Serialised — a new call waits for
+     * any in-flight generation to finish so the chat session's internal state
+     * stays consistent.
+     */
+    generate(args: GenerateArgs): Promise<string>;
+    private loadBinding;
+    private resolveDflashPlan;
+}
+export declare const localInferenceEngine: LocalInferenceEngine;
+//# sourceMappingURL=engine.d.ts.map

package/dist/services/local-inference/engine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../../../../../src/services/local-inference/engine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAgB7D,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,sDAAsD;IACtD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yBAAyB;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yCAAyC;IACzC,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AA4DD,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,WAAW,CAA2B;IAC9C,OAAO,CAAC,aAAa,CAA6B;IAClD,OAAO,CAAC,aAAa,CAAiC;IACtD,OAAO,CAAC,UAAU,CAAuB;IACzC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,aAAa,CAAmC;IACxD,oFAAoF;IACpF,OAAO,CAAC,eAAe,CAAuC;IAExD,SAAS,IAAI,OAAO,CAAC,OAAO,CAAC;IAQnC,gBAAgB,IAAI,MAAM,GAAG,IAAI;IAOjC,cAAc,IAAI,OAAO;IAInB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAyBvB,IAAI,CACR,SAAS,EAAE,MAAM,EACjB,QAAQ,CAAC,EAAE,sBAAsB,GAChC,OAAO,CAAC,IAAI,CAAC;IA6DhB;;;;OAIG;IACG,QAAQ,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC;YA6BrC,WAAW;YAoBX,iBAAiB;CAoChC;AAED,eAAO,MAAM,oBAAoB,sBAA6B,CAAC"}

package/dist/services/local-inference/engine.js

@@ -0,0 +1,223 @@
+/**
+ * Standalone llama.cpp engine.
+ *
+ * Owns one `Llama` binding instance, at most one loaded `LlamaModel`, and
+ * a cached `LlamaChatSession` that wraps it. Model swap is unload-then-load
+ * so we never double-allocate VRAM.
+ *
+ * Two consumption paths:
+ *   1. The Model Hub UI calls `load()` / `unload()` to make "Activate" work.
+ *   2. The agent runtime calls `generate()` via the registered
+ *      `ModelType.TEXT_SMALL` / `TEXT_LARGE` handlers (see
+ *      `ensure-local-inference-handler.ts`).
+ *
+ * Dynamic import keeps the binding optional: if `node-llama-cpp` is not
+ * installed, `available()` returns false and callers surface a clear error
+ * instead of crashing the process.
+ */
+var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
+    if (typeof path === "string" && /^\.\.?\//.test(path)) {
+        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
+            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
+        });
+    }
+    return path;
+};
+import { findCatalogModel } from "./catalog.js";
+import { dflashLlamaServer, dflashRequired, getDflashRuntimeStatus, } from "./dflash-server.js";
+import { listInstalledModels } from "./registry.js";
+const NODE_LLAMA_CPP_MODULE_ID = "node-llama-cpp";
+function normalizeKvCacheTypeForBinding(name) {
+    return name.trim().toUpperCase();
+}
+export class LocalInferenceEngine {
+    llama = null;
+    loadedModel = null;
+    loadedContext = null;
+    loadedSession = null;
+    loadedPath = null;
+    bindingChecked = false;
+    bindingModule = null;
+    /** Serialises generate calls so concurrent requests don't corrupt session state. */
+    generationQueue = Promise.resolve();
+    async available() {
+        if (!this.bindingChecked) {
+            this.bindingModule = await this.loadBinding();
+            this.bindingChecked = true;
+        }
+        return this.bindingModule !== null;
+    }
+    currentModelPath() {
+        if (dflashLlamaServer.hasLoadedModel()) {
+            return dflashLlamaServer.currentModelPath();
+        }
+        return this.loadedPath;
+    }
+    hasLoadedModel() {
+        return this.loadedModel !== null || dflashLlamaServer.hasLoadedModel();
+    }
+    async unload() {
+        await dflashLlamaServer.stop();
+        if (!this.loadedModel)
+            return;
+        const session = this.loadedSession;
+        const context = this.loadedContext;
+        const model = this.loadedModel;
+        this.loadedSession = null;
+        this.loadedContext = null;
+        this.loadedModel = null;
+        this.loadedPath = null;
+        // Dispose bottom-up: session first, then context, then the model. Each
+        // dispose is wrapped because a partial failure must not strand state.
+        try {
+            await session?.dispose?.();
+        }
+        catch {
+            /* best effort */
+        }
+        try {
+            await context?.dispose();
+        }
+        catch {
+            /* best effort */
+        }
+        await model.dispose();
+    }
+    async load(modelPath, resolved) {
+        if (this.loadedPath === modelPath && this.loadedModel)
+            return;
+        if (dflashLlamaServer.currentModelPath() === modelPath)
+            return;
+        const dflashPlan = await this.resolveDflashPlan(modelPath);
+        if (dflashPlan) {
+            try {
+                await this.unload();
+                await dflashLlamaServer.start(dflashPlan);
+                return;
+            }
+            catch (err) {
+                if (dflashRequired())
+                    throw err;
+                console.warn("[local-inference] DFlash llama-server unavailable; falling back to node-llama-cpp:", err instanceof Error ? err.message : String(err));
+            }
+        }
+        if (!(await this.available()) || !this.bindingModule) {
+            throw new Error("node-llama-cpp is not installed in this build; add it as a dependency to enable local inference");
+        }
+        if (this.loadedModel) {
+            await this.unload();
+        }
+        if (!this.llama) {
+            this.llama = await this.bindingModule.getLlama({ gpu: "auto" });
+        }
+        const model = await this.llama.loadModel({
+            modelPath,
+            gpuLayers: resolved?.gpuLayers ?? "auto",
+            useMmap: resolved?.mmap,
+            useMlock: resolved?.mlock,
+            defaultContextFlashAttention: resolved?.flashAttention,
+        });
+        const context = await model.createContext({
+            contextSize: resolved?.contextSize,
+            flashAttention: resolved?.flashAttention,
+            experimentalKvCacheKeyType: resolved?.cacheTypeK
+                ? normalizeKvCacheTypeForBinding(resolved.cacheTypeK)
+                : undefined,
+            experimentalKvCacheValueType: resolved?.cacheTypeV
+                ? normalizeKvCacheTypeForBinding(resolved.cacheTypeV)
+                : undefined,
+        });
+        const sequence = context.getSequence();
+        const session = new this.bindingModule.LlamaChatSession({
+            contextSequence: sequence,
+        });
+        this.loadedModel = model;
+        this.loadedContext = context;
+        this.loadedSession = session;
+        this.loadedPath = modelPath;
+    }
+    /**
+     * Generate text from the loaded model. Serialised — a new call waits for
+     * any in-flight generation to finish so the chat session's internal state
+     * stays consistent.
+     */
+    async generate(args) {
+        if (dflashLlamaServer.hasLoadedModel()) {
+            return dflashLlamaServer.generate(args);
+        }
+        if (!this.loadedSession) {
+            throw new Error("No local model is active. Select one in Settings → Local models before using local inference.");
+        }
+        const session = this.loadedSession;
+        const run = async () => {
+            // Agent model handlers are stateless per call. Drop any prior chat
+            // history so sequential prompts don't thread through accumulated
+            // context and drift output quality.
+            await session.resetChatHistory?.();
+            return session.prompt(args.prompt, {
+                maxTokens: args.maxTokens ?? 2048,
+                temperature: args.temperature ?? 0.7,
+                topP: args.topP ?? 0.9,
+                customStopTriggers: args.stopSequences,
+            });
+        };
+        const job = this.generationQueue.then(run, run);
+        this.generationQueue = job.catch(() => {
+            /* swallow so queue remains usable after a failure */
+        });
+        return job;
+    }
+    async loadBinding() {
+        try {
+            const mod = (await import(__rewriteRelativeImportExtension(
+            /* @vite-ignore */ NODE_LLAMA_CPP_MODULE_ID)));
+            if (mod &&
+                typeof mod === "object" &&
+                "getLlama" in mod &&
+                "LlamaChatSession" in mod &&
+                typeof mod.getLlama === "function") {
+                return mod;
+            }
+            return null;
+        }
+        catch {
+            return null;
+        }
+    }
+    async resolveDflashPlan(modelPath) {
+        const installed = await listInstalledModels();
+        const target = installed.find((m) => m.path === modelPath);
+        if (!target)
+            return null;
+        const catalog = findCatalogModel(target.id);
+        const dflash = catalog?.runtime?.dflash;
+        if (!dflash)
+            return null;
+        const status = getDflashRuntimeStatus();
+        if (!status.enabled) {
+            if (status.required)
+                throw new Error(`[dflash] ${status.reason}`);
+            return null;
+        }
+        const drafter = installed.find((m) => m.id === dflash.drafterModelId);
+        if (!drafter) {
+            const message = `[dflash] ${catalog.displayName} requires companion drafter ${dflash.drafterModelId}. Download the model again or start a download for the companion id.`;
+            if (status.required)
+                throw new Error(message);
+            console.warn(`${message} Falling back to node-llama-cpp.`);
+            return null;
+        }
+        return {
+            targetModelPath: target.path,
+            drafterModelPath: drafter.path,
+            contextSize: dflash.contextSize,
+            draftContextSize: dflash.draftContextSize,
+            draftMin: dflash.draftMin,
+            draftMax: dflash.draftMax,
+            gpuLayers: dflash.gpuLayers,
+            draftGpuLayers: dflash.draftGpuLayers,
+            disableThinking: dflash.disableThinking,
+        };
+    }
+}
+export const localInferenceEngine = new LocalInferenceEngine();

package/dist/services/local-inference/external-scanner.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Discover GGUF files already on disk from other local-inference tools.
+ *
+ * Users often have LM Studio, Jan, Ollama, or raw HuggingFace downloads
+ * lying around. We scan their default cache paths and surface those models
+ * in the Model Hub with `source: "external-scan"` so Eliza can load them
+ * without re-downloading. Eliza never modifies or deletes these files —
+ * the uninstall endpoint refuses when `source !== "eliza-download"`.
+ *
+ * Ollama is special: its blobs live under `models/blobs/sha256-*` with no
+ * `.gguf` extension, and the human name only exists in adjacent manifests.
+ * We parse the manifests to recover the mapping; blobs we can't map stay
+ * hidden rather than surfacing as opaque hashes.
+ */
+import type { InstalledModel } from "./types";
+export declare function scanExternalModels(): Promise<InstalledModel[]>;
+//# sourceMappingURL=external-scanner.d.ts.map

package/dist/services/local-inference/external-scanner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"external-scanner.d.ts","sourceRoot":"","sources":["../../../../../../src/services/local-inference/external-scanner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAMH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AA6P9C,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC,CAuCpE"}