agent-orcha 0.0.5 → 0.0.8

package/dist/lib/local-llm/engine-registry.js

@@ -0,0 +1,56 @@
+import { LlamaCppEngine } from "./engines/llama-cpp-engine.js";
+import { MlxServeEngine } from "./engines/mlx-serve-engine.js";
+import { ModelManager } from "./model-manager.js";
+export class EngineRegistry {
+    engines = new Map();
+    _baseDir = '';
+    constructor() {
+        this.register(new LlamaCppEngine());
+        this.register(new MlxServeEngine());
+    }
+    register(engine) {
+        this.engines.set(engine.engineName, engine);
+    }
+    getEngine(name) {
+        return this.engines.get(name);
+    }
+    getAvailableEngines() {
+        return [...this.engines.values()].filter(e => e.isAvailable());
+    }
+    getAllEngines() {
+        return [...this.engines.values()];
+    }
+    getAllStatus() {
+        const result = {};
+        for (const [name, engine] of this.engines) {
+            result[name] = engine.getStatus();
+        }
+        return result;
+    }
+    setBaseDir(dir) {
+        this._baseDir = dir;
+        for (const engine of this.engines.values()) {
+            engine.setBaseDir(dir);
+        }
+    }
+    killAllOrphans() {
+        for (const engine of this.engines.values()) {
+            engine.killOrphans();
+        }
+    }
+    async unloadAll() {
+        for (const engine of this.engines.values()) {
+            await engine.unloadChat();
+            await engine.unloadEmbedding();
+        }
+    }
+    async resolveModelPath(modelName) {
+        const manager = new ModelManager(this._baseDir);
+        const result = await manager.findModelFile(modelName);
+        if (!result)
+            throw new Error(`Local model "${modelName}" not found. Download it first.`);
+        return result;
+    }
+}
+export const engineRegistry = new EngineRegistry();
+//# sourceMappingURL=engine-registry.js.map

package/dist/lib/local-llm/engine-registry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"engine-registry.js","sourceRoot":"","sources":["../../../lib/local-llm/engine-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAGlD,MAAM,OAAO,cAAc;IACjB,OAAO,GAAG,IAAI,GAAG,EAAuB,CAAC;IACzC,QAAQ,GAAG,EAAE,CAAC;IAEtB;QACE,IAAI,CAAC,QAAQ,CAAC,IAAI,cAAc,EAAE,CAAC,CAAC;QACpC,IAAI,CAAC,QAAQ,CAAC,IAAI,cAAc,EAAE,CAAC,CAAC;IACtC,CAAC;IAED,QAAQ,CAAC,MAAmB;QAC1B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IAC9C,CAAC;IAED,SAAS,CAAC,IAAY;QACpB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAED,mBAAmB;QACjB,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IACjE,CAAC;IAED,aAAa;QACX,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACpC,CAAC;IAED,YAAY;QACV,MAAM,MAAM,GAAiC,EAAE,CAAC;QAChD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAC1C,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;QACpC,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,UAAU,CAAC,GAAW;QACpB,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC;QACpB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAED,cAAc;QACZ,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,MAAM,CAAC,WAAW,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,SAAS;QACb,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC;YAC1B,MAAM,MAAM,CAAC,eAAe,EAAE,CAAC;QACjC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,SAAiB;QACtC,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QACtD,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,gBAAgB,SAAS,iCAAiC,CAAC,CAAC;QACzF,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAED,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,cAAc,EAAE,CAAC"}

package/dist/lib/local-llm/engines/llama-cpp-engine.d.ts

@@ -0,0 +1,31 @@
+import type { LocalEngine, EngineChatStatus, EngineServerStatus, EngineStatus, LoadOptions } from '../engine-interface.ts';
+export declare class LlamaCppEngine implements LocalEngine {
+    readonly engineName = "llama-cpp";
+    private _baseDir;
+    private chatServer;
+    private embeddingServer;
+    private _detectedContextSize;
+    private _memoryEstimate;
+    private _supportsVision;
+    setBaseDir(dir: string): void;
+    isAvailable(): boolean;
+    loadChat(modelPath: string, opts?: LoadOptions): Promise<void>;
+    unloadChat(): Promise<void>;
+    swapChat(modelPath: string, opts?: LoadOptions): Promise<void>;
+    ensureRunningChat(modelName: string, opts?: LoadOptions): Promise<void>;
+    getChatStatus(): EngineChatStatus;
+    getChatBaseUrl(): string | null;
+    loadEmbedding(modelPath: string): Promise<void>;
+    unloadEmbedding(): Promise<void>;
+    ensureRunningEmbedding(modelName: string): Promise<void>;
+    getEmbeddingStatus(): EngineServerStatus;
+    getEmbeddingBaseUrl(): string | null;
+    getStatus(): EngineStatus;
+    killOrphans(): void;
+    getBinaryVersion(): string | null;
+    getBinarySource(): 'managed' | 'system' | null;
+    checkForUpdate(): Promise<any>;
+    updateBinary(): Promise<void>;
+    private resolveModelPath;
+}
+//# sourceMappingURL=llama-cpp-engine.d.ts.map

package/dist/lib/local-llm/engines/llama-cpp-engine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llama-cpp-engine.d.ts","sourceRoot":"","sources":["../../../../lib/local-llm/engines/llama-cpp-engine.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,WAAW,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAE3H,qBAAa,cAAe,YAAW,WAAW;IAChD,QAAQ,CAAC,UAAU,eAAe;IAElC,OAAO,CAAC,QAAQ,CAAM;IACtB,OAAO,CAAC,UAAU,CAAmC;IACrD,OAAO,CAAC,eAAe,CAAmC;IAC1D,OAAO,CAAC,oBAAoB,CAAuB;IACnD,OAAO,CAAC,eAAe,CAAiF;IACxG,OAAO,CAAC,eAAe,CAAS;IAEhC,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAI7B,WAAW,IAAI,OAAO;IAMhB,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IA8C9D,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAK9D,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAO7E,aAAa,IAAI,gBAAgB;IAYjC,cAAc,IAAI,MAAM,GAAG,IAAI;IAMzB,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAM/C,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC;IAMhC,sBAAsB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAO9D,kBAAkB,IAAI,kBAAkB;IAWxC,mBAAmB,IAAI,MAAM,GAAG,IAAI;IAMpC,SAAS,IAAI,YAAY;IASzB,WAAW,IAAI,IAAI;IAMnB,gBAAgB,IAAI,MAAM,GAAG,IAAI;IAIjC,eAAe,IAAI,SAAS,GAAG,QAAQ,GAAG,IAAI;IAMxC,cAAc,IAAI,OAAO,CAAC,GAAG,CAAC;IAI9B,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;YAMrB,gBAAgB;CAM/B"}

package/dist/lib/local-llm/engines/llama-cpp-engine.js

@@ -0,0 +1,164 @@
+import * as path from 'path';
+import { LlamaServerProcess } from "../llama-server-process.js";
+import { killOrphanedServers } from "../llama-server-process.js";
+import { ModelManager } from "../model-manager.js";
+import { readGGUFModelInfo, calculateOptimalContextSize, kvCacheBytesPerToken } from "../gguf-reader.js";
+import { detectGpu, getBinaryVersion, isSystemBinary, updateBinary, checkForUpdate } from "../binary-manager.js";
+import { logger } from "../../logger.js";
+export class LlamaCppEngine {
+    engineName = 'llama-cpp';
+    _baseDir = '';
+    chatServer = null;
+    embeddingServer = null;
+    _detectedContextSize = null;
+    _memoryEstimate = null;
+    _supportsVision = false;
+    setBaseDir(dir) {
+        this._baseDir = dir;
+    }
+    isAvailable() {
+        return getBinaryVersion(this._baseDir) !== null;
+    }
+    // ─── Chat ───────────────────────────────────────────────────────────────────
+    async loadChat(modelPath, opts) {
+        if (!this.chatServer)
+            this.chatServer = new LlamaServerProcess(this._baseDir);
+        if (this.chatServer.running && this.chatServer.modelPath === modelPath)
+            return;
+        let contextSize = opts?.contextSize;
+        const modelInfo = await readGGUFModelInfo(modelPath);
+        if (!contextSize && modelInfo) {
+            contextSize = calculateOptimalContextSize(modelInfo);
+        }
+        if (modelInfo && contextSize) {
+            const kvBytes = contextSize * kvCacheBytesPerToken(modelInfo);
+            this._memoryEstimate = {
+                modelBytes: modelInfo.fileSizeBytes,
+                kvCacheBytes: kvBytes,
+                totalBytes: modelInfo.fileSizeBytes + kvBytes,
+            };
+        }
+        // Auto-detect multimodal projector (mmproj) for vision support
+        const modelFileName = path.basename(modelPath);
+        const manager = new ModelManager(this._baseDir);
+        const mmproj = await manager.findMmprojForModel(modelFileName);
+        this._supportsVision = !!mmproj;
+        if (mmproj) {
+            logger.info(`[LlamaCppEngine] Vision enabled with mmproj: ${path.basename(mmproj)}`);
+        }
+        this._detectedContextSize = contextSize ?? null;
+        const gpu = detectGpu();
+        const isGpu = gpu.accel !== 'none';
+        const isMetal = gpu.accel === 'metal';
+        await this.chatServer.start({
+            modelPath,
+            contextSize,
+            mmproj: mmproj ?? undefined,
+            gpuLayers: isGpu ? -1 : 0,
+            flashAttn: isGpu,
+            ...(isGpu ? { batchSize: 4096, ubatchSize: 1024 } : {}),
+            ...(isMetal ? { cacheTypeK: 'q8_0', cacheTypeV: 'q8_0', mlock: true } : {}),
+            ...(opts?.reasoningBudget !== undefined ? { reasoningBudget: opts.reasoningBudget } : {}),
+        });
+    }
+    async unloadChat() {
+        if (this.chatServer) {
+            await this.chatServer.stop();
+        }
+    }
+    async swapChat(modelPath, opts) {
+        await this.unloadChat();
+        await this.loadChat(modelPath, opts);
+    }
+    async ensureRunningChat(modelName, opts) {
+        if (this.chatServer?.running)
+            return;
+        logger.info(`[LlamaCppEngine] Auto-starting chat model: ${modelName}`);
+        const { filePath } = await this.resolveModelPath(modelName);
+        await this.loadChat(filePath, opts);
+    }
+    getChatStatus() {
+        const running = this.chatServer?.running ?? false;
+        return {
+            running,
+            activeModel: running ? (this.chatServer?.modelPath ?? null) : null,
+            port: this.chatServer?.port ?? null,
+            contextSize: this._detectedContextSize,
+            memoryEstimate: this._memoryEstimate,
+            supportsVision: this._supportsVision,
+        };
+    }
+    getChatBaseUrl() {
+        return this.chatServer?.ready ? this.chatServer.getBaseUrl() : null;
+    }
+    // ─── Embedding ──────────────────────────────────────────────────────────────
+    async loadEmbedding(modelPath) {
+        if (!this.embeddingServer)
+            this.embeddingServer = new LlamaServerProcess(this._baseDir, true);
+        if (this.embeddingServer.running && this.embeddingServer.modelPath === modelPath)
+            return;
+        await this.embeddingServer.start({ modelPath, embedding: true });
+    }
+    async unloadEmbedding() {
+        if (this.embeddingServer) {
+            await this.embeddingServer.stop();
+        }
+    }
+    async ensureRunningEmbedding(modelName) {
+        if (this.embeddingServer?.running)
+            return;
+        logger.info(`[LlamaCppEngine] Auto-starting embedding model: ${modelName}`);
+        const { filePath } = await this.resolveModelPath(modelName);
+        await this.loadEmbedding(filePath);
+    }
+    getEmbeddingStatus() {
+        const running = this.embeddingServer?.running ?? false;
+        return {
+            running,
+            activeModel: running ? (this.embeddingServer?.modelPath ?? null) : null,
+            port: this.embeddingServer?.port ?? null,
+            contextSize: null,
+            memoryEstimate: null,
+        };
+    }
+    getEmbeddingBaseUrl() {
+        return this.embeddingServer?.ready ? this.embeddingServer.getBaseUrl() : null;
+    }
+    // ─── Combined ───────────────────────────────────────────────────────────────
+    getStatus() {
+        return {
+            engineName: this.engineName,
+            available: this.isAvailable(),
+            chat: this.getChatStatus(),
+            embedding: this.getEmbeddingStatus(),
+        };
+    }
+    killOrphans() {
+        killOrphanedServers(this._baseDir);
+    }
+    // ─── Binary management ─────────────────────────────────────────────────────
+    getBinaryVersion() {
+        return getBinaryVersion(this._baseDir);
+    }
+    getBinarySource() {
+        const version = getBinaryVersion(this._baseDir);
+        if (!version)
+            return null;
+        return isSystemBinary() ? 'system' : 'managed';
+    }
+    async checkForUpdate() {
+        return checkForUpdate(this._baseDir);
+    }
+    async updateBinary() {
+        return updateBinary(this._baseDir);
+    }
+    // ─── Private ────────────────────────────────────────────────────────────────
+    async resolveModelPath(modelName) {
+        const manager = new ModelManager(this._baseDir);
+        const result = await manager.findModelFile(modelName);
+        if (!result)
+            throw new Error(`Local model "${modelName}" not found. Download it first.`);
+        return result;
+    }
+}
+//# sourceMappingURL=llama-cpp-engine.js.map

package/dist/lib/local-llm/engines/llama-cpp-engine.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llama-cpp-engine.js","sourceRoot":"","sources":["../../../../lib/local-llm/engines/llama-cpp-engine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AACjE,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,iBAAiB,EAAE,2BAA2B,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACzG,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACjH,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAGzC,MAAM,OAAO,cAAc;IAChB,UAAU,GAAG,WAAW,CAAC;IAE1B,QAAQ,GAAG,EAAE,CAAC;IACd,UAAU,GAA8B,IAAI,CAAC;IAC7C,eAAe,GAA8B,IAAI,CAAC;IAClD,oBAAoB,GAAkB,IAAI,CAAC;IAC3C,eAAe,GAA4E,IAAI,CAAC;IAChG,eAAe,GAAG,KAAK,CAAC;IAEhC,UAAU,CAAC,GAAW;QACpB,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC;IACtB,CAAC;IAED,WAAW;QACT,OAAO,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC;IAClD,CAAC;IAED,+EAA+E;IAE/E,KAAK,CAAC,QAAQ,CAAC,SAAiB,EAAE,IAAkB;QAClD,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,IAAI,CAAC,UAAU,GAAG,IAAI,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC9E,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,KAAK,SAAS;YAAE,OAAO;QAE/E,IAAI,WAAW,GAAG,IAAI,EAAE,WAAW,CAAC;QAEpC,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAC,SAAS,CAAC,CAAC;QACrD,IAAI,CAAC,WAAW,IAAI,SAAS,EAAE,CAAC;YAC9B,WAAW,GAAG,2BAA2B,CAAC,SAAS,CAAC,CAAC;QACvD,CAAC;QAED,IAAI,SAAS,IAAI,WAAW,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,WAAW,GAAG,oBAAoB,CAAC,SAAS,CAAC,CAAC;YAC9D,IAAI,CAAC,eAAe,GAAG;gBACrB,UAAU,EAAE,SAAS,CAAC,aAAa;gBACnC,YAAY,EAAE,OAAO;gBACrB,UAAU,EAAE,SAAS,CAAC,aAAa,GAAG,OAAO;aAC9C,CAAC;QACJ,CAAC;QAED,+DAA+D;QAC/D,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC/C,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,kBAAkB,CAAC,aAAa,CAAC,CAAC;QAC/D,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;QAEhC,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,CAAC,IAAI,CAAC,gDAAgD,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACvF,CAAC;QAED,IAAI,CAAC,oBAAoB,GAAG,WAAW,IAAI,IAAI,CAAC;QAChD,MAAM,GAAG,GAAG,SAAS,EAAE,CAAC;QACxB,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,KAAK,MAAM,CAAC;QACnC,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,KAAK,OAAO,CAAC;QACtC,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;YAC1B,SAAS;YACT,WAAW;YACX,MAAM,EAAE,MAAM,IAAI,SAAS;YAC3B,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACzB,SAAS,EAAE,KAAK;YAChB,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvD,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3E,GAAG,CAAC,IAAI,EAAE,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC1F,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,UAAU;QACd,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,SAAiB,EAAE,IAAkB;QAClD,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACxB,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,SAAiB,EAAE,IAAkB;QAC3D,IAAI,IAAI,CAAC,UAAU,EAAE,OAAO;YAAE,OAAO;QACrC,MAAM,CAAC,IAAI,CAAC,8CAA8C,SAAS,EAAE,CAAC,CAAC;QACvE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAC5D,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IACtC,CAAC;IAED,aAAa;QACX,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,OAAO,IAAI,KAAK,CAAC;QAClD,OAAO;YACL,OAAO;YACP,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,EAAE,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;YAClE,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,IAAI,IAAI;YACnC,WAAW,EAAE,IAAI,CAAC,oBAAoB;YACtC,cAAc,EAAE,IAAI,CAAC,eAAe;YACpC,cAAc,EAAE,IAAI,CAAC,eAAe;SACrC,CAAC;IACJ,CAAC;IAED,cAAc;QACZ,OAAO,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACtE,CAAC;IAED,+EAA+E;IAE/E,KAAK,CAAC,aAAa,CAAC,SAAiB;QACnC,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,IAAI,CAAC,eAAe,GAAG,IAAI,kBAAkB,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC9F,IAAI,IAAI,CAAC,eAAe,CAAC,OAAO,IAAI,IAAI,CAAC,eAAe,CAAC,SAAS,KAAK,SAAS;YAAE,OAAO;QACzF,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACzB,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QACpC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,sBAAsB,CAAC,SAAiB;QAC5C,IAAI,IAAI,CAAC,eAAe,EAAE,OAAO;YAAE,OAAO;QAC1C,MAAM,CAAC,IAAI,CAAC,mDAAmD,SAAS,EAAE,CAAC,CAAC;QAC5E,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAC5D,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IACrC,CAAC;IAED,kBAAkB;QAChB,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,EAAE,OAAO,IAAI,KAAK,CAAC;QACvD,OAAO;YACL,OAAO;YACP,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,EAAE,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;YACvE,IAAI,EAAE,IAAI,CAAC,eAAe,EAAE,IAAI,IAAI,IAAI;YACxC,WAAW,EAAE,IAAI;YACjB,cAAc,EAAE,IAAI;SACrB,CAAC;IACJ,CAAC;IAED,mBAAmB;QACjB,OAAO,IAAI,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IAChF,CAAC;IAED,+EAA+E;IAE/E,SAAS;QACP,OAAO;YACL,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,SAAS,EAAE,IAAI,CAAC,WAAW,EAAE;YAC7B,IAAI,EAAE,IAAI,CAAC,aAAa,EAAE;YAC1B,SAAS,EAAE,IAAI,CAAC,kBAAkB,EAAE;SACrC,CAAC;IACJ,CAAC;IAED,WAAW;QACT,mBAAmB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrC,CAAC;IAED,8EAA8E;IAE9E,gBAAgB;QACd,OAAO,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACzC,CAAC;IAED,eAAe;QACb,MAAM,OAAO,GAAG,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAChD,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC;QAC1B,OAAO,cAAc,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,cAAc;QAClB,OAAO,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,OAAO,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrC,CAAC;IAED,+EAA+E;IAEvE,KAAK,CAAC,gBAAgB,CAAC,SAAiB;QAC9C,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QACtD,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,gBAAgB,SAAS,iCAAiC,CAAC,CAAC;QACzF,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}

package/dist/lib/local-llm/engines/mlx-serve-engine.d.ts

@@ -0,0 +1,31 @@
+import type { LocalEngine, EngineChatStatus, EngineServerStatus, EngineStatus, LoadOptions } from '../engine-interface.ts';
+export declare class MlxServeEngine implements LocalEngine {
+    readonly engineName = "mlx-serve";
+    private _baseDir;
+    private chatServer;
+    private embeddingServer;
+    private _detectedContextSize;
+    private _memoryEstimate;
+    private _supportsVision;
+    setBaseDir(dir: string): void;
+    isAvailable(): boolean;
+    loadChat(modelPath: string, opts?: LoadOptions): Promise<void>;
+    unloadChat(): Promise<void>;
+    swapChat(modelPath: string, opts?: LoadOptions): Promise<void>;
+    ensureRunningChat(modelName: string, opts?: LoadOptions): Promise<void>;
+    getChatStatus(): EngineChatStatus;
+    getChatBaseUrl(): string | null;
+    loadEmbedding(modelPath: string): Promise<void>;
+    unloadEmbedding(): Promise<void>;
+    ensureRunningEmbedding(modelName: string): Promise<void>;
+    getEmbeddingStatus(): EngineServerStatus;
+    getEmbeddingBaseUrl(): string | null;
+    getStatus(): EngineStatus;
+    killOrphans(): void;
+    getBinaryVersion(): string | null;
+    getBinarySource(): 'managed' | 'system' | null;
+    checkForUpdate(): Promise<any>;
+    updateBinary(): Promise<void>;
+    private resolveModelPath;
+}
+//# sourceMappingURL=mlx-serve-engine.d.ts.map

package/dist/lib/local-llm/engines/mlx-serve-engine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mlx-serve-engine.d.ts","sourceRoot":"","sources":["../../../../lib/local-llm/engines/mlx-serve-engine.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,WAAW,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAE3H,qBAAa,cAAe,YAAW,WAAW;IAChD,QAAQ,CAAC,UAAU,eAAe;IAElC,OAAO,CAAC,QAAQ,CAAM;IACtB,OAAO,CAAC,UAAU,CAAiC;IACnD,OAAO,CAAC,eAAe,CAAiC;IACxD,OAAO,CAAC,oBAAoB,CAAuB;IACnD,OAAO,CAAC,eAAe,CAAiF;IACxG,OAAO,CAAC,eAAe,CAAS;IAEhC,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAI7B,WAAW,IAAI,OAAO;IAMhB,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAwC9D,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAK9D,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAO7E,aAAa,IAAI,gBAAgB;IAYjC,cAAc,IAAI,MAAM,GAAG,IAAI;IAMzB,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ/C,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC;IAMhC,sBAAsB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAO9D,kBAAkB,IAAI,kBAAkB;IAWxC,mBAAmB,IAAI,MAAM,GAAG,IAAI;IAMpC,SAAS,IAAI,YAAY;IASzB,WAAW,IAAI,IAAI;IAMnB,gBAAgB,IAAI,MAAM,GAAG,IAAI;IAIjC,eAAe,IAAI,SAAS,GAAG,QAAQ,GAAG,IAAI;IAMxC,cAAc,IAAI,OAAO,CAAC,GAAG,CAAC;IAI9B,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;YAMrB,gBAAgB;CAM/B"}

package/dist/lib/local-llm/engines/mlx-serve-engine.js

@@ -0,0 +1,161 @@
+import { MlxServerProcess } from "../mlx-server-process.js";
+import { killOrphanedMlxServers } from "../mlx-server-process.js";
+import { ModelManager } from "../model-manager.js";
+import { getMlxBinaryVersion, isMlxSystemBinary, updateMlxBinary, checkForMlxUpdate } from "../mlx-binary-manager.js";
+import { logger } from "../../logger.js";
+export class MlxServeEngine {
+    engineName = 'mlx-serve';
+    _baseDir = '';
+    chatServer = null;
+    embeddingServer = null;
+    _detectedContextSize = null;
+    _memoryEstimate = null;
+    _supportsVision = false;
+    setBaseDir(dir) {
+        this._baseDir = dir;
+    }
+    isAvailable() {
+        return getMlxBinaryVersion(this._baseDir) !== null && process.platform === 'darwin' && process.arch === 'arm64';
+    }
+    // ─── Chat ───────────────────────────────────────────────────────────────────
+    async loadChat(modelPath, opts) {
+        if (this.chatServer?.running && this.chatServer.modelPath === modelPath)
+            return;
+        if (!this.chatServer)
+            this.chatServer = new MlxServerProcess(this._baseDir);
+        this._memoryEstimate = null;
+        this._supportsVision = false;
+        this._detectedContextSize = opts?.contextSize ?? null;
+        await this.chatServer.start({ modelPath, contextSize: opts?.contextSize, reasoningBudget: opts?.reasoningBudget });
+        // Fetch /props from mlx-serve to get memory usage and context size
+        try {
+            const baseUrl = this.chatServer.getBaseUrl();
+            const res = await fetch(`${baseUrl}/props`);
+            if (res.ok) {
+                const props = await res.json();
+                const nCtx = props.default_generation_settings?.n_ctx;
+                if (nCtx && !this._detectedContextSize) {
+                    this._detectedContextSize = nCtx;
+                }
+                const ctxSize = this._detectedContextSize ?? nCtx ?? 0;
+                const info = props.model_info;
+                // KV cache: layers × 2(K+V) × kv_heads × head_dim × 2(float16) × ctx
+                const kvCacheBytes = info
+                    ? info.num_hidden_layers * 2 * info.num_key_value_heads * info.head_dim * 2 * ctxSize
+                    : 0;
+                const modelBytes = props.memory?.active_bytes ?? 0;
+                if (modelBytes || kvCacheBytes) {
+                    this._memoryEstimate = {
+                        modelBytes,
+                        kvCacheBytes,
+                        totalBytes: modelBytes + kvCacheBytes,
+                    };
+                }
+            }
+        }
+        catch (err) {
+            logger.warn('[MlxServeEngine] Failed to fetch /props:', err);
+        }
+    }
+    async unloadChat() {
+        if (this.chatServer) {
+            await this.chatServer.stop();
+        }
+    }
+    async swapChat(modelPath, opts) {
+        await this.unloadChat();
+        await this.loadChat(modelPath, opts);
+    }
+    async ensureRunningChat(modelName, opts) {
+        if (this.chatServer?.running)
+            return;
+        logger.info(`[MlxServeEngine] Auto-starting chat model: ${modelName}`);
+        const { filePath } = await this.resolveModelPath(modelName);
+        await this.loadChat(filePath, opts);
+    }
+    getChatStatus() {
+        const running = this.chatServer?.running ?? false;
+        return {
+            running,
+            activeModel: running ? (this.chatServer?.modelPath ?? null) : null,
+            port: this.chatServer?.port ?? null,
+            contextSize: this._detectedContextSize,
+            memoryEstimate: this._memoryEstimate,
+            supportsVision: this._supportsVision,
+        };
+    }
+    getChatBaseUrl() {
+        return this.chatServer?.ready ? this.chatServer.getBaseUrl() : null;
+    }
+    // ─── Embedding ──────────────────────────────────────────────────────────────
+    async loadEmbedding(modelPath) {
+        if (this.embeddingServer?.running && this.embeddingServer.modelPath === modelPath)
+            return;
+        if (!this.embeddingServer)
+            this.embeddingServer = new MlxServerProcess(this._baseDir, 'embedding');
+        await this.embeddingServer.start({ modelPath });
+        logger.info(`[MlxServeEngine] Embedding server ready on port ${this.embeddingServer.port}`);
+    }
+    async unloadEmbedding() {
+        if (this.embeddingServer) {
+            await this.embeddingServer.stop();
+        }
+    }
+    async ensureRunningEmbedding(modelName) {
+        if (this.embeddingServer?.running)
+            return;
+        logger.info(`[MlxServeEngine] Auto-starting embedding model: ${modelName}`);
+        const { filePath } = await this.resolveModelPath(modelName);
+        await this.loadEmbedding(filePath);
+    }
+    getEmbeddingStatus() {
+        const running = this.embeddingServer?.running ?? false;
+        return {
+            running,
+            activeModel: running ? (this.embeddingServer?.modelPath ?? null) : null,
+            port: this.embeddingServer?.port ?? null,
+            contextSize: null,
+            memoryEstimate: null,
+        };
+    }
+    getEmbeddingBaseUrl() {
+        return this.embeddingServer?.ready ? this.embeddingServer.getBaseUrl() : null;
+    }
+    // ─── Combined ───────────────────────────────────────────────────────────────
+    getStatus() {
+        return {
+            engineName: this.engineName,
+            available: this.isAvailable(),
+            chat: this.getChatStatus(),
+            embedding: this.getEmbeddingStatus(),
+        };
+    }
+    killOrphans() {
+        killOrphanedMlxServers(this._baseDir);
+    }
+    // ─── Binary management ─────────────────────────────────────────────────────
+    getBinaryVersion() {
+        return getMlxBinaryVersion(this._baseDir);
+    }
+    getBinarySource() {
+        const version = getMlxBinaryVersion(this._baseDir);
+        if (!version)
+            return null;
+        return isMlxSystemBinary() ? 'system' : 'managed';
+    }
+    async checkForUpdate() {
+        return checkForMlxUpdate(this._baseDir);
+    }
+    async updateBinary() {
+        return updateMlxBinary(this._baseDir);
+    }
+    // ─── Private ────────────────────────────────────────────────────────────────
+    async resolveModelPath(modelName) {
+        const manager = new ModelManager(this._baseDir);
+        const result = await manager.findModelFile(modelName);
+        if (!result)
+            throw new Error(`Local model "${modelName}" not found. Download it first.`);
+        return result;
+    }
+}
+//# sourceMappingURL=mlx-serve-engine.js.map

package/dist/lib/local-llm/engines/mlx-serve-engine.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mlx-serve-engine.js","sourceRoot":"","sources":["../../../../lib/local-llm/engines/mlx-serve-engine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AACtH,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAGzC,MAAM,OAAO,cAAc;IAChB,UAAU,GAAG,WAAW,CAAC;IAE1B,QAAQ,GAAG,EAAE,CAAC;IACd,UAAU,GAA4B,IAAI,CAAC;IAC3C,eAAe,GAA4B,IAAI,CAAC;IAChD,oBAAoB,GAAkB,IAAI,CAAC;IAC3C,eAAe,GAA4E,IAAI,CAAC;IAChG,eAAe,GAAG,KAAK,CAAC;IAEhC,UAAU,CAAC,GAAW;QACpB,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC;IACtB,CAAC;IAED,WAAW;QACT,OAAO,mBAAmB,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO,CAAC;IAClH,CAAC;IAED,+EAA+E;IAE/E,KAAK,CAAC,QAAQ,CAAC,SAAiB,EAAE,IAAkB;QAClD,IAAI,IAAI,CAAC,UAAU,EAAE,OAAO,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,KAAK,SAAS;YAAE,OAAO;QAChF,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,IAAI,CAAC,UAAU,GAAG,IAAI,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE5E,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;QAC5B,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;QAC7B,IAAI,CAAC,oBAAoB,GAAG,IAAI,EAAE,WAAW,IAAI,IAAI,CAAC;QAEtD,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,WAAW,EAAE,eAAe,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;QAEnH,mEAAmE;QACnE,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;YAC7C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,QAAQ,CAAC,CAAC;YAC5C,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;gBACX,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,IAAI,EAAS,CAAC;gBACtC,MAAM,IAAI,GAAG,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC;gBACtD,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;oBACvC,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC;gBACnC,CAAC;gBACD,MAAM,OAAO,GAAG,IAAI,CAAC,oBAAoB,IAAI,IAAI,IAAI,CAAC,CAAC;gBACvD,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC;gBAC9B,qEAAqE;gBACrE,MAAM,YAAY,GAAG,IAAI;oBACvB,CAAC,CAAC,IAAI,CAAC,iBAAiB,GAAG,CAAC,GAAG,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,GAAG,OAAO;oBACrF,CAAC,CAAC,CAAC,CAAC;gBACN,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,EAAE,YAAY,IAAI,CAAC,CAAC;gBACnD,IAAI,UAAU,IAAI,YAAY,EAAE,CAAC;oBAC/B,IAAI,CAAC,eAAe,GAAG;wBACrB,UAAU;wBACV,YAAY;wBACZ,UAAU,EAAE,UAAU,GAAG,YAAY;qBACtC,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,0CAA0C,EAAE,GAAG,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IAED,KAAK,CAAC,UAAU;QACd,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,SAAiB,EAAE,IAAkB;QAClD,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACxB,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,SAAiB,EAAE,IAAkB;QAC3D,IAAI,IAAI,CAAC,UAAU,EAAE,OAAO;YAAE,OAAO;QACrC,MAAM,CAAC,IAAI,CAAC,8CAA8C,SAAS,EAAE,CAAC,CAAC;QACvE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAC5D,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IACtC,CAAC;IAED,aAAa;QACX,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,OAAO,IAAI,KAAK,CAAC;QAClD,OAAO;YACL,OAAO;YACP,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,EAAE,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;YAClE,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,IAAI,IAAI;YACnC,WAAW,EAAE,IAAI,CAAC,oBAAoB;YACtC,cAAc,EAAE,IAAI,CAAC,eAAe;YACpC,cAAc,EAAE,IAAI,CAAC,eAAe;SACrC,CAAC;IACJ,CAAC;IAED,cAAc;QACZ,OAAO,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACtE,CAAC;IAED,+EAA+E;IAE/E,KAAK,CAAC,aAAa,CAAC,SAAiB;QACnC,IAAI,IAAI,CAAC,eAAe,EAAE,OAAO,IAAI,IAAI,CAAC,eAAe,CAAC,SAAS,KAAK,SAAS;YAAE,OAAO;QAC1F,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,IAAI,CAAC,eAAe,GAAG,IAAI,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;QAEnG,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;QAChD,MAAM,CAAC,IAAI,CAAC,mDAAmD,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9F,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACzB,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QACpC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,sBAAsB,CAAC,SAAiB;QAC5C,IAAI,IAAI,CAAC,eAAe,EAAE,OAAO;YAAE,OAAO;QAC1C,MAAM,CAAC,IAAI,CAAC,mDAAmD,SAAS,EAAE,CAAC,CAAC;QAC5E,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAC5D,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IACrC,CAAC;IAED,kBAAkB;QAChB,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,EAAE,OAAO,IAAI,KAAK,CAAC;QACvD,OAAO;YACL,OAAO;YACP,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,EAAE,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;YACvE,IAAI,EAAE,IAAI,CAAC,eAAe,EAAE,IAAI,IAAI,IAAI;YACxC,WAAW,EAAE,IAAI;YACjB,cAAc,EAAE,IAAI;SACrB,CAAC;IACJ,CAAC;IAED,mBAAmB;QACjB,OAAO,IAAI,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IAChF,CAAC;IAED,+EAA+E;IAE/E,SAAS;QACP,OAAO;YACL,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,SAAS,EAAE,IAAI,CAAC,WAAW,EAAE;YAC7B,IAAI,EAAE,IAAI,CAAC,aAAa,EAAE;YAC1B,SAAS,EAAE,IAAI,CAAC,kBAAkB,EAAE;SACrC,CAAC;IACJ,CAAC;IAED,WAAW;QACT,sBAAsB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACxC,CAAC;IAED,8EAA8E;IAE9E,gBAAgB;QACd,OAAO,mBAAmB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC5C,CAAC;IAED,eAAe;QACb,MAAM,OAAO,GAAG,mBAAmB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACnD,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC;QAC1B,OAAO,iBAAiB,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,cAAc;QAClB,OAAO,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,OAAO,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACxC,CAAC;IAED,+EAA+E;IAEvE,KAAK,CAAC,gBAAgB,CAAC,SAAiB;QAC9C,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QACtD,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,gBAAgB,SAAS,iCAAiC,CAAC,CAAC;QACzF,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}

package/dist/lib/local-llm/gguf-reader.d.ts

@@ -0,0 +1,20 @@
+export interface GGUFModelInfo {
+    contextLength: number;
+    blockCount: number;
+    embeddingLength: number;
+    headCount: number;
+    headCountKv: number;
+    fileSizeBytes: number;
+}
+/**
+ * Reads model architecture info from a GGUF file's metadata header.
+ * Only reads the first 1MB — no model loading required.
+ */
+export declare function readGGUFModelInfo(modelPath: string): Promise<GGUFModelInfo | null>;
+/**
+ * Estimates KV cache bytes per token for a model.
+ * KV cache = 2 (K+V) * n_layers * n_kv_heads * head_dim * 2 bytes (f16)
+ */
+export declare function kvCacheBytesPerToken(info: GGUFModelInfo): number;
+export declare function calculateOptimalContextSize(info: GGUFModelInfo): number;
+//# sourceMappingURL=gguf-reader.d.ts.map

package/dist/lib/local-llm/gguf-reader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gguf-reader.d.ts","sourceRoot":"","sources":["../../../lib/local-llm/gguf-reader.ts"],"names":[],"mappings":"AASA,MAAM,WAAW,aAAa;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;;GAGG;AACH,wBAAsB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAkExF;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,aAAa,GAAG,MAAM,CAGhE;AA2BD,wBAAgB,2BAA2B,CAAC,IAAI,EAAE,aAAa,GAAG,MAAM,CAuBvE"}