llama-cpp-pro 0.2.2

package/dist/esm/isomorphic/provider.native.js

@@ -0,0 +1,173 @@
+import { Capacitor, registerPlugin } from '@capacitor/core';
+import { LlmError } from './errors';
+import { DefaultModelScheduler } from './model.scheduler';
+const EVENT_ON_TOKEN = '@LlamaCpp_onToken';
+const MAX_MODELS = 5;
+/** Prefer the plugin instance registered by src/index.ts. */
+const getPlugin = () => {
+    var _a, _b, _c, _d;
+    const caps = Capacitor;
+    return ((_d = (_b = (_a = caps.Plugins) === null || _a === void 0 ? void 0 : _a.LlamaCpp) !== null && _b !== void 0 ? _b : (_c = caps.getPlugin) === null || _c === void 0 ? void 0 : _c.call(caps, 'LlamaCpp')) !== null && _d !== void 0 ? _d : registerPlugin('LlamaCpp'));
+};
+export class NativeProvider {
+    constructor() {
+        this.platform = 'native';
+        this.contextByModel = new Map();
+        this.nextContextId = 1;
+        this.scheduler = new DefaultModelScheduler(MAX_MODELS);
+    }
+    async initialize(opts) {
+        await getPlugin().setContextLimit({ limit: MAX_MODELS });
+        await this.loadModel(opts);
+    }
+    async loadModel(opts) {
+        if (!opts.modelId) {
+            throw new LlmError('INVALID_REQUEST', 'modelId is required');
+        }
+        if (!opts.modelPath) {
+            throw new LlmError('INVALID_REQUEST', 'modelPath is required on native provider');
+        }
+        if (this.contextByModel.has(opts.modelId)) {
+            return;
+        }
+        const modelBytes = typeof opts.modelBytes === 'number' ? opts.modelBytes : 0;
+        const reserveBytes = typeof opts.reserveBytes === 'number' ? opts.reserveBytes : undefined;
+        const memory = await this.getMemorySnapshot();
+        if (typeof opts.availableMemoryBytes === 'number') {
+            memory.freeBytes = opts.availableMemoryBytes;
+        }
+        if (typeof opts.totalMemoryBytes === 'number') {
+            memory.totalBytes = opts.totalMemoryBytes;
+        }
+        this.scheduler.ensureCapacity(opts.modelId, modelBytes, memory, reserveBytes);
+        const contextId = this.nextContextId++;
+        await getPlugin().initContext({
+            contextId,
+            params: {
+                model: opts.modelPath,
+                n_ctx: opts.n_ctx,
+                n_threads: opts.n_threads,
+                embedding: opts.embedding,
+            },
+        });
+        this.contextByModel.set(opts.modelId, contextId);
+        this.scheduler.markLoaded(opts.modelId);
+    }
+    async unloadModel(modelId) {
+        const contextId = this.contextByModel.get(modelId);
+        if (contextId === undefined) {
+            return;
+        }
+        await getPlugin().releaseContext({ contextId });
+        this.contextByModel.delete(modelId);
+        this.scheduler.markUnloaded(modelId);
+    }
+    async generate(req) {
+        var _a, _b;
+        const contextId = this.contextByModel.get(req.modelId);
+        if (contextId === undefined) {
+            throw new LlmError('MODEL_NOT_LOADED', `Model '${req.modelId}' is not loaded`);
+        }
+        const prompt = (_a = req.prompt) !== null && _a !== void 0 ? _a : (_b = req.messages) === null || _b === void 0 ? void 0 : _b.map((m) => `${m.role}: ${m.content}`).join('\n');
+        if (!prompt) {
+            throw new LlmError('INVALID_REQUEST', 'prompt or messages is required');
+        }
+        const completion = await getPlugin().completion({
+            contextId,
+            params: {
+                prompt,
+                n_predict: req.max_tokens,
+                temperature: req.temperature,
+                emit_partial_completion: false,
+            },
+        });
+        return {
+            text: completion.content || completion.text || '',
+            tokens_predicted: completion.tokens_predicted || 0,
+            tokens_evaluated: completion.tokens_evaluated || 0,
+            finish_reason: completion.stopped_limit ? 'length' : 'stop',
+        };
+    }
+    async generateStream(req, onToken) {
+        var _a, _b, _c;
+        const contextId = this.contextByModel.get(req.modelId);
+        if (contextId === undefined) {
+            throw new LlmError('MODEL_NOT_LOADED', `Model '${req.modelId}' is not loaded`);
+        }
+        const prompt = (_a = req.prompt) !== null && _a !== void 0 ? _a : (_b = req.messages) === null || _b === void 0 ? void 0 : _b.map((m) => `${m.role}: ${m.content}`).join('\n');
+        if (!prompt) {
+            throw new LlmError('INVALID_REQUEST', 'prompt or messages is required');
+        }
+        let tokenIndex = 0;
+        const listener = await getPlugin().addListener(EVENT_ON_TOKEN, (evt) => {
+            var _a, _b;
+            if (evt.contextId !== contextId)
+                return;
+            const token = (_b = (_a = evt.tokenResult) === null || _a === void 0 ? void 0 : _a.token) !== null && _b !== void 0 ? _b : '';
+            if (!token)
+                return;
+            onToken({ modelId: req.modelId, token, index: tokenIndex++ });
+        });
+        try {
+            const completion = await getPlugin().completion({
+                contextId,
+                params: {
+                    prompt,
+                    n_predict: req.max_tokens,
+                    temperature: req.temperature,
+                    emit_partial_completion: true,
+                },
+            });
+            return {
+                text: completion.content || completion.text || '',
+                tokens_predicted: completion.tokens_predicted || 0,
+                tokens_evaluated: completion.tokens_evaluated || 0,
+                finish_reason: completion.stopped_limit ? 'length' : 'stop',
+            };
+        }
+        finally {
+            (_c = listener === null || listener === void 0 ? void 0 : listener.remove) === null || _c === void 0 ? void 0 : _c.call(listener);
+        }
+    }
+    async embed(req) {
+        const contextId = this.contextByModel.get(req.modelId);
+        if (contextId === undefined) {
+            throw new LlmError('MODEL_NOT_LOADED', `Model '${req.modelId}' is not loaded`);
+        }
+        const inputs = Array.isArray(req.input) ? req.input : [req.input];
+        const vectors = [];
+        for (const text of inputs) {
+            const res = await getPlugin().embedding({
+                contextId,
+                text,
+                params: {},
+            });
+            vectors.push(res.embedding || []);
+        }
+        return { vectors };
+    }
+    async getMemorySnapshot() {
+        var _a;
+        const memoryFromPerformance = (_a = globalThis === null || globalThis === void 0 ? void 0 : globalThis.performance) === null || _a === void 0 ? void 0 : _a.memory;
+        if (memoryFromPerformance) {
+            const totalBytes = Number(memoryFromPerformance.jsHeapSizeLimit);
+            const usedBytes = Number(memoryFromPerformance.usedJSHeapSize);
+            const freeBytes = Number(memoryFromPerformance.jsHeapSizeLimit - memoryFromPerformance.usedJSHeapSize);
+            const usedRatio = totalBytes > 0 ? usedBytes / totalBytes : 0;
+            const pressure = usedRatio >= 0.85 ? 'high' : usedRatio >= 0.7 ? 'medium' : 'low';
+            return { totalBytes, usedBytes, freeBytes, pressure };
+        }
+        return { pressure: 'unknown' };
+    }
+    async health() {
+        return {
+            ok: true,
+            details: {
+                loadedModels: this.contextByModel.size,
+                maxModels: MAX_MODELS,
+                schedulerLoadedModels: this.scheduler.listLoaded().length,
+            },
+        };
+    }
+}
+//# sourceMappingURL=provider.native.js.map

package/dist/esm/isomorphic/provider.native.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"provider.native.js","sourceRoot":"","sources":["../../../src/isomorphic/provider.native.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAY5D,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACpC,OAAO,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAE1D,MAAM,cAAc,GAAG,mBAAmB,CAAC;AAC3C,MAAM,UAAU,GAAG,CAAC,CAAC;AASrB,6DAA6D;AAC7D,MAAM,SAAS,GAAG,GAAmB,EAAE;;IACrC,MAAM,IAAI,GAAG,SAGZ,CAAC;IACF,OAAO,CACL,MAAA,MAAA,MAAA,IAAI,CAAC,OAAO,0CAAE,QAAQ,mCACtB,MAAA,IAAI,CAAC,SAAS,qDAAG,UAAU,CAAC,mCAC5B,cAAc,CAAiB,UAAU,CAAC,CAC3C,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,OAAO,cAAc;IAA3B;QACW,aAAQ,GAAG,QAAiB,CAAC;QAC9B,mBAAc,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC3C,kBAAa,GAAG,CAAC,CAAC;QAClB,cAAS,GAAG,IAAI,qBAAqB,CAAC,UAAU,CAAC,CAAC;IAmK5D,CAAC;IAjKC,KAAK,CAAC,UAAU,CAAC,IAAuB;QACtC,MAAM,SAAS,EAAE,CAAC,eAAe,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;QACzD,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAAuB;QACrC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,IAAI,QAAQ,CAAC,iBAAiB,EAAE,qBAAqB,CAAC,CAAC;QAC/D,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,IAAI,QAAQ,CAAC,iBAAiB,EAAE,0CAA0C,CAAC,CAAC;QACpF,CAAC;QACD,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1C,OAAO;QACT,CAAC;QAED,MAAM,UAAU,GAAG,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7E,MAAM,YAAY,GAAG,OAAO,IAAI,CAAC,YAAY,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC;QAC3F,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC9C,IAAI,OAAO,IAAI,CAAC,oBAAoB,KAAK,QAAQ,EAAE,CAAC;YAClD,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,oBAAoB,CAAC;QAC/C,CAAC;QACD,IAAI,OAAO,IAAI,CAAC,gBAAgB,KAAK,QAAQ,EAAE,CAAC;YAC9C,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC;QAC5C,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;QAE9E,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACvC,MAAM,SAAS,EAAE,CAAC,WAAW,CAAC;YAC5B,SAAS;YACT,MAAM,EAAE;gBACN,KAAK,EAAE,IAAI,CAAC,SAAS;gBACrB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,SAAS,EAAE,IAAI,CAAC,SAAS;aAC1B;SACF,CAAC,CAAC;QACH,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QACjD,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,OAAe;QAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACnD,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YAC5B,OAAO;QACT,CAAC;QACD,MAAM,SAAS,EAAE,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACpC,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,GAAoB;;QACjC,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACvD,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YAC5B,MAAM,IAAI,QAAQ,CAAC,kBAAkB,EAAE,UAAU,GAAG,CAAC,OAAO,iBAAiB,CAAC,CAAC;QACjF,CAAC;QACD,MAAM,MAAM,GAAG,MAAA,GAAG,CAAC,MAAM,mCAAI,MAAA,GAAG,CAAC,QAAQ,0CAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,EAAE,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5F,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,QAAQ,CAAC,iBAAiB,EAAE,gCAAgC,CAAC,CAAC;QAC1E,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,SAAS,EAAE,CAAC,UAAU,CAAC;YAC9C,SAAS;YACT,MAAM,EAAE;gBACN,MAAM;gBACN,SAAS,EAAE,GAAG,CAAC,UAAU;gBACzB,WAAW,EAAE,GAAG,CAAC,WAAW;gBAC5B,uBAAuB,EAAE,KAAK;aAC/B;SACF,CAAC,CAAC;QAEH,OAAO;YACL,IAAI,EAAE,UAAU,CAAC,OAAO,IAAI,UAAU,CAAC,IAAI,IAAI,EAAE;YACjD,gBAAgB,EAAE,UAAU,CAAC,gBAAgB,IAAI,CAAC;YAClD,gBAAgB,EAAE,UAAU,CAAC,gBAAgB,IAAI,CAAC;YAClD,aAAa,EAAE,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM;SAC5D,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,GAAoB,EAAE,OAAoC;;QAC7E,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACvD,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YAC5B,MAAM,IAAI,QAAQ,CAAC,kBAAkB,EAAE,UAAU,GAAG,CAAC,OAAO,iBAAiB,CAAC,CAAC;QACjF,CAAC;QACD,MAAM,MAAM,GAAG,MAAA,GAAG,CAAC,MAAM,mCAAI,MAAA,GAAG,CAAC,QAAQ,0CAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,EAAE,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5F,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,QAAQ,CAAC,iBAAiB,EAAE,gCAAgC,CAAC,CAAC;QAC1E,CAAC;QAED,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,MAAM,QAAQ,GAAG,MAAO,SAAS,EAAU,CAAC,WAAW,CAAC,cAAc,EAAE,CAAC,GAAqB,EAAE,EAAE;;YAChG,IAAI,GAAG,CAAC,SAAS,KAAK,SAAS;gBAAE,OAAO;YACxC,MAAM,KAAK,GAAG,MAAA,MAAA,GAAG,CAAC,WAAW,0CAAE,KAAK,mCAAI,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK;gBAAE,OAAO;YACnB,OAAO,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,SAAS,EAAE,CAAC,UAAU,CAAC;gBAC9C,SAAS;gBACT,MAAM,EAAE;oBACN,MAAM;oBACN,SAAS,EAAE,GAAG,CAAC,UAAU;oBACzB,WAAW,EAAE,GAAG,CAAC,WAAW;oBAC5B,uBAAuB,EAAE,IAAI;iBAC9B;aACF,CAAC,CAAC;YAEH,OAAO;gBACL,IAAI,EAAE,UAAU,CAAC,OAAO,IAAI,UAAU,CAAC,IAAI,IAAI,EAAE;gBACjD,gBAAgB,EAAE,UAAU,CAAC,gBAAgB,IAAI,CAAC;gBAClD,gBAAgB,EAAE,UAAU,CAAC,gBAAgB,IAAI,CAAC;gBAClD,aAAa,EAAE,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM;aAC5D,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,MAAA,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAE,MAAM,wDAAI,CAAC;QACvB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,GAAiB;QAC3B,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACvD,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YAC5B,MAAM,IAAI,QAAQ,CAAC,kBAAkB,EAAE,UAAU,GAAG,CAAC,OAAO,iBAAiB,CAAC,CAAC;QACjF,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAClE,MAAM,OAAO,GAAe,EAAE,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,MAAM,SAAS,EAAE,CAAC,SAAS,CAAC;gBACtC,SAAS;gBACT,IAAI;gBACJ,MAAM,EAAE,EAAE;aACX,CAAC,CAAC;YACH,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,iBAAiB;;QACrB,MAAM,qBAAqB,GAAG,MAAC,UAAkB,aAAlB,UAAU,uBAAV,UAAU,CAAU,WAAW,0CAAE,MAAM,CAAC;QACvE,IAAI,qBAAqB,EAAE,CAAC;YAC1B,MAAM,UAAU,GAAG,MAAM,CAAC,qBAAqB,CAAC,eAAe,CAAC,CAAC;YACjE,MAAM,SAAS,GAAG,MAAM,CAAC,qBAAqB,CAAC,cAAc,CAAC,CAAC;YAC/D,MAAM,SAAS,GAAG,MAAM,CAAC,qBAAqB,CAAC,eAAe,GAAG,qBAAqB,CAAC,cAAc,CAAC,CAAC;YACvG,MAAM,SAAS,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9D,MAAM,QAAQ,GAAG,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,IAAI,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;YAClF,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;QACxD,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,MAAM;QACV,OAAO;YACL,EAAE,EAAE,IAAI;YACR,OAAO,EAAE;gBACP,YAAY,EAAE,IAAI,CAAC,cAAc,CAAC,IAAI;gBACtC,SAAS,EAAE,UAAU;gBACrB,qBAAqB,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,CAAC,MAAM;aAC1D;SACF,CAAC;IACJ,CAAC;CACF","sourcesContent":["import { Capacitor, registerPlugin } from '@capacitor/core';\nimport type { LlamaCppPlugin } from '../definitions';\nimport type {\n  EmbedRequest,\n  EmbedResult,\n  GenerateRequest,\n  GenerateResult,\n  InitializeOptions,\n  LlmProvider,\n  MemorySnapshot,\n  TokenEvent,\n} from './provider.interface';\nimport { LlmError } from './errors';\nimport { DefaultModelScheduler } from './model.scheduler';\n\nconst EVENT_ON_TOKEN = '@LlamaCpp_onToken';\nconst MAX_MODELS = 5;\n\ntype TokenNativeEvent = {\n  contextId: number;\n  tokenResult: {\n    token?: string;\n  };\n};\n\n/** Prefer the plugin instance registered by src/index.ts. */\nconst getPlugin = (): LlamaCppPlugin => {\n  const caps = Capacitor as unknown as {\n    Plugins?: Record<string, LlamaCppPlugin>;\n    getPlugin?: (name: string) => LlamaCppPlugin | undefined;\n  };\n  return (\n    caps.Plugins?.LlamaCpp ??\n    caps.getPlugin?.('LlamaCpp') ??\n    registerPlugin<LlamaCppPlugin>('LlamaCpp')\n  );\n};\n\nexport class NativeProvider implements LlmProvider {\n  readonly platform = 'native' as const;\n  private contextByModel = new Map<string, number>();\n  private nextContextId = 1;\n  private scheduler = new DefaultModelScheduler(MAX_MODELS);\n\n  async initialize(opts: InitializeOptions): Promise<void> {\n    await getPlugin().setContextLimit({ limit: MAX_MODELS });\n    await this.loadModel(opts);\n  }\n\n  async loadModel(opts: InitializeOptions): Promise<void> {\n    if (!opts.modelId) {\n      throw new LlmError('INVALID_REQUEST', 'modelId is required');\n    }\n    if (!opts.modelPath) {\n      throw new LlmError('INVALID_REQUEST', 'modelPath is required on native provider');\n    }\n    if (this.contextByModel.has(opts.modelId)) {\n      return;\n    }\n\n    const modelBytes = typeof opts.modelBytes === 'number' ? opts.modelBytes : 0;\n    const reserveBytes = typeof opts.reserveBytes === 'number' ? opts.reserveBytes : undefined;\n    const memory = await this.getMemorySnapshot();\n    if (typeof opts.availableMemoryBytes === 'number') {\n      memory.freeBytes = opts.availableMemoryBytes;\n    }\n    if (typeof opts.totalMemoryBytes === 'number') {\n      memory.totalBytes = opts.totalMemoryBytes;\n    }\n    this.scheduler.ensureCapacity(opts.modelId, modelBytes, memory, reserveBytes);\n\n    const contextId = this.nextContextId++;\n    await getPlugin().initContext({\n      contextId,\n      params: {\n        model: opts.modelPath,\n        n_ctx: opts.n_ctx,\n        n_threads: opts.n_threads,\n        embedding: opts.embedding,\n      },\n    });\n    this.contextByModel.set(opts.modelId, contextId);\n    this.scheduler.markLoaded(opts.modelId);\n  }\n\n  async unloadModel(modelId: string): Promise<void> {\n    const contextId = this.contextByModel.get(modelId);\n    if (contextId === undefined) {\n      return;\n    }\n    await getPlugin().releaseContext({ contextId });\n    this.contextByModel.delete(modelId);\n    this.scheduler.markUnloaded(modelId);\n  }\n\n  async generate(req: GenerateRequest): Promise<GenerateResult> {\n    const contextId = this.contextByModel.get(req.modelId);\n    if (contextId === undefined) {\n      throw new LlmError('MODEL_NOT_LOADED', `Model '${req.modelId}' is not loaded`);\n    }\n    const prompt = req.prompt ?? req.messages?.map((m) => `${m.role}: ${m.content}`).join('\\n');\n    if (!prompt) {\n      throw new LlmError('INVALID_REQUEST', 'prompt or messages is required');\n    }\n\n    const completion = await getPlugin().completion({\n      contextId,\n      params: {\n        prompt,\n        n_predict: req.max_tokens,\n        temperature: req.temperature,\n        emit_partial_completion: false,\n      },\n    });\n\n    return {\n      text: completion.content || completion.text || '',\n      tokens_predicted: completion.tokens_predicted || 0,\n      tokens_evaluated: completion.tokens_evaluated || 0,\n      finish_reason: completion.stopped_limit ? 'length' : 'stop',\n    };\n  }\n\n  async generateStream(req: GenerateRequest, onToken: (event: TokenEvent) => void): Promise<GenerateResult> {\n    const contextId = this.contextByModel.get(req.modelId);\n    if (contextId === undefined) {\n      throw new LlmError('MODEL_NOT_LOADED', `Model '${req.modelId}' is not loaded`);\n    }\n    const prompt = req.prompt ?? req.messages?.map((m) => `${m.role}: ${m.content}`).join('\\n');\n    if (!prompt) {\n      throw new LlmError('INVALID_REQUEST', 'prompt or messages is required');\n    }\n\n    let tokenIndex = 0;\n    const listener = await (getPlugin() as any).addListener(EVENT_ON_TOKEN, (evt: TokenNativeEvent) => {\n      if (evt.contextId !== contextId) return;\n      const token = evt.tokenResult?.token ?? '';\n      if (!token) return;\n      onToken({ modelId: req.modelId, token, index: tokenIndex++ });\n    });\n\n    try {\n      const completion = await getPlugin().completion({\n        contextId,\n        params: {\n          prompt,\n          n_predict: req.max_tokens,\n          temperature: req.temperature,\n          emit_partial_completion: true,\n        },\n      });\n\n      return {\n        text: completion.content || completion.text || '',\n        tokens_predicted: completion.tokens_predicted || 0,\n        tokens_evaluated: completion.tokens_evaluated || 0,\n        finish_reason: completion.stopped_limit ? 'length' : 'stop',\n      };\n    } finally {\n      listener?.remove?.();\n    }\n  }\n\n  async embed(req: EmbedRequest): Promise<EmbedResult> {\n    const contextId = this.contextByModel.get(req.modelId);\n    if (contextId === undefined) {\n      throw new LlmError('MODEL_NOT_LOADED', `Model '${req.modelId}' is not loaded`);\n    }\n\n    const inputs = Array.isArray(req.input) ? req.input : [req.input];\n    const vectors: number[][] = [];\n    for (const text of inputs) {\n      const res = await getPlugin().embedding({\n        contextId,\n        text,\n        params: {},\n      });\n      vectors.push(res.embedding || []);\n    }\n    return { vectors };\n  }\n\n  async getMemorySnapshot(): Promise<MemorySnapshot> {\n    const memoryFromPerformance = (globalThis as any)?.performance?.memory;\n    if (memoryFromPerformance) {\n      const totalBytes = Number(memoryFromPerformance.jsHeapSizeLimit);\n      const usedBytes = Number(memoryFromPerformance.usedJSHeapSize);\n      const freeBytes = Number(memoryFromPerformance.jsHeapSizeLimit - memoryFromPerformance.usedJSHeapSize);\n      const usedRatio = totalBytes > 0 ? usedBytes / totalBytes : 0;\n      const pressure = usedRatio >= 0.85 ? 'high' : usedRatio >= 0.7 ? 'medium' : 'low';\n      return { totalBytes, usedBytes, freeBytes, pressure };\n    }\n    return { pressure: 'unknown' };\n  }\n\n  async health(): Promise<{ ok: boolean; details?: Record<string, unknown> }> {\n    return {\n      ok: true,\n      details: {\n        loadedModels: this.contextByModel.size,\n        maxModels: MAX_MODELS,\n        schedulerLoadedModels: this.scheduler.listLoaded().length,\n      },\n    };\n  }\n}\n\n"]}

package/dist/esm/isomorphic/provider.web.d.ts

@@ -0,0 +1,88 @@
+import type { EmbedRequest, EmbedResult, GenerateRequest, GenerateResult, InitializeOptions, LlmProvider, MemorySnapshot, PlatformKind, TokenEvent } from './provider.interface';
+import type { DetokenizeResult, TokenizeResult } from '../workers/wasm.engine';
+/** Verify that the browser supports everything the web WASM path needs. */
+export declare function checkWasmCapabilities(): {
+    supported: boolean;
+    missing: string[];
+};
+/** Returns true only when COOP/COEP headers are set for WASM threads. */
+export declare function checkCrossOriginIsolation(): boolean;
+type WorkerFactory = () => Worker;
+export declare class WebProvider implements LlmProvider {
+    private workerFactoryOverride?;
+    private static globalWorkerFactory?;
+    readonly platform: PlatformKind;
+    private loadedModelIds;
+    private worker;
+    private reqCounter;
+    private pending;
+    private scheduler;
+    constructor(workerFactoryOverride?: WorkerFactory | undefined);
+    static setWorkerFactory(factory?: WorkerFactory): void;
+    private resolveWorkerUrl;
+    private defaultWorkerFactory;
+    private ensureWorker;
+    private sendRequest;
+    initialize(opts: InitializeOptions): Promise<void>;
+    loadModel(opts: InitializeOptions): Promise<void>;
+    private readMeasuredFootprintFromWorker;
+    unloadModel(modelId: string): Promise<void>;
+    generate(req: GenerateRequest): Promise<GenerateResult>;
+    generateStream(req: GenerateRequest, onToken: (event: TokenEvent) => void): Promise<GenerateResult>;
+    embed(req: EmbedRequest): Promise<EmbedResult>;
+    getMemorySnapshot(): Promise<MemorySnapshot>;
+    /** Worker WASM linear memory + loaded-model registry (for scheduling UI). */
+    fetchWorkerMemory(): Promise<Record<string, unknown>>;
+    getWasmMemoryStatus(): Promise<Record<string, unknown>>;
+    tokenize(modelId: string, text: string): Promise<TokenizeResult>;
+    detokenize(modelId: string, tokens: number[]): Promise<DetokenizeResult>;
+    convertJsonSchemaToGrammar(schemaJson: string): Promise<string>;
+    private requireLoaded;
+    rerank(modelId: string, query: string, documents: string[]): Promise<Array<{
+        index: number;
+        score: number;
+    }>>;
+    bench(modelId: string, pp: number, tg: number, pl: number, nr: number): Promise<string>;
+    saveSession(modelId: string, filepath: string, tokenSize: number): Promise<number>;
+    loadSession(modelId: string, filepath: string): Promise<{
+        tokens_loaded: number;
+        prompt: string;
+    }>;
+    applyLoraAdapters(modelId: string, loraAdapters: Array<{
+        path: string;
+        scaled?: number;
+    }>): Promise<void>;
+    removeLoraAdapters(modelId: string): Promise<void>;
+    getLoadedLoraAdapters(modelId: string): Promise<Array<{
+        path: string;
+        scaled?: number;
+    }>>;
+    initMultimodal(modelId: string, path: string, useGpu?: boolean): Promise<boolean>;
+    isMultimodalEnabled(modelId: string): Promise<boolean>;
+    getMultimodalSupport(modelId: string): Promise<{
+        vision: boolean;
+        audio: boolean;
+    }>;
+    releaseMultimodal(modelId: string): Promise<void>;
+    initVocoder(modelId: string, path: string, nBatch?: number): Promise<boolean>;
+    isVocoderEnabled(modelId: string): Promise<boolean>;
+    releaseVocoder(modelId: string): Promise<void>;
+    getFormattedAudioCompletion(modelId: string, speaker: object | null, textToSpeak: string): Promise<{
+        prompt: string;
+        grammar?: string;
+    }>;
+    getAudioCompletionGuideTokens(modelId: string, textToSpeak: string): Promise<number[]>;
+    decodeAudioTokens(modelId: string, tokens: number[]): Promise<number[]>;
+    /**
+     * Terminate the worker mid-inference. WASM is single-threaded, so posting
+     * an abort message cannot be received while generate() is running. Worker
+     * termination is the only reliable interrupt. The model will need to be
+     * reloaded on the next generate() call.
+     */
+    stopGeneration(): void;
+    health(): Promise<{
+        ok: boolean;
+        details?: Record<string, unknown>;
+    }>;
+}
+export {};