inference-server 1.0.0-beta.19

package/dist/engines/gpt4all/engine.js

@@ -0,0 +1,357 @@
+import path from 'node:path';
+import fs from 'node:fs';
+import { loadModel, createCompletion, createEmbedding, DEFAULT_MODEL_LIST_URL, } from 'gpt4all';
+import { LogLevels } from '../../lib/logger.js';
+import { downloadModelFile } from '../../lib/downloadModelFile.js';
+import { acquireFileLock } from '../../lib/acquireFileLock.js';
+import { validateModelFile } from '../../lib/validateModelFile.js';
+import { createChatMessageArray } from './util.js';
+export const autoGpu = true;
+export async function prepareModel({ config, log }, onProgress, signal) {
+    fs.mkdirSync(path.dirname(config.location), { recursive: true });
+    const releaseFileLock = await acquireFileLock(config.location);
+    if (signal?.aborted) {
+        releaseFileLock();
+        return;
+    }
+    log(LogLevels.info, `Preparing gpt4all model at ${config.location}`, {
+        model: config.id,
+    });
+    let modelMeta;
+    let modelList;
+    const modelMetaPath = path.join(path.dirname(config.location), 'models.json');
+    try {
+        if (!fs.existsSync(modelMetaPath)) {
+            const res = await fetch(DEFAULT_MODEL_LIST_URL);
+            modelList = (await res.json());
+            fs.writeFileSync(modelMetaPath, JSON.stringify(modelList, null, 2));
+        }
+        else {
+            modelList = JSON.parse(fs.readFileSync(modelMetaPath, 'utf-8'));
+        }
+        const foundModelMeta = modelList.find((item) => {
+            if (config.md5 && item.md5sum) {
+                return item.md5sum === config.md5;
+            }
+            if (config.url && item.url) {
+                return item.url === config.url;
+            }
+            return item.filename === path.basename(config.location);
+        });
+        if (foundModelMeta) {
+            modelMeta = foundModelMeta;
+        }
+        const validationError = await validateModelFile({
+            ...config,
+            md5: config.md5 || modelMeta?.md5sum,
+        });
+        if (signal?.aborted) {
+            return;
+        }
+        if (validationError) {
+            if (config.url) {
+                log(LogLevels.info, 'Downloading', {
+                    model: config.id,
+                    url: config.url,
+                    location: config.location,
+                    error: validationError,
+                });
+                await downloadModelFile({
+                    url: config.url,
+                    filePath: config.location,
+                    modelsCachePath: config.modelsCachePath,
+                    onProgress,
+                    signal,
+                });
+            }
+            else {
+                throw new Error(`${validationError} - No URL provided`);
+            }
+        }
+        const finalValidationError = await validateModelFile({
+            ...config,
+            md5: config.md5 || modelMeta?.md5sum,
+        });
+        if (finalValidationError) {
+            throw new Error(`Downloaded files are invalid: ${finalValidationError}`);
+        }
+        if (signal?.aborted) {
+            return;
+        }
+        return modelMeta;
+    }
+    catch (error) {
+        throw error;
+    }
+    finally {
+        releaseFileLock();
+    }
+}
+export async function createInstance({ config, log }, signal) {
+    log(LogLevels.info, `Load GPT4All model ${config.location}`);
+    let device = config.device?.gpu ?? 'cpu';
+    if (typeof device === 'boolean') {
+        device = device ? 'gpu' : 'cpu';
+    }
+    else if (device === 'auto') {
+        device = 'cpu';
+    }
+    const loadOpts = {
+        modelPath: path.dirname(config.location),
+        // file: config.file,
+        modelConfigFile: path.dirname(config.location) + '/models.json',
+        allowDownload: false,
+        device: device,
+        ngl: config.device?.gpuLayers ?? 100,
+        nCtx: config.contextSize ?? 2048,
+        // verbose: true,
+        // signal?: // TODO no way to cancel load
+    };
+    let modelType;
+    if (config.task === 'text-completion') {
+        modelType = 'inference';
+    }
+    else if (config.task === 'embedding') {
+        modelType = 'embedding';
+    }
+    else {
+        throw new Error(`Unsupported task type: ${config.task}`);
+    }
+    const instance = await loadModel(path.basename(config.location), {
+        ...loadOpts,
+        type: modelType,
+    });
+    if (config.device?.cpuThreads) {
+        instance.llm.setThreadCount(config.device.cpuThreads);
+    }
+    if ('generate' in instance) {
+        if (config.initialMessages?.length) {
+            let messages = createChatMessageArray(config.initialMessages);
+            let systemPrompt;
+            if (messages[0].role === 'system') {
+                systemPrompt = messages[0].content;
+                messages = messages.slice(1);
+            }
+            await instance.createChatSession({
+                systemPrompt,
+                messages,
+            });
+        }
+        else if (config.prefix) {
+            await instance.generate(config.prefix, {
+                nPredict: 0,
+            });
+        }
+        else {
+            await instance.generate('', {
+                nPredict: 0,
+            });
+        }
+    }
+    return instance;
+}
+export async function disposeInstance(instance) {
+    instance.dispose();
+}
+export async function processTextCompletionTask({ request, config, onChunk }, instance, signal) {
+    if (!('generate' in instance)) {
+        throw new Error('Instance does not support text completion.');
+    }
+    if (!request.prompt) {
+        throw new Error('Prompt is required for text completion.');
+    }
+    let finishReason = 'eogToken';
+    let suffixToRemove;
+    const defaults = config.completionDefaults ?? {};
+    const stopTriggers = request.stop ?? defaults.stop ?? [];
+    const includesStopTriggers = (text) => stopTriggers.find((t) => text.includes(t));
+    const result = await instance.generate(request.prompt, {
+        // @ts-ignore
+        special: true, // allows passing in raw prompt (including <|start|> etc.)
+        promptTemplate: '%1',
+        temperature: request.temperature ?? defaults.temperature,
+        nPredict: request.maxTokens ?? defaults.maxTokens,
+        topP: request.topP ?? defaults.topP,
+        topK: request.topK ?? defaults.topK,
+        minP: request.minP ?? defaults.minP,
+        nBatch: config?.batchSize,
+        repeatLastN: request.repeatPenaltyNum ?? defaults.repeatPenaltyNum,
+        // repeat penalty is doing something different than both frequency and presence penalty
+        // so not falling back to them here.
+        repeatPenalty: request.repeatPenalty ?? defaults.repeatPenalty,
+        // seed: args.seed, // https://github.com/nomic-ai/gpt4all/issues/1952
+        // @ts-ignore
+        onResponseToken: (tokenId, text) => {
+            const matchingTrigger = includesStopTriggers(text);
+            if (matchingTrigger) {
+                finishReason = 'stopTrigger';
+                suffixToRemove = text;
+                return false;
+            }
+            if (onChunk) {
+                onChunk({
+                    text,
+                    tokens: [tokenId],
+                });
+            }
+            return !signal?.aborted;
+        },
+        // @ts-ignore
+        onResponseTokens: ({ tokenIds, text }) => {
+            const matchingTrigger = includesStopTriggers(text);
+            if (matchingTrigger) {
+                finishReason = 'stopTrigger';
+                suffixToRemove = text;
+                return false;
+            }
+            if (onChunk) {
+                onChunk({
+                    text,
+                    tokens: tokenIds,
+                });
+            }
+            return !signal?.aborted;
+        },
+    });
+    if (result.tokensGenerated === request.maxTokens) {
+        finishReason = 'maxTokens';
+    }
+    let responseText = result.text;
+    if (suffixToRemove) {
+        responseText = responseText.slice(0, -suffixToRemove.length);
+    }
+    return {
+        finishReason,
+        text: responseText,
+        promptTokens: result.tokensIngested,
+        completionTokens: result.tokensGenerated,
+        contextTokens: instance.activeChatSession?.promptContext.nPast ?? 0,
+    };
+}
+export async function processChatCompletionTask({ request, config, resetContext, log, onChunk }, instance, signal) {
+    if (!('createChatSession' in instance)) {
+        throw new Error('Instance does not support chat completion.');
+    }
+    let session = instance.activeChatSession;
+    if (!session || resetContext) {
+        log(LogLevels.debug, 'Resetting chat context');
+        let messages = createChatMessageArray(request.messages);
+        let systemPrompt;
+        if (messages[0].role === 'system') {
+            systemPrompt = messages[0].content;
+            messages = messages.slice(1);
+        }
+        // drop last user message
+        if (messages[messages.length - 1].role === 'user') {
+            messages = messages.slice(0, -1);
+        }
+        session = await instance.createChatSession({
+            systemPrompt,
+            messages,
+        });
+    }
+    const conversationMessages = createChatMessageArray(request.messages).filter((m) => m.role !== 'system');
+    const lastMessage = conversationMessages[conversationMessages.length - 1];
+    if (!(lastMessage.role === 'user' && lastMessage.content)) {
+        throw new Error('Chat completions require a final user message.');
+    }
+    const input = lastMessage.content;
+    let finishReason = 'eogToken';
+    let suffixToRemove;
+    const defaults = config.completionDefaults ?? {};
+    const stopTriggers = request.stop ?? defaults.stop ?? [];
+    const includesStopTriggers = (text) => stopTriggers.find((t) => text.includes(t));
+    const result = await createCompletion(session, input, {
+        temperature: request.temperature ?? defaults.temperature,
+        nPredict: request.maxTokens ?? defaults.maxTokens,
+        topP: request.topP ?? defaults.topP,
+        topK: request.topK ?? defaults.topK,
+        minP: request.minP ?? defaults.minP,
+        nBatch: config.batchSize,
+        repeatLastN: request.repeatPenaltyNum ?? defaults.repeatPenaltyNum,
+        repeatPenalty: request.repeatPenalty ?? defaults.repeatPenalty,
+        // seed: args.seed, // see https://github.com/nomic-ai/gpt4all/issues/1952
+        // @ts-ignore
+        onResponseToken: (tokenId, text) => {
+            const matchingTrigger = includesStopTriggers(text);
+            if (matchingTrigger) {
+                finishReason = 'stopTrigger';
+                suffixToRemove = text;
+                return false;
+            }
+            if (onChunk) {
+                onChunk({
+                    text,
+                    tokens: [tokenId],
+                });
+            }
+            return !signal?.aborted;
+        },
+        // @ts-ignore
+        onResponseTokens: ({ tokenIds, text }) => {
+            const matchingTrigger = includesStopTriggers(text);
+            if (matchingTrigger) {
+                finishReason = 'stopTrigger';
+                suffixToRemove = text;
+                return false;
+            }
+            if (onChunk) {
+                onChunk({
+                    tokens: tokenIds,
+                    text,
+                });
+            }
+            return !signal?.aborted;
+        },
+    });
+    if (result.usage.completion_tokens === request.maxTokens) {
+        finishReason = 'maxTokens';
+    }
+    let response = result.choices[0].message.content;
+    if (suffixToRemove) {
+        response = response.slice(0, -suffixToRemove.length);
+    }
+    return {
+        finishReason,
+        message: {
+            role: 'assistant',
+            content: response,
+        },
+        promptTokens: result.usage.prompt_tokens,
+        completionTokens: result.usage.completion_tokens,
+        contextTokens: session.promptContext.nPast,
+    };
+}
+export async function processEmbeddingTask({ request, config }, instance, signal) {
+    if (!('embed' in instance)) {
+        throw new Error('Instance does not support embedding.');
+    }
+    if (!request.input) {
+        throw new Error('Input is required for embedding.');
+    }
+    const texts = [];
+    if (typeof request.input === 'string') {
+        texts.push(request.input);
+    }
+    else if (Array.isArray(request.input)) {
+        for (const input of request.input) {
+            if (typeof input === 'string') {
+                texts.push(input);
+            }
+            else if (input.type === 'text') {
+                texts.push(input.content);
+            }
+            else if (input.type === 'image') {
+                throw new Error('Image inputs not implemented.');
+            }
+        }
+    }
+    const res = await createEmbedding(instance, texts, {
+        dimensionality: request.dimensions,
+    });
+    return {
+        embeddings: res.embeddings,
+        inputTokens: res.n_prompt_tokens,
+    };
+}
+//# sourceMappingURL=engine.js.map

package/dist/engines/gpt4all/engine.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"engine.js","sourceRoot":"","sources":["../../../src/engines/gpt4all/engine.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,MAAM,SAAS,CAAA;AACxB,OAAO,EACN,SAAS,EACT,gBAAgB,EAChB,eAAe,EAKf,sBAAsB,GACtB,MAAM,SAAS,CAAA;AAehB,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAA;AAClD,OAAO,EAAE,iBAAiB,EAAE,MAAM,mCAAmC,CAAA;AACrE,OAAO,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAA;AACjE,OAAO,EAAE,iBAAiB,EAAE,MAAM,mCAAmC,CAAA;AACrE,OAAO,EAAE,sBAAsB,EAAE,MAAM,WAAW,CAAA;AA8BlD,MAAM,CAAC,MAAM,OAAO,GAAG,IAAI,CAAA;AAE3B,MAAM,CAAC,KAAK,UAAU,YAAY,CACjC,EAAE,MAAM,EAAE,GAAG,EAAqC,EAClD,UAAqD,EACrD,MAAoB;IAEpB,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAChE,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IAC9D,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;QACrB,eAAe,EAAE,CAAA;QACjB,OAAM;IACP,CAAC;IACD,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,8BAA8B,MAAM,CAAC,QAAQ,EAAE,EAAE;QACpE,KAAK,EAAE,MAAM,CAAC,EAAE;KAChB,CAAC,CAAA;IACF,IAAI,SAAuC,CAAA;IAC3C,IAAI,SAA6B,CAAA;IACjC,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,aAAa,CAAC,CAAA;IAC7E,IAAI,CAAC;QACJ,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;YACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,sBAAsB,CAAC,CAAA;YAC/C,SAAS,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAuB,CAAA;YACpD,EAAE,CAAC,aAAa,CAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAA;QACpE,CAAC;aAAM,CAAC;YACP,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC,CAAA;QAChE,CAAC;QACD,MAAM,cAAc,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;YAC9C,IAAI,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAC/B,OAAO,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,GAAG,CAAA;YAClC,CAAC;YACD,IAAI,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC5B,OAAO,IAAI,CAAC,GAAG,KAAK,MAAM,CAAC,GAAG,CAAA;YAC/B,CAAC;YACD,OAAO,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;QACxD,CAAC,CAAC,CAAA;QACF,IAAI,cAAc,EAAE,CAAC;YACpB,SAAS,GAAG,cAAc,CAAA;QAC3B,CAAC;QAED,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAAC;YAC/C,GAAG,MAAM;YACT,GAAG,EAAE,MAAM,CAAC,GAAG,IAAI,SAAS,EAAE,MAAM;SACpC,CAAC,CAAA;QACF,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;YACrB,OAAM;QACP,CAAC;QACD,IAAI,eAAe,EAAE,CAAC;YACrB,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC;gBAChB,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,aAAa,EAAE;oBAClC,KAAK,EAAE,MAAM,CAAC,EAAE;oBAChB,GAAG,EAAE,MAAM,CAAC,GAAG;oBACf,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,KAAK,EAAE,eAAe;iBACtB,CAAC,CAAA;gBACF,MAAM,iBAAiB,CAAC;oBACvB,GAAG,EAAE,MAAM,CAAC,GAAG;oBACf,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,eAAe,EAAE,MAAM,CAAC,eAAe;oBACvC,UAAU;oBACV,MAAM;iBACN,CAAC,CAAA;YACH,CAAC;iBAAM,CAAC;gBACP,MAAM,IAAI,KAAK,CAAC,GAAG,eAAe,oBAAoB,CAAC,CAAA;YACxD,CAAC;QACF,CAAC;QAED,MAAM,oBAAoB,GAAG,MAAM,iBAAiB,CAAC;YACpD,GAAG,MAAM;YACT,GAAG,EAAE,MAAM,CAAC,GAAG,IAAI,SAAS,EAAE,MAAM;SACpC,CAAC,CAAA;QACF,IAAI,oBAAoB,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,iCAAiC,oBAAoB,EAAE,CAAC,CAAA;QACzE,CAAC;QACD,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;YACrB,OAAM;QACP,CAAC;QAED,OAAO,SAAS,CAAA;IACjB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,KAAK,CAAA;IACZ,CAAC;YAAS,CAAC;QACV,eAAe,EAAE,CAAA;IAClB,CAAC;AACF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,EAAE,MAAM,EAAE,GAAG,EAAqC,EAAE,MAAoB;IAC5G,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,sBAAsB,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAA;IAC5D,IAAI,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,IAAI,KAAK,CAAA;IACxC,IAAI,OAAO,MAAM,KAAK,SAAS,EAAE,CAAC;QACjC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAA;IAChC,CAAC;SAAM,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QAC9B,MAAM,GAAG,KAAK,CAAA;IACf,CAAC;IACD,MAAM,QAAQ,GAAqB;QAClC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC;QACxC,qBAAqB;QACrB,eAAe,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,cAAc;QAC/D,aAAa,EAAE,KAAK;QACpB,MAAM,EAAE,MAAM;QACd,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,IAAI,GAAG;QACpC,IAAI,EAAE,MAAM,CAAC,WAAW,IAAI,IAAI;QAChC,iBAAiB;QACjB,yCAAyC;KACzC,CAAA;IAED,IAAI,SAAoC,CAAA;IACxC,IAAI,MAAM,CAAC,IAAI,KAAK,iBAAiB,EAAE,CAAC;QACvC,SAAS,GAAG,WAAW,CAAA;IACxB,CAAC;SAAM,IAAI,MAAM,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QACxC,SAAS,GAAG,WAAW,CAAA;IACxB,CAAC;SAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,0BAA0B,MAAM,CAAC,IAAI,EAAE,CAAC,CAAA;IACzD,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE;QAChE,GAAG,QAAQ;QACX,IAAI,EAAE,SAAS;KACf,CAAC,CAAA;IACF,IAAI,MAAM,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC;QAC/B,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;IACtD,CAAC;IAED,IAAI,UAAU,IAAI,QAAQ,EAAE,CAAC;QAC5B,IAAI,MAAM,CAAC,eAAe,EAAE,MAAM,EAAE,CAAC;YACpC,IAAI,QAAQ,GAAG,sBAAsB,CAAC,MAAM,CAAC,eAAe,CAAC,CAAA;YAC7D,IAAI,YAAY,CAAA;YAChB,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACnC,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;gBAClC,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;YAC7B,CAAC;YACD,MAAM,QAAQ,CAAC,iBAAiB,CAAC;gBAChC,YAAY;gBACZ,QAAQ;aACR,CAAC,CAAA;QACH,CAAC;aAAM,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE;gBACtC,QAAQ,EAAE,CAAC;aACX,CAAC,CAAA;QACH,CAAC;aAAM,CAAC;YACP,MAAM,QAAQ,CAAC,QAAQ,CAAC,EAAE,EAAE;gBAC3B,QAAQ,EAAE,CAAC;aACX,CAAC,CAAA;QACH,CAAC;IACF,CAAC;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,QAAyB;IAC9D,QAAQ,CAAC,OAAO,EAAE,CAAA;AACnB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC9C,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAgD,EAC1E,QAAyB,EACzB,MAAoB;IAEpB,IAAI,CAAC,CAAC,UAAU,IAAI,QAAQ,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAA;IAC9D,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAA;IAC3D,CAAC;IAED,IAAI,YAAY,GAA2B,UAAU,CAAA;IACrD,IAAI,cAAkC,CAAA;IAEtC,MAAM,QAAQ,GAAG,MAAM,CAAC,kBAAkB,IAAI,EAAE,CAAA;IAChD,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAA;IACxD,MAAM,oBAAoB,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA;IACzF,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE;QACtD,aAAa;QACb,OAAO,EAAE,IAAI,EAAE,0DAA0D;QACzE,cAAc,EAAE,IAAI;QACpB,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,QAAQ,CAAC,WAAW;QACxD,QAAQ,EAAE,OAAO,CAAC,SAAS,IAAI,QAAQ,CAAC,SAAS;QACjD,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI;QACnC,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI;QACnC,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI;QACnC,MAAM,EAAE,MAAM,EAAE,SAAS;QACzB,WAAW,EAAE,OAAO,CAAC,gBAAgB,IAAI,QAAQ,CAAC,gBAAgB;QAClE,uFAAuF;QACvF,oCAAoC;QACpC,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,QAAQ,CAAC,aAAa;QAC9D,sEAAsE;QACtE,aAAa;QACb,eAAe,EAAE,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE;YAClC,MAAM,eAAe,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAA;YAClD,IAAI,eAAe,EAAE,CAAC;gBACrB,YAAY,GAAG,aAAa,CAAA;gBAC5B,cAAc,GAAG,IAAI,CAAA;gBACrB,OAAO,KAAK,CAAA;YACb,CAAC;YACD,IAAI,OAAO,EAAE,CAAC;gBACb,OAAO,CAAC;oBACP,IAAI;oBACJ,MAAM,EAAE,CAAC,OAAO,CAAC;iBACjB,CAAC,CAAA;YACH,CAAC;YACD,OAAO,CAAC,MAAM,EAAE,OAAO,CAAA;QACxB,CAAC;QACD,aAAa;QACb,gBAAgB,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE;YACxC,MAAM,eAAe,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAA;YAClD,IAAI,eAAe,EAAE,CAAC;gBACrB,YAAY,GAAG,aAAa,CAAA;gBAC5B,cAAc,GAAG,IAAI,CAAA;gBACrB,OAAO,KAAK,CAAA;YACb,CAAC;YACD,IAAI,OAAO,EAAE,CAAC;gBACb,OAAO,CAAC;oBACP,IAAI;oBACJ,MAAM,EAAE,QAAQ;iBAChB,CAAC,CAAA;YACH,CAAC;YACD,OAAO,CAAC,MAAM,EAAE,OAAO,CAAA;QACxB,CAAC;KACD,CAAC,CAAA;IAEF,IAAI,MAAM,CAAC,eAAe,KAAK,OAAO,CAAC,SAAS,EAAE,CAAC;QAClD,YAAY,GAAG,WAAW,CAAA;IAC3B,CAAC;IAED,IAAI,YAAY,GAAG,MAAM,CAAC,IAAI,CAAA;IAC9B,IAAI,cAAc,EAAE,CAAC;QACpB,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,CAAA;IAC7D,CAAC;IAED,OAAO;QACN,YAAY;QACZ,IAAI,EAAE,YAAY;QAClB,YAAY,EAAE,MAAM,CAAC,cAAc;QACnC,gBAAgB,EAAE,MAAM,CAAC,eAAe;QACxC,aAAa,EAAE,QAAQ,CAAC,iBAAiB,EAAE,aAAa,CAAC,KAAK,IAAI,CAAC;KACnE,CAAA;AACF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC9C,EAAE,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,GAAG,EAAE,OAAO,EAAgD,EAC7F,QAAyB,EACzB,MAAoB;IAEpB,IAAI,CAAC,CAAC,mBAAmB,IAAI,QAAQ,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAA;IAC9D,CAAC;IACD,IAAI,OAAO,GAAG,QAAQ,CAAC,iBAAiB,CAAA;IACxC,IAAI,CAAC,OAAO,IAAI,YAAY,EAAE,CAAC;QAC9B,GAAG,CAAC,SAAS,CAAC,KAAK,EAAE,wBAAwB,CAAC,CAAA;QAC9C,IAAI,QAAQ,GAAG,sBAAsB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;QACvD,IAAI,YAAY,CAAA;QAChB,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnC,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;YAClC,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;QAC7B,CAAC;QACD,yBAAyB;QACzB,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACnD,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QACjC,CAAC;QAED,OAAO,GAAG,MAAM,QAAQ,CAAC,iBAAiB,CAAC;YAC1C,YAAY;YACZ,QAAQ;SACR,CAAC,CAAA;IACH,CAAC;IAED,MAAM,oBAAoB,GAAG,sBAAsB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAA;IAExG,MAAM,WAAW,GAAG,oBAAoB,CAAC,oBAAoB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACzE,IAAI,CAAC,CAAC,WAAW,CAAC,IAAI,KAAK,MAAM,IAAI,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3D,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAA;IAClE,CAAC;IACD,MAAM,KAAK,GAAoB,WAAW,CAAC,OAAO,CAAA;IAElD,IAAI,YAAY,GAA2B,UAAU,CAAA;IACrD,IAAI,cAAkC,CAAA;IAEtC,MAAM,QAAQ,GAAG,MAAM,CAAC,kBAAkB,IAAI,EAAE,CAAA;IAChD,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAA;IACxD,MAAM,oBAAoB,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA;IACzF,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE;QACrD,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,QAAQ,CAAC,WAAW;QACxD,QAAQ,EAAE,OAAO,CAAC,SAAS,IAAI,QAAQ,CAAC,SAAS;QACjD,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI;QACnC,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI;QACnC,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI;QACnC,MAAM,EAAE,MAAM,CAAC,SAAS;QACxB,WAAW,EAAE,OAAO,CAAC,gBAAgB,IAAI,QAAQ,CAAC,gBAAgB;QAClE,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,QAAQ,CAAC,aAAa;QAC9D,0EAA0E;QAC1E,aAAa;QACb,eAAe,EAAE,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE;YAClC,MAAM,eAAe,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAA;YAClD,IAAI,eAAe,EAAE,CAAC;gBACrB,YAAY,GAAG,aAAa,CAAA;gBAC5B,cAAc,GAAG,IAAI,CAAA;gBACrB,OAAO,KAAK,CAAA;YACb,CAAC;YACD,IAAI,OAAO,EAAE,CAAC;gBACb,OAAO,CAAC;oBACP,IAAI;oBACJ,MAAM,EAAE,CAAC,OAAO,CAAC;iBACjB,CAAC,CAAA;YACH,CAAC;YACD,OAAO,CAAC,MAAM,EAAE,OAAO,CAAA;QACxB,CAAC;QACD,aAAa;QACb,gBAAgB,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE;YACxC,MAAM,eAAe,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAA;YAClD,IAAI,eAAe,EAAE,CAAC;gBACrB,YAAY,GAAG,aAAa,CAAA;gBAC5B,cAAc,GAAG,IAAI,CAAA;gBACrB,OAAO,KAAK,CAAA;YACb,CAAC;YACD,IAAI,OAAO,EAAE,CAAC;gBACb,OAAO,CAAC;oBACP,MAAM,EAAE,QAAQ;oBAChB,IAAI;iBACJ,CAAC,CAAA;YACH,CAAC;YAED,OAAO,CAAC,MAAM,EAAE,OAAO,CAAA;QACxB,CAAC;KACD,CAAC,CAAA;IAEF,IAAI,MAAM,CAAC,KAAK,CAAC,iBAAiB,KAAK,OAAO,CAAC,SAAS,EAAE,CAAC;QAC1D,YAAY,GAAG,WAAW,CAAA;IAC3B,CAAC;IAED,IAAI,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAA;IAChD,IAAI,cAAc,EAAE,CAAC;QACpB,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,CAAA;IACrD,CAAC;IAED,OAAO;QACN,YAAY;QACZ,OAAO,EAAE;YACR,IAAI,EAAE,WAAW;YACjB,OAAO,EAAE,QAAQ;SACjB;QACD,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,aAAa;QACxC,gBAAgB,EAAE,MAAM,CAAC,KAAK,CAAC,iBAAiB;QAChD,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,KAAK;KAC1C,CAAA;AACF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,EAAE,OAAO,EAAE,MAAM,EAAuB,EACxC,QAAyB,EACzB,MAAoB;IAEpB,IAAI,CAAC,CAAC,OAAO,IAAI,QAAQ,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAA;IACxD,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAA;IACpD,CAAC;IACD,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,IAAI,OAAO,OAAO,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;QACvC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IAC1B,CAAC;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzC,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YACnC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC/B,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAClB,CAAC;iBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAClC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;YAC1B,CAAC;iBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;gBACnC,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAA;YACjD,CAAC;QACF,CAAC;IACF,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,eAAe,CAAC,QAAQ,EAAE,KAAK,EAAE;QAClD,cAAc,EAAE,OAAO,CAAC,UAAU;KAClC,CAAC,CAAA;IAEF,OAAO;QACN,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,WAAW,EAAE,GAAG,CAAC,eAAe;KAChC,CAAA;AACF,CAAC"}

package/dist/engines/gpt4all/util.d.ts

@@ -0,0 +1,3 @@
+import { ChatMessage as GPT4AllChatMessage } from 'gpt4all';
+import { ChatMessage } from '../../types/index.js';
+export declare function createChatMessageArray(messages: ChatMessage[]): GPT4AllChatMessage[];

package/dist/engines/gpt4all/util.js

@@ -0,0 +1,29 @@
+import { flattenMessageTextContent } from '../../lib/flattenMessageTextContent.js';
+export function createChatMessageArray(messages) {
+    const chatMessages = [];
+    let systemPrompt;
+    for (const message of messages) {
+        if (message.role === 'user' || message.role === 'assistant') {
+            chatMessages.push({
+                role: message.role,
+                content: flattenMessageTextContent(message.content),
+            });
+        }
+        else if (message.role === 'system') {
+            if (systemPrompt) {
+                systemPrompt += '\n\n' + message.content;
+            }
+            else {
+                systemPrompt = flattenMessageTextContent(message.content);
+            }
+        }
+    }
+    if (systemPrompt) {
+        chatMessages.unshift({
+            role: 'system',
+            content: systemPrompt,
+        });
+    }
+    return chatMessages;
+}
+//# sourceMappingURL=util.js.map

package/dist/engines/gpt4all/util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../../../src/engines/gpt4all/util.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,yBAAyB,EAAE,MAAM,2CAA2C,CAAA;AAErF,MAAM,UAAU,sBAAsB,CACrC,QAAuB;IAEvB,MAAM,YAAY,GAAyB,EAAE,CAAA;IAC7C,IAAI,YAAgC,CAAA;IACpC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAC7D,YAAY,CAAC,IAAI,CAAC;gBACjB,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,OAAO,EAAE,yBAAyB,CAAC,OAAO,CAAC,OAAO,CAAC;aACnD,CAAC,CAAA;QACH,CAAC;aAAM,IAAI,OAAO,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACtC,IAAI,YAAY,EAAE,CAAC;gBAClB,YAAY,IAAI,MAAM,GAAG,OAAO,CAAC,OAAO,CAAA;YACzC,CAAC;iBAAM,CAAC;gBACP,YAAY,GAAG,yBAAyB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;YAC1D,CAAC;QACF,CAAC;IACF,CAAC;IACD,IAAI,YAAY,EAAE,CAAC;QAClB,YAAY,CAAC,OAAO,CAAC;YACpB,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,YAAY;SACrB,CAAC,CAAA;IACH,CAAC;IACD,OAAO,YAAY,CAAA;AACpB,CAAC"}

package/dist/engines/index.d.ts

@@ -0,0 +1,19 @@
+import type { ModelPool } from '../pool.js';
+import type { ModelStore } from '../store.js';
+import { ModelEngine, EngineStartContext } from '../types/index.js';
+export declare const BuiltInEngines: {
+    readonly gpt4all: "gpt4all";
+    readonly nodeLlamaCpp: "node-llama-cpp";
+    readonly transformersJs: "transformers-js";
+    readonly stableDiffusionCpp: "stable-diffusion-cpp";
+};
+export type BuiltInEngineName = typeof BuiltInEngines[keyof typeof BuiltInEngines];
+export declare const builtInEngineNames: string[];
+export declare class CustomEngine implements ModelEngine {
+    pool: ModelPool;
+    store: ModelStore;
+    start({ pool, store }: EngineStartContext): Promise<void>;
+    prepareModel(): Promise<void>;
+    createInstance(): Promise<void>;
+    disposeInstance(): Promise<void>;
+}

package/dist/engines/index.js

@@ -0,0 +1,21 @@
+export const BuiltInEngines = {
+    gpt4all: 'gpt4all',
+    nodeLlamaCpp: 'node-llama-cpp',
+    transformersJs: 'transformers-js',
+    stableDiffusionCpp: 'stable-diffusion-cpp',
+};
+export const builtInEngineNames = [
+    ...Object.values(BuiltInEngines),
+];
+export class CustomEngine {
+    pool;
+    store;
+    async start({ pool, store }) {
+        this.pool = pool;
+        this.store = store;
+    }
+    async prepareModel() { }
+    async createInstance() { }
+    async disposeInstance() { }
+}
+//# sourceMappingURL=index.js.map

package/dist/engines/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/engines/index.ts"],"names":[],"mappings":"AAIA,MAAM,CAAC,MAAM,cAAc,GAAG;IAC7B,OAAO,EAAE,SAAS;IAClB,YAAY,EAAE,gBAAgB;IAC9B,cAAc,EAAE,iBAAiB;IACjC,kBAAkB,EAAE,sBAAsB;CACjC,CAAA;AAIV,MAAM,CAAC,MAAM,kBAAkB,GAAa;IAC3C,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC;CACvB,CAAA;AAEV,MAAM,OAAO,YAAY;IACxB,IAAI,CAAY;IAChB,KAAK,CAAa;IAClB,KAAK,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,KAAK,EAAsB;QAC9C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA;QAChB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;IACnB,CAAC;IACD,KAAK,CAAC,YAAY,KAAI,CAAC;IACvB,KAAK,CAAC,cAAc,KAAI,CAAC;IACzB,KAAK,CAAC,eAAe,KAAI,CAAC;CAC1B"}

package/dist/engines/node-llama-cpp/engine.d.ts

@@ -0,0 +1,49 @@
+import { LlamaChat, LlamaModel, LlamaContext, LlamaCompletion, LlamaContextSequence, LlamaGrammar, ChatHistoryItem, LlamaChatResponse, LlamaEmbeddingContext, GgufFileInfo, LLamaChatContextShiftOptions, LlamaContextOptions } from 'node-llama-cpp';
+import { EngineChatCompletionResult, EngineTextCompletionResult, EngineTextCompletionArgs, EngineChatCompletionArgs, EngineContext, ToolDefinition, EngineEmbeddingArgs, EngineEmbeddingResult, FileDownloadProgress, ModelConfig, TextCompletionParams, TextCompletionGrammar, ChatMessage } from '../../types/index.js';
+export interface NodeLlamaCppInstance {
+    model: LlamaModel;
+    context: LlamaContext;
+    chat?: LlamaChat;
+    chatHistory: ChatHistoryItem[];
+    grammars: Record<string, LlamaGrammar>;
+    pendingFunctionCalls: Record<string, any>;
+    lastEvaluation?: LlamaChatResponse['lastEvaluation'];
+    embeddingContext?: LlamaEmbeddingContext;
+    completion?: LlamaCompletion;
+    contextSequence: LlamaContextSequence;
+}
+export interface NodeLlamaCppModelMeta {
+    gguf: GgufFileInfo;
+}
+export interface NodeLlamaCppModelConfig extends ModelConfig {
+    location: string;
+    grammars?: Record<string, TextCompletionGrammar>;
+    sha256?: string;
+    completionDefaults?: TextCompletionParams;
+    initialMessages?: ChatMessage[];
+    prefix?: string;
+    tools?: {
+        definitions: Record<string, ToolDefinition>;
+        includeToolDocumentation?: boolean;
+        parallelism?: number;
+    };
+    contextSize?: number;
+    batchSize?: number;
+    lora?: LlamaContextOptions['lora'];
+    contextShiftStrategy?: LLamaChatContextShiftOptions['strategy'];
+    device?: {
+        gpu?: boolean | 'auto' | (string & {});
+        gpuLayers?: number;
+        cpuThreads?: number;
+        memLock?: boolean;
+    };
+}
+export declare const autoGpu = true;
+export declare function prepareModel({ config, log }: EngineContext<NodeLlamaCppModelConfig>, onProgress?: (progress: FileDownloadProgress) => void, signal?: AbortSignal): Promise<{
+    gguf: GgufFileInfo;
+} | undefined>;
+export declare function createInstance({ config, log }: EngineContext<NodeLlamaCppModelConfig>, signal?: AbortSignal): Promise<NodeLlamaCppInstance>;
+export declare function disposeInstance(instance: NodeLlamaCppInstance): Promise<void>;
+export declare function processChatCompletionTask({ request, config, resetContext, log, onChunk }: EngineChatCompletionArgs<NodeLlamaCppModelConfig>, instance: NodeLlamaCppInstance, signal?: AbortSignal): Promise<EngineChatCompletionResult>;
+export declare function processTextCompletionTask({ request, config, resetContext, log, onChunk }: EngineTextCompletionArgs<NodeLlamaCppModelConfig>, instance: NodeLlamaCppInstance, signal?: AbortSignal): Promise<EngineTextCompletionResult>;
+export declare function processEmbeddingTask({ request, config, log }: EngineEmbeddingArgs<NodeLlamaCppModelConfig>, instance: NodeLlamaCppInstance, signal?: AbortSignal): Promise<EngineEmbeddingResult>;