node-llama-cpp 3.0.0-beta.4 → 3.0.0-beta.40

package/bins/mac-x64/_nlcBuildMetadata.json

@@ -0,0 +1 @@
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"mac","platformInfo":{"name":"macOS","version":"22.6.0"},"arch":"x64","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b3488"}}}

package/bins/win-arm64/_nlcBuildMetadata.json

@@ -0,0 +1 @@
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"win","platformInfo":{"name":"Windows","version":"10.0.20348"},"arch":"arm64","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b3488"}}}

package/bins/win-x64/_nlcBuildMetadata.json

@@ -0,0 +1 @@
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"win","platformInfo":{"name":"Windows","version":"10.0.17763"},"arch":"x64","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b3488"}}}

package/bins/win-x64-vulkan/_nlcBuildMetadata.json

@@ -0,0 +1 @@
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"win","platformInfo":{"name":"Windows","version":"10.0.17763"},"arch":"x64","gpu":"vulkan","llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b3488"}}}

package/dist/ChatWrapper.d.ts

@@ -1,49 +1,19 @@
-import { ChatHistoryItem, ChatModelFunctions, ChatModelResponse } from "./types.js";
+import { ChatHistoryItem, ChatModelFunctionCall, ChatModelFunctions, ChatModelResponse, ChatWrapperGenerateContextStateOptions, ChatWrapperGeneratedContextState, ChatWrapperGenerateInitialHistoryOptions, ChatWrapperSettings } from "./types.js";
 import { LlamaText } from "./utils/LlamaText.js";
-export type ChatWrapperSettings = {
-    readonly functions: {
-        readonly call: {
-            readonly optionalPrefixSpace: boolean;
-            readonly prefix: string;
-            readonly paramsPrefix: string;
-            readonly suffix: string;
-        };
-        readonly result: {
-            readonly prefix: string;
-            readonly suffix: string;
-        };
-    };
-};
 export declare abstract class ChatWrapper {
+    static defaultSettings: ChatWrapperSettings;
     abstract readonly wrapperName: string;
     readonly settings: ChatWrapperSettings;
-    generateContextText(history: readonly ChatHistoryItem[], { availableFunctions, documentFunctionParams }?: {
-        availableFunctions?: ChatModelFunctions;
-        documentFunctionParams?: boolean;
-    }): {
-        contextText: LlamaText;
-        stopGenerationTriggers: LlamaText[];
-        ignoreStartText?: LlamaText[];
-        functionCall?: {
-            initiallyEngaged: boolean;
-            disengageInitiallyEngaged: LlamaText[];
-        };
-    };
-    generateFunctionCallAndResult(name: string, params: any, result: any): string;
-    generateFunctionCall(name: string, params: any): string;
-    generateFunctionCallResult(functionName: string, functionParams: any, result: any): string;
-    generateModelResponseText(modelResponse: ChatModelResponse["response"]): string;
+    generateContextState({ chatHistory, availableFunctions, documentFunctionParams }: ChatWrapperGenerateContextStateOptions): ChatWrapperGeneratedContextState;
+    generateFunctionCallsAndResults(functionCalls: ChatModelFunctionCall[], useRawCall?: boolean): import("./utils/LlamaText.js")._LlamaText;
+    generateFunctionCall(name: string, params: any): LlamaText;
+    generateFunctionCallResult(functionName: string, functionParams: any, result: any): LlamaText;
+    generateModelResponseText(modelResponse: ChatModelResponse["response"], useRawCall?: boolean): LlamaText;
     generateAvailableFunctionsSystemText(availableFunctions: ChatModelFunctions, { documentParams }: {
         documentParams?: boolean;
-    }): string;
+    }): LlamaText;
     addAvailableFunctionsSystemMessageToHistory(history: readonly ChatHistoryItem[], availableFunctions?: ChatModelFunctions, { documentParams }?: {
         documentParams?: boolean;
     }): readonly ChatHistoryItem[];
-    /**
-     * Functions that should be made available as part of the function calling grammar and are handled by the chat wrapper
-     * for grammar purposes only
-     */
-    getInternalBuiltinFunctions({ initialFunctionCallEngaged }: {
-        initialFunctionCallEngaged: boolean;
-    }): ChatModelFunctions;
+    generateInitialChatHistory({ systemPrompt }: ChatWrapperGenerateInitialHistoryOptions): ChatHistoryItem[];
 }

package/dist/ChatWrapper.js

@@ -1,99 +1,151 @@
-import { LlamaText } from "./utils/LlamaText.js";
-import { getTypeScriptTypeStringForGbnfJsonSchema } from "./utils/getTypeScriptTypeStringForGbnfJsonSchema.js";
+import { LlamaText, SpecialTokensText } from "./utils/LlamaText.js";
+import { ChatModelFunctionsDocumentationGenerator } from "./chatWrappers/utils/ChatModelFunctionsDocumentationGenerator.js";
+import { jsonDumps } from "./chatWrappers/utils/jsonDumps.js";
+import { defaultChatSystemPrompt } from "./config.js";
 export class ChatWrapper {
-    settings = {
+    static defaultSettings = {
+        supportsSystemMessages: true,
         functions: {
             call: {
                 optionalPrefixSpace: true,
-                prefix: "[[call: ",
-                paramsPrefix: "(",
-                suffix: ")]]"
+                prefix: "||call: ",
+                paramsPrefix: LlamaText(new SpecialTokensText("(")),
+                suffix: LlamaText(new SpecialTokensText(")"))
             },
             result: {
-                prefix: " [[result: ",
-                suffix: "]]"
+                prefix: LlamaText(new SpecialTokensText("\n"), "||result: "),
+                suffix: LlamaText(new SpecialTokensText("\n"))
             }
         }
     };
-    generateContextText(history, { availableFunctions, documentFunctionParams } = {}) {
-        const historyWithFunctions = this.addAvailableFunctionsSystemMessageToHistory(history, availableFunctions, {
+    settings = ChatWrapper.defaultSettings;
+    generateContextState({ chatHistory, availableFunctions, documentFunctionParams }) {
+        const historyWithFunctions = this.addAvailableFunctionsSystemMessageToHistory(chatHistory, availableFunctions, {
             documentParams: documentFunctionParams
         });
         const texts = historyWithFunctions
             .map((item) => {
             if (item.type === "system")
-                return LlamaText `system: ${item.text}`;
+                return LlamaText(["system: ", LlamaText.fromJSON(item.text)]);
             else if (item.type === "user")
-                return LlamaText `user: ${item.text}`;
+                return LlamaText(["user: ", item.text]);
             else if (item.type === "model")
-                return LlamaText `model: ${this.generateModelResponseText(item.response)}`;
+                return LlamaText(["model: ", this.generateModelResponseText(item.response)]);
             return item;
         });
         return {
-            contextText: LlamaText(texts).joinValues("\n"),
+            contextText: LlamaText.joinValues("\n", texts),
             stopGenerationTriggers: []
         };
     }
-    generateFunctionCallAndResult(name, params, result) {
-        return this.generateFunctionCall(name, params) + this.generateFunctionCallResult(name, params, result);
+    generateFunctionCallsAndResults(functionCalls, useRawCall = true) {
+        const calls = [];
+        const results = [];
+        const res = [];
+        if (functionCalls.length === 0)
+            return LlamaText([]);
+        for (const functionCall of functionCalls) {
+            if (useRawCall && functionCall.rawCall != null)
+                calls.push(LlamaText.fromJSON(functionCall.rawCall));
+            else
+                calls.push(this.generateFunctionCall(functionCall.name, functionCall.params));
+            results.push(this.generateFunctionCallResult(functionCall.name, functionCall.params, functionCall.result));
+        }
+        if (this.settings.functions.parallelism == null) {
+            for (let i = 0; i < calls.length; i++) {
+                res.push(calls[i]);
+                res.push(results[i]);
+            }
+            return LlamaText(res);
+        }
+        res.push(LlamaText(this.settings.functions.parallelism.call.sectionPrefix ?? ""));
+        for (let i = 0; i < calls.length; i++) {
+            if (i > 0)
+                res.push(LlamaText(this.settings.functions.parallelism.call.betweenCalls ?? ""));
+            res.push(calls[i]);
+        }
+        res.push(LlamaText(this.settings.functions.parallelism.call.sectionSuffix ?? ""));
+        res.push(LlamaText(this.settings.functions.parallelism.result?.sectionPrefix ?? ""));
+        for (let i = 0; i < results.length; i++) {
+            if (i > 0)
+                res.push(LlamaText(this.settings.functions.parallelism.result?.betweenResults ?? ""));
+            res.push(results[i]);
+        }
+        res.push(LlamaText(this.settings.functions.parallelism.result?.sectionSuffix ?? ""));
+        return LlamaText(res);
     }
     generateFunctionCall(name, params) {
-        return this.settings.functions.call.prefix +
-            name +
-            this.settings.functions.call.paramsPrefix +
+        return LlamaText([
+            this.settings.functions.call.prefix,
+            name,
+            this.settings.functions.call.paramsPrefix,
             (params === undefined
                 ? ""
-                : JSON.stringify(params)) +
-            this.settings.functions.call.suffix;
+                : jsonDumps(params)),
+            this.settings.functions.call.suffix
+        ]);
     }
     generateFunctionCallResult(functionName, functionParams, result) {
-        const resolveParameters = (text) => text.replaceAll("{{functionName}}", functionName)
-            .replaceAll("{{functionParams}}", functionParams === undefined ? "" : JSON.stringify(functionParams));
-        return resolveParameters(this.settings.functions.result.prefix) +
+        function resolveParameters(text) {
+            return LlamaText(text)
+                .mapValues((value) => {
+                if (typeof value !== "string")
+                    return value;
+                return value
+                    .replaceAll("{{functionName}}", functionName)
+                    .replaceAll("{{functionParams}}", functionParams === undefined ? "" : jsonDumps(functionParams));
+            });
+        }
+        return LlamaText([
+            resolveParameters(this.settings.functions.result.prefix),
             (result === undefined
                 ? "void"
-                : JSON.stringify(result)) +
-            resolveParameters(this.settings.functions.result.suffix);
+                : jsonDumps(result)),
+            resolveParameters(this.settings.functions.result.suffix)
+        ]);
     }
-    generateModelResponseText(modelResponse) {
-        return modelResponse
-            .map((item) => {
-            if (typeof item === "string")
-                return item;
-            return item.raw ?? this.generateFunctionCallAndResult(item.name, item.params, item.result);
-        })
-            .join("\n");
+    generateModelResponseText(modelResponse, useRawCall = true) {
+        const res = [];
+        const pendingFunctionCalls = [];
+        const addFunctionCalls = () => {
+            if (pendingFunctionCalls.length === 0)
+                return;
+            res.push(this.generateFunctionCallsAndResults(pendingFunctionCalls, useRawCall));
+            pendingFunctionCalls.length = 0;
+        };
+        for (const response of modelResponse) {
+            if (typeof response === "string") {
+                addFunctionCalls();
+                res.push(LlamaText(response));
+                continue;
+            }
+            if (response.startsNewChunk)
+                addFunctionCalls();
+            pendingFunctionCalls.push(response);
+        }
+        addFunctionCalls();
+        return LlamaText(res);
     }
     generateAvailableFunctionsSystemText(availableFunctions, { documentParams = true }) {
-        const availableFunctionNames = Object.keys(availableFunctions ?? {});
-        if (availableFunctionNames.length === 0)
-            return "";
-        return "The assistant calls the provided functions as needed to retrieve information instead of relying on things it already knows.\n" +
-            "Provided functions:\n```\n" +
-            availableFunctionNames
-                .map((functionName) => {
-                const functionDefinition = availableFunctions[functionName];
-                let res = "";
-                if (functionDefinition?.description != null && functionDefinition.description.trim() !== "")
-                    res += "// " + functionDefinition.description.split("\n").join("\n// ") + "\n";
-                res += "function " + functionName + "(";
-                if (documentParams && functionDefinition?.params != null)
-                    res += "params: " + getTypeScriptTypeStringForGbnfJsonSchema(functionDefinition.params);
-                else if (!documentParams && functionDefinition?.params != null)
-                    res += "params";
-                res += ");";
-                return res;
-            })
-                .join("\n\n") +
-            "\n```\n\n" +
-            "Calling any of the provided functions can be done like this:\n" +
-            this.settings.functions.call.prefix.trimStart() +
-            "functionName" +
-            this.settings.functions.call.paramsPrefix +
-            '{ someKey: "someValue" }' +
-            this.settings.functions.call.suffix + "\n\n" +
-            "After calling a function the result will appear afterwards and be visible only to the assistant, so the assistant has to tell the user about it outside of the function call context.\n" +
-            "The assistant calls the functions in advance before telling the user about the result";
+        const functionsDocumentationGenerator = new ChatModelFunctionsDocumentationGenerator(availableFunctions);
+        if (!functionsDocumentationGenerator.hasAnyFunctions)
+            return LlamaText([]);
+        return LlamaText.joinValues("\n", [
+            "The assistant calls the provided functions as needed to retrieve information instead of relying on existing knowledge.",
+            "To fulfill a request, the assistant calls relevant functions in advance when needed before responding to the request, and does not tell the user prior to calling a function.",
+            "Provided functions:",
+            "```typescript",
+            functionsDocumentationGenerator.getTypeScriptFunctionSignatures({ documentParams }),
+            "```",
+            "",
+            "Calling any of the provided functions can be done like this:",
+            this.generateFunctionCall("getSomeInfo", { someKey: "someValue" }),
+            "",
+            "Note that the || prefix is mandatory.",
+            "The assistant does not inform the user about using functions and does not explain anything before calling a function.",
+            "After calling a function, the raw result appears afterwards and is not part of the conversation.",
+            "To make information be part of the conversation, the assistant paraphrases and repeats the information without the function syntax."
+        ]);
     }
     addAvailableFunctionsSystemMessageToHistory(history, availableFunctions, { documentParams = true } = {}) {
         const availableFunctionNames = Object.keys(availableFunctions ?? {});
@@ -103,18 +155,23 @@ export class ChatWrapper {
         const firstNonSystemMessageIndex = res.findIndex((item) => item.type !== "system");
         res.splice(Math.max(0, firstNonSystemMessageIndex), 0, {
             type: "system",
-            text: this.generateAvailableFunctionsSystemText(availableFunctions, { documentParams })
+            text: this.generateAvailableFunctionsSystemText(availableFunctions, { documentParams }).toJSON()
         });
         return res;
     }
-    /**
-     * Functions that should be made available as part of the function calling grammar and are handled by the chat wrapper
-     * for grammar purposes only
-     */
-    getInternalBuiltinFunctions({ initialFunctionCallEngaged }) {
-        if (initialFunctionCallEngaged)
-            return {};
-        return {};
+    generateInitialChatHistory({ systemPrompt = defaultChatSystemPrompt }) {
+        return [{
+                type: "system",
+                text: LlamaText(systemPrompt ?? defaultChatSystemPrompt).toJSON()
+            }];
+    }
+    /** @internal */
+    static _getOptionConfigurationsToTestIfCanSupersedeJinjaTemplate() {
+        return [{}];
+    }
+    /** @internal */ // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    static _checkModelCompatibility(options) {
+        return true;
     }
 }
 //# sourceMappingURL=ChatWrapper.js.map

package/dist/ChatWrapper.js.map

@@ -1 +1 @@
-{"version":3,"file":"ChatWrapper.js","sourceRoot":"","sources":["../src/ChatWrapper.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,SAAS,EAAC,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAC,wCAAwC,EAAC,MAAM,qDAAqD,CAAC;AAiB7G,MAAM,OAAgB,WAAW;IAGb,QAAQ,GAAwB;QAC5C,SAAS,EAAE;YACP,IAAI,EAAE;gBACF,mBAAmB,EAAE,IAAI;gBACzB,MAAM,EAAE,UAAU;gBAClB,YAAY,EAAE,GAAG;gBACjB,MAAM,EAAE,KAAK;aAChB;YACD,MAAM,EAAE;gBACJ,MAAM,EAAE,aAAa;gBACrB,MAAM,EAAE,IAAI;aACf;SACJ;KACJ,CAAC;IAEK,mBAAmB,CAAC,OAAmC,EAAE,EAAC,kBAAkB,EAAE,sBAAsB,KAGvG,EAAE;QASF,MAAM,oBAAoB,GAAG,IAAI,CAAC,2CAA2C,CAAC,OAAO,EAAE,kBAAkB,EAAE;YACvG,cAAc,EAAE,sBAAsB;SACzC,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,oBAAoB;aAC7B,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACV,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ;gBACtB,OAAO,SAAS,CAAC,WAAW,IAAI,CAAC,IAAI,EAAE,CAAC;iBACvC,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM;gBACzB,OAAO,SAAS,CAAC,SAAS,IAAI,CAAC,IAAI,EAAE,CAAC;iBACrC,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO;gBAC1B,OAAO,SAAS,CAAC,UAAU,IAAI,CAAC,yBAAyB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YAE/E,OAAO,IAAoB,CAAC;QAChC,CAAC,CAAC,CAAC;QAEP,OAAO;YACH,WAAW,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC;YAC9C,sBAAsB,EAAE,EAAE;SAC7B,CAAC;IACN,CAAC;IAEM,6BAA6B,CAAC,IAAY,EAAE,MAAW,EAAE,MAAW;QACvE,OAAO,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC,0BAA0B,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3G,CAAC;IAEM,oBAAoB,CAAC,IAAY,EAAE,MAAW;QACjD,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM;YACtC,IAAI;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY;YACzC,CACI,MAAM,KAAK,SAAS;gBAChB,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAC/B;YACD,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC;IAC5C,CAAC;IAEM,0BAA0B,CAAC,YAAoB,EAAE,cAAmB,EAAE,MAAW;QACpF,MAAM,iBAAiB,GAAG,CAAC,IAAY,EAAE,EAAE,CACvC,IAAI,CAAC,UAAU,CAAC,kBAAkB,EAAE,YAAY,CAAC;aAC5C,UAAU,CAAC,oBAAoB,EAAE,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC;QAE9G,OAAO,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC;YAC3D,CACI,MAAM,KAAK,SAAS;gBAChB,CAAC,CAAC,MAAM;gBACR,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAC/B;YACD,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACjE,CAAC;IAEM,yBAAyB,CAAC,aAA4C;QACzE,OAAO,aAAa;aACf,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACV,IAAI,OAAO,IAAI,KAAK,QAAQ;gBACxB,OAAO,IAAI,CAAC;YAEhB,OAAO,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,6BAA6B,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/F,CAAC,CAAC;aACD,IAAI,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IAEM,oCAAoC,CAAC,kBAAsC,EAAE,EAAC,cAAc,GAAG,IAAI,EAEzG;QACG,MAAM,sBAAsB,GAAG,MAAM,CAAC,IAAI,CAAC,kBAAkB,IAAI,EAAE,CAAC,CAAC;QAErE,IAAI,sBAAsB,CAAC,MAAM,KAAK,CAAC;YACnC,OAAO,EAAE,CAAC;QAEd,OAAO,+HAA+H;YAClI,4BAA4B;YAC5B,sBAAsB;iBACjB,GAAG,CAAC,CAAC,YAAY,EAAE,EAAE;gBAClB,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,YAAY,CAAC,CAAC;gBAC5D,IAAI,GAAG,GAAG,EAAE,CAAC;gBAEb,IAAI,kBAAkB,EAAE,WAAW,IAAI,IAAI,IAAI,kBAAkB,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE;oBACvF,GAAG,IAAI,KAAK,GAAG,kBAAkB,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC;gBAEnF,GAAG,IAAI,WAAW,GAAG,YAAY,GAAG,GAAG,CAAC;gBAExC,IAAI,cAAc,IAAI,kBAAkB,EAAE,MAAM,IAAI,IAAI;oBACpD,GAAG,IAAI,UAAU,GAAG,wCAAwC,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;qBACvF,IAAI,CAAC,cAAc,IAAI,kBAAkB,EAAE,MAAM,IAAI,IAAI;oBAC1D,GAAG,IAAI,QAAQ,CAAC;gBAEpB,GAAG,IAAI,IAAI,CAAC;gBAEZ,OAAO,GAAG,CAAC;YACf,CAAC,CAAC;iBACD,IAAI,CAAC,MAAM,CAAC;YACjB,WAAW;YAEX,gEAAgE;YAChE,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE;YAC/C,cAAc;YACd,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY;YACzC,0BAA0B;YAC1B,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,MAAM;YAE5C,yLAAyL;YACzL,uFAAuF,CAAC;IAChG,CAAC;IAEM,2CAA2C,CAAC,OAAmC,EAAE,kBAAuC,EAAE,EAC7H,cAAc,GAAG,IAAI,KAGrB,EAAE;QACF,MAAM,sBAAsB,GAAG,MAAM,CAAC,IAAI,CAAC,kBAAkB,IAAI,EAAE,CAAC,CAAC;QAErE,IAAI,kBAAkB,IAAI,IAAI,IAAI,sBAAsB,CAAC,MAAM,KAAK,CAAC;YACjE,OAAO,OAAO,CAAC;QAEnB,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;QAE5B,MAAM,0BAA0B,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;QACnF,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,0BAA0B,CAAC,EAAE,CAAC,EAAE;YACnD,IAAI,EAAE,QAAQ;YACd,IAAI,EAAE,IAAI,CAAC,oCAAoC,CAAC,kBAAkB,EAAE,EAAC,cAAc,EAAC,CAAC;SACxF,CAAC,CAAC;QAEH,OAAO,GAAG,CAAC;IACf,CAAC;IAED;;;OAGG;IACI,2BAA2B,CAAC,EAAC,0BAA0B,EAAwC;QAClG,IAAI,0BAA0B;YAC1B,OAAO,EAAE,CAAC;QAEd,OAAO,EAAE,CAAC;IACd,CAAC;CACJ"}
+{"version":3,"file":"ChatWrapper.js","sourceRoot":"","sources":["../src/ChatWrapper.ts"],"names":[],"mappings":"AAIA,OAAO,EAAC,SAAS,EAAE,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAC,wCAAwC,EAAC,MAAM,kEAAkE,CAAC;AAC1H,OAAO,EAAC,SAAS,EAAC,MAAM,mCAAmC,CAAC;AAC5D,OAAO,EAAC,uBAAuB,EAAC,MAAM,aAAa,CAAC;AAEpD,MAAM,OAAgB,WAAW;IACtB,MAAM,CAAC,eAAe,GAAwB;QACjD,sBAAsB,EAAE,IAAI;QAC5B,SAAS,EAAE;YACP,IAAI,EAAE;gBACF,mBAAmB,EAAE,IAAI;gBACzB,MAAM,EAAE,UAAU;gBAClB,YAAY,EAAE,SAAS,CAAC,IAAI,iBAAiB,CAAC,GAAG,CAAC,CAAC;gBACnD,MAAM,EAAE,SAAS,CAAC,IAAI,iBAAiB,CAAC,GAAG,CAAC,CAAC;aAChD;YACD,MAAM,EAAE;gBACJ,MAAM,EAAE,SAAS,CAAC,IAAI,iBAAiB,CAAC,IAAI,CAAC,EAAE,YAAY,CAAC;gBAC5D,MAAM,EAAE,SAAS,CAAC,IAAI,iBAAiB,CAAC,IAAI,CAAC,CAAC;aACjD;SACJ;KACJ,CAAC;IAGc,QAAQ,GAAwB,WAAW,CAAC,eAAe,CAAC;IAErE,oBAAoB,CAAC,EACxB,WAAW,EAAE,kBAAkB,EAAE,sBAAsB,EAClB;QACrC,MAAM,oBAAoB,GAAG,IAAI,CAAC,2CAA2C,CAAC,WAAW,EAAE,kBAAkB,EAAE;YAC3G,cAAc,EAAE,sBAAsB;SACzC,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,oBAAoB;aAC7B,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACV,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ;gBACtB,OAAO,SAAS,CAAC,CAAC,UAAU,EAAE,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;iBAC7D,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM;gBACzB,OAAO,SAAS,CAAC,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;iBACvC,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO;gBAC1B,OAAO,SAAS,CAAC,CAAC,SAAS,EAAE,IAAI,CAAC,yBAAyB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAEjF,OAAO,IAAoB,CAAC;QAChC,CAAC,CAAC,CAAC;QAEP,OAAO;YACH,WAAW,EAAE,SAAS,CAAC,UAAU,CAAC,IAAI,EAAE,KAAK,CAAC;YAC9C,sBAAsB,EAAE,EAAE;SAC7B,CAAC;IACN,CAAC;IAEM,+BAA+B,CAAC,aAAsC,EAAE,aAAsB,IAAI;QACrG,MAAM,KAAK,GAAgB,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAgB,EAAE,CAAC;QAChC,MAAM,GAAG,GAAgB,EAAE,CAAC;QAE5B,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;YAC1B,OAAO,SAAS,CAAC,EAAE,CAAC,CAAC;QAEzB,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;YACvC,IAAI,UAAU,IAAI,YAAY,CAAC,OAAO,IAAI,IAAI;gBAC1C,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;;gBAErD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;YAElF,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,0BAA0B,CAAC,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;QAC/G,CAAC;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,WAAW,IAAI,IAAI,EAAE,CAAC;YAC9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACpC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnB,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YACzB,CAAC;YAED,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC;QAC1B,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC,CAAC;QAClF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,IAAI,CAAC,GAAG,CAAC;gBACL,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,CAAC;YAErF,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC,CAAC;QAElF,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,WAAW,CAAC,MAAM,EAAE,aAAa,IAAI,EAAE,CAAC,CAAC,CAAC;QACrF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,IAAI,CAAC,GAAG,CAAC;gBACL,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,WAAW,CAAC,MAAM,EAAE,cAAc,IAAI,EAAE,CAAC,CAAC,CAAC;YAE1F,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,WAAW,CAAC,MAAM,EAAE,aAAa,IAAI,EAAE,CAAC,CAAC,CAAC;QAErF,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAEM,oBAAoB,CAAC,IAAY,EAAE,MAAW;QACjD,OAAO,SAAS,CAAC;YACb,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM;YACnC,IAAI;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY;YACzC,CACI,MAAM,KAAK,SAAS;gBAChB,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAC1B;YACD,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM;SACtC,CAAC,CAAC;IACP,CAAC;IAEM,0BAA0B,CAAC,YAAoB,EAAE,cAAmB,EAAE,MAAW;QACpF,SAAS,iBAAiB,CAAC,IAAwB;YAC/C,OAAO,SAAS,CAAC,IAAI,CAAC;iBACjB,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE;gBACjB,IAAI,OAAO,KAAK,KAAK,QAAQ;oBACzB,OAAO,KAAK,CAAC;gBAEjB,OAAO,KAAK;qBACP,UAAU,CAAC,kBAAkB,EAAE,YAAY,CAAC;qBAC5C,UAAU,CAAC,oBAAoB,EAAE,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC;YACzG,CAAC,CAAC,CAAC;QACX,CAAC;QAED,OAAO,SAAS,CAAC;YACb,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC;YACxD,CACI,MAAM,KAAK,SAAS;gBAChB,CAAC,CAAC,MAAM;gBACR,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAC1B;YACD,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC;SAC3D,CAAC,CAAC;IACP,CAAC;IAEM,yBAAyB,CAAC,aAA4C,EAAE,aAAsB,IAAI;QACrG,MAAM,GAAG,GAAgB,EAAE,CAAC;QAC5B,MAAM,oBAAoB,GAA4B,EAAE,CAAC;QAEzD,MAAM,gBAAgB,GAAG,GAAG,EAAE;YAC1B,IAAI,oBAAoB,CAAC,MAAM,KAAK,CAAC;gBACjC,OAAO;YAEX,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,+BAA+B,CAAC,oBAAoB,EAAE,UAAU,CAAC,CAAC,CAAC;YACjF,oBAAoB,CAAC,MAAM,GAAG,CAAC,CAAC;QACpC,CAAC,CAAC;QAEF,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;YACnC,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;gBAC/B,gBAAgB,EAAE,CAAC;gBACnB,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAC9B,SAAS;YACb,CAAC;YAED,IAAI,QAAQ,CAAC,cAAc;gBACvB,gBAAgB,EAAE,CAAC;YAEvB,oBAAoB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxC,CAAC;QAED,gBAAgB,EAAE,CAAC;QAEnB,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAEM,oCAAoC,CAAC,kBAAsC,EAAE,EAAC,cAAc,GAAG,IAAI,EAEzG;QACG,MAAM,+BAA+B,GAAG,IAAI,wCAAwC,CAAC,kBAAkB,CAAC,CAAC;QAEzG,IAAI,CAAC,+BAA+B,CAAC,eAAe;YAChD,OAAO,SAAS,CAAC,EAAE,CAAC,CAAC;QAEzB,OAAO,SAAS,CAAC,UAAU,CAAC,IAAI,EAAE;YAC9B,wHAAwH;YACxH,+KAA+K;YAC/K,qBAAqB;YACrB,eAAe;YACf,+BAA+B,CAAC,+BAA+B,CAAC,EAAC,cAAc,EAAC,CAAC;YACjF,KAAK;YACL,EAAE;YACF,8DAA8D;YAC9D,IAAI,CAAC,oBAAoB,CAAC,aAAa,EAAE,EAAC,OAAO,EAAE,WAAW,EAAC,CAAC;YAChE,EAAE;YACF,uCAAuC;YACvC,uHAAuH;YACvH,kGAAkG;YAClG,qIAAqI;SACxI,CAAC,CAAC;IACP,CAAC;IAEM,2CAA2C,CAAC,OAAmC,EAAE,kBAAuC,EAAE,EAC7H,cAAc,GAAG,IAAI,KAGrB,EAAE;QACF,MAAM,sBAAsB,GAAG,MAAM,CAAC,IAAI,CAAC,kBAAkB,IAAI,EAAE,CAAC,CAAC;QAErE,IAAI,kBAAkB,IAAI,IAAI,IAAI,sBAAsB,CAAC,MAAM,KAAK,CAAC;YACjE,OAAO,OAAO,CAAC;QAEnB,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;QAE5B,MAAM,0BAA0B,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;QACnF,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,0BAA0B,CAAC,EAAE,CAAC,EAAE;YACnD,IAAI,EAAE,QAAQ;YACd,IAAI,EAAE,IAAI,CAAC,oCAAoC,CAAC,kBAAkB,EAAE,EAAC,cAAc,EAAC,CAAC,CAAC,MAAM,EAAE;SACjG,CAAC,CAAC;QAEH,OAAO,GAAG,CAAC;IACf,CAAC;IAEM,0BAA0B,CAAC,EAC9B,YAAY,GAAG,uBAAuB,EACC;QACvC,OAAO,CAAC;gBACJ,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,SAAS,CAAC,YAAY,IAAI,uBAAuB,CAAC,CAAC,MAAM,EAAE;aACpE,CAAC,CAAC;IACP,CAAC;IAED,gBAAgB;IACT,MAAM,CAAC,yDAAyD;QACnE,OAAO,CAAC,EAAE,CAA6F,CAAC;IAC5G,CAAC;IAED,gBAAgB,CAAC,6DAA6D;IACvE,MAAM,CAAC,wBAAwB,CAAC,OAAiD;QACpF,OAAO,IAAI,CAAC;IAChB,CAAC"}

package/dist/apiDocsIndex.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/apiDocsIndex.js

@@ -0,0 +1,7 @@
+/** @internal */
+import { _LlamaText } from "./utils/LlamaText.js";
+/** @internal */
+export * from "./index.js";
+/** @internal */
+export { _LlamaText as LlamaText };
+//# sourceMappingURL=apiDocsIndex.js.map

package/dist/apiDocsIndex.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"apiDocsIndex.js","sourceRoot":"","sources":["../src/apiDocsIndex.ts"],"names":[],"mappings":"AAAA,gBAAgB;AAChB,OAAO,EACH,UAAU,EACb,MAAM,sBAAsB,CAAC;AAE9B,gBAAgB;AAChB,cAAc,YAAY,CAAC;AAE3B,gBAAgB;AAChB,OAAO,EACH,UAAU,IAAI,SAAS,EAC1B,CAAC"}

package/dist/{utils/getBin.d.ts → bindings/AddonTypes.d.ts}

@@ -1,40 +1,79 @@
 import { Token } from "../types.js";
-export declare function getPrebuildBinPath(): Promise<string | null>;
-export declare function loadBin(): Promise<BindingModule>;
 export type BindingModule = {
     AddonModel: {
         new (modelPath: string, params: {
+            addonExports?: BindingModule;
             gpuLayers?: number;
             vocabOnly?: boolean;
             useMmap?: boolean;
             useMlock?: boolean;
+            checkTensors?: boolean;
+            onLoadProgress?(loadPercentage: number): void;
+            hasLoadAbortSignal?: boolean;
+            overridesList?: Array<[key: string, value: number | bigint | boolean | string, type: 0 | 1 | undefined]>;
         }): AddonModel;
     };
+    AddonModelLora: {
+        new (model: AddonModel, filePath: string): AddonModelLora;
+    };
     AddonContext: {
         new (model: AddonModel, params: {
             seed?: number;
             contextSize?: number;
             batchSize?: number;
+            sequences?: number;
+            flashAttention?: boolean;
             logitsAll?: boolean;
-            embedding?: boolean;
+            embeddings?: boolean;
             threads?: number;
         }): AddonContext;
     };
     AddonGrammar: {
         new (grammarPath: string, params?: {
-            printGrammar?: boolean;
+            addonExports?: BindingModule;
+            debugPrintGrammar?: boolean;
         }): AddonGrammar;
     };
     AddonGrammarEvaluationState: {
         new (grammar: AddonGrammar): AddonGrammarEvaluationState;
     };
     systemInfo(): string;
+    getSupportsGpuOffloading(): boolean;
+    getSupportsMmap(): boolean;
+    getSupportsMlock(): boolean;
+    getBlockSizeForGgmlType(ggmlType: number): number | undefined;
+    getTypeSizeForGgmlType(ggmlType: number): number | undefined;
+    getConsts(): {
+        ggmlMaxDims: number;
+        ggmlTypeF16Size: number;
+        ggmlTypeF32Size: number;
+        ggmlTensorOverhead: number;
+        llamaMaxRngState: number;
+        llamaPosSize: number;
+        llamaSeqIdSize: number;
+    };
+    setLogger(logger: (level: number, message: string) => void): void;
+    setLoggerLogLevel(level: number): void;
+    getGpuVramInfo(): {
+        total: number;
+        used: number;
+    };
+    getGpuDeviceInfo(): {
+        deviceNames: string[];
+    };
+    getGpuType(): "cuda" | "vulkan" | "metal" | undefined;
+    init(): Promise<void>;
+    dispose(): Promise<void>;
 };
 export type AddonModel = {
-    dispose(): void;
+    init(): Promise<boolean>;
+    loadLora(lora: AddonModelLora): Promise<void>;
+    abortActiveModelLoad(): void;
+    dispose(): Promise<void>;
     tokenize(text: string, specialTokens: boolean): Uint32Array;
-    detokenize(tokens: Uint32Array): string;
+    detokenize(tokens: Uint32Array, specialTokens?: boolean): string;
     getTrainContextSize(): number;
+    getEmbeddingVectorSize(): number;
     getTotalSize(): number;
     getTotalParameters(): number;
     getModelDescription(): ModelTypeDescription;
@@ -46,15 +85,22 @@ export type AddonModel = {
     suffixToken(): Token;
     eotToken(): Token;
     getTokenString(token: number): string;
+    getTokenAttributes(token: Token): number;
+    isEogToken(token: Token): boolean;
+    getVocabularyType(): number;
+    shouldPrependBosToken(): boolean;
+    getModelSize(): number;
 };
 export type AddonContext = {
-    dispose(): void;
+    init(): Promise<boolean>;
+    dispose(): Promise<void>;
     getContextSize(): number;
     initBatch(size: number): void;
     addToBatch(sequenceId: number, firstTokenSequenceIndex: number, tokens: Uint32Array, generateLogitAtTheEnd: boolean): BatchLogitIndex | undefined;
     decodeBatch(): Promise<void>;
     sampleToken(batchLogitIndex: BatchLogitIndex, options?: {
         temperature?: number;
+        minP?: number;
         topK?: number;
         topP?: number;
         repeatPenalty?: number;
@@ -62,13 +108,18 @@ export type AddonContext = {
         repeatPenaltyPresencePenalty?: number;
         repeatPenaltyFrequencyPenalty?: number;
         grammarEvaluationState?: AddonGrammarEvaluationState;
+        tokenBiasKeys?: Uint32Array;
+        tokenBiasValues?: Float32Array;
     }): Promise<Token>;
     disposeSequence(sequenceId: number): void;
-    removeTokenCellsFromSequence(sequenceId: number, startPos: number, endPos: number): void;
+    removeTokenCellsFromSequence(sequenceId: number, startPos: number, endPos: number): boolean;
     shiftSequenceTokenCells(sequenceId: number, startPos: number, endPos: number, shiftDelta: number): void;
     acceptGrammarEvaluationStateToken(grammarEvaluationState: AddonGrammarEvaluationState, token: Token): void;
-    getEmbedding(): Float64Array;
+    canBeNextTokenForGrammarEvaluationState(grammarEvaluationState: AddonGrammarEvaluationState, token: Token): boolean;
+    getEmbedding(inputTokensLength: number): Float64Array;
+    getStateSize(): number;
     printTimings(): void;
+    setLora(lora: AddonModelLora, scale: number): void;
 };
 export type BatchLogitIndex = number & {
     __batchLogitIndex: never;
@@ -79,6 +130,12 @@ export type AddonGrammar = "AddonGrammar" & {
 export type AddonGrammarEvaluationState = "AddonGrammarEvaluationState" & {
     __brand: never;
 };
+export type AddonModelLora = {
+    usages: number;
+    readonly filePath: string;
+    readonly disposed: boolean;
+    dispose(): Promise<void>;
+};
 export type ModelTypeDescription = `${AddonModelArchName} ${AddonModelTypeName} ${AddonModelFileTypeName}`;
 export type AddonModelArchName = "unknown" | "llama" | "falcon" | "gpt2" | "gptj" | "gptneox" | "mpt" | "baichuan" | "starcoder" | "persimmon" | "refact" | "bloom" | "stablelm";
 export type AddonModelTypeName = "1B" | "3B" | "7B" | "8B" | "13B" | "15B" | "30B" | "34B" | "40B" | "65B" | "70B" | "?B";

package/dist/bindings/AddonTypes.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=AddonTypes.js.map

package/dist/bindings/AddonTypes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AddonTypes.js","sourceRoot":"","sources":["../../src/bindings/AddonTypes.ts"],"names":[],"mappings":""}