node-llama-cpp 3.0.0-beta.9 → 3.0.1

package/dist/chatWrappers/MistralChatWrapper.js

@@ -0,0 +1,169 @@
+import { ChatWrapper } from "../ChatWrapper.js";
+import { SpecialToken, LlamaText, SpecialTokensText } from "../utils/LlamaText.js";
+import { jsonDumps } from "./utils/jsonDumps.js";
+import { chunkChatItems } from "./utils/chunkChatItems.js";
+// source:
+// https://github.com/mistralai/platform-docs-public/blob/02c3f50e427ce5cf96bba9710501598f621babea/docs/guides/tokenization.mdx#v3-tokenizer
+//
+// source: https://docs.mistral.ai/guides/tokenization/#v3-tokenizer
+export class MistralChatWrapper extends ChatWrapper {
+    wrapperName = "Mistral";
+    settings = {
+        supportsSystemMessages: true,
+        functions: {
+            call: {
+                optionalPrefixSpace: true,
+                prefix: '{"name": "',
+                paramsPrefix: '", "arguments": ',
+                suffix: "}"
+            },
+            result: {
+                prefix: '{"name": "{{functionName}}", "content": ',
+                suffix: "}"
+            },
+            parallelism: {
+                call: {
+                    sectionPrefix: LlamaText(new SpecialTokensText("[TOOL_CALLS]"), "["),
+                    betweenCalls: ", ",
+                    sectionSuffix: LlamaText("]", new SpecialToken("EOS"))
+                },
+                result: {
+                    sectionPrefix: LlamaText(new SpecialTokensText("[TOOL_RESULTS]"), "["),
+                    betweenResults: ", ",
+                    sectionSuffix: LlamaText("]", new SpecialTokensText("[/TOOL_RESULTS]"))
+                }
+            }
+        }
+    };
+    /** @internal */ _addSpaceBeforeEos;
+    constructor({ addSpaceBeforeEos = false } = {}) {
+        super();
+        this._addSpaceBeforeEos = addSpaceBeforeEos;
+    }
+    addAvailableFunctionsSystemMessageToHistory(history) {
+        return history;
+    }
+    generateContextState({ chatHistory, availableFunctions, documentFunctionParams }) {
+        const toolsText = this._generateAvailableToolsText({ availableFunctions, documentFunctionParams });
+        const { systemMessage, chatHistory: chatHistoryWithoutSystemMessage } = this._splitSystemMessageFromChatHistory(chatHistory);
+        const { lastInteraction, chatHistory: cleanChatHistory } = this._splitLastInteractionFromChatHistory(chatHistoryWithoutSystemMessage);
+        const chunkedChatHistory = chunkChatItems(cleanChatHistory, {
+            generateModelResponseText: this.generateModelResponseText.bind(this)
+        });
+        const chunkedLastInteraction = chunkChatItems(lastInteraction, {
+            generateModelResponseText: this.generateModelResponseText.bind(this)
+        });
+        const contextText = LlamaText(new SpecialToken("BOS"), chunkedChatHistory.map(({ system, user, model }) => {
+            return LlamaText([
+                new SpecialTokensText("[INST]"),
+                LlamaText.joinValues("\n\n", [
+                    system,
+                    user
+                ].filter((item) => item.values.length > 0)),
+                new SpecialTokensText("[/INST]"),
+                model,
+                this._addSpaceBeforeEos
+                    ? " "
+                    : "",
+                new SpecialToken("EOS")
+            ]);
+        }), toolsText === ""
+            ? ""
+            : [
+                new SpecialTokensText("[AVAILABLE_TOOLS]"),
+                toolsText,
+                new SpecialTokensText("[/AVAILABLE_TOOLS]")
+            ], chunkedLastInteraction.map(({ system, user, model }, index) => {
+            const isLastItem = index === chunkedLastInteraction.length - 1;
+            return LlamaText([
+                new SpecialTokensText("[INST]"),
+                (isLastItem && LlamaText(systemMessage).values.length > 0)
+                    ? [systemMessage, "\n\n"]
+                    : "",
+                LlamaText.joinValues("\n\n", [
+                    system,
+                    user
+                ].filter((item) => item.values.length > 0)),
+                new SpecialTokensText("[/INST]"),
+                model,
+                this._addSpaceBeforeEos
+                    ? " "
+                    : "",
+                isLastItem
+                    ? LlamaText([])
+                    : new SpecialToken("EOS")
+            ]);
+        }));
+        return {
+            contextText,
+            stopGenerationTriggers: [
+                LlamaText(new SpecialToken("EOS")),
+                LlamaText("</s>")
+            ]
+        };
+    }
+    generateInitialChatHistory({ systemPrompt } = {}) {
+        if (systemPrompt == null || systemPrompt.trim() === "")
+            return [];
+        return [{
+                type: "system",
+                text: LlamaText(systemPrompt).toJSON()
+            }];
+    }
+    /** @internal */
+    _generateAvailableToolsText({ availableFunctions, documentFunctionParams = true }) {
+        const availableFunctionNames = Object.keys(availableFunctions ?? {});
+        if (availableFunctions == null || availableFunctionNames.length === 0)
+            return "";
+        const availableTools = availableFunctionNames.map((functionName) => {
+            const functionDefinition = availableFunctions[functionName];
+            return {
+                type: "function",
+                function: {
+                    name: functionName,
+                    description: functionDefinition?.description != null && functionDefinition.description.trim() !== ""
+                        ? functionDefinition.description
+                        : undefined,
+                    parameters: documentFunctionParams && functionDefinition?.params != null
+                        ? functionDefinition.params
+                        : undefined
+                }
+            };
+        });
+        return jsonDumps(availableTools);
+    }
+    /** @internal */
+    _splitSystemMessageFromChatHistory(history) {
+        const systemMessages = [];
+        const newHistory = history.slice();
+        while (newHistory.length > 0 && newHistory[0].type === "system")
+            systemMessages.push(LlamaText.fromJSON(newHistory.shift().text));
+        return {
+            systemMessage: LlamaText.joinValues("\n\n", systemMessages),
+            chatHistory: newHistory
+        };
+    }
+    /** @internal */
+    _splitLastInteractionFromChatHistory(history) {
+        const lastInteraction = [];
+        const newHistory = history.slice();
+        while (newHistory.length > 0) {
+            const item = newHistory.pop();
+            lastInteraction.unshift(item);
+            if (item.type === "user")
+                break;
+        }
+        return {
+            lastInteraction,
+            chatHistory: newHistory
+        };
+    }
+    /** @internal */
+    static _getOptionConfigurationsToTestIfCanSupersedeJinjaTemplate() {
+        return [
+            { addSpaceBeforeEos: false },
+            { addSpaceBeforeEos: true }
+        ];
+    }
+}
+//# sourceMappingURL=MistralChatWrapper.js.map

package/dist/chatWrappers/MistralChatWrapper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MistralChatWrapper.js","sourceRoot":"","sources":["../../src/chatWrappers/MistralChatWrapper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,WAAW,EAAqC,MAAM,mBAAmB,CAAC;AAKlF,OAAO,EAAC,YAAY,EAAE,SAAS,EAAE,iBAAiB,EAAC,MAAM,uBAAuB,CAAC;AACjF,OAAO,EAAC,SAAS,EAAC,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAC,cAAc,EAAC,MAAM,2BAA2B,CAAC;AAEzD,UAAU;AACV,4IAA4I;AAC5I,EAAE;AACF,oEAAoE;AACpE,MAAM,OAAO,kBAAmB,SAAQ,WAAW;IAC/B,WAAW,GAAW,SAAS,CAAC;IAEvB,QAAQ,GAAwB;QACrD,sBAAsB,EAAE,IAAI;QAC5B,SAAS,EAAE;YACP,IAAI,EAAE;gBACF,mBAAmB,EAAE,IAAI;gBACzB,MAAM,EAAE,YAAY;gBACpB,YAAY,EAAE,kBAAkB;gBAChC,MAAM,EAAE,GAAG;aACd;YACD,MAAM,EAAE;gBACJ,MAAM,EAAE,0CAA0C;gBAClD,MAAM,EAAE,GAAG;aACd;YACD,WAAW,EAAE;gBACT,IAAI,EAAE;oBACF,aAAa,EAAE,SAAS,CAAC,IAAI,iBAAiB,CAAC,cAAc,CAAC,EAAE,GAAG,CAAC;oBACpE,YAAY,EAAE,IAAI;oBAClB,aAAa,EAAE,SAAS,CAAC,GAAG,EAAE,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;iBACzD;gBACD,MAAM,EAAE;oBACJ,aAAa,EAAE,SAAS,CAAC,IAAI,iBAAiB,CAAC,gBAAgB,CAAC,EAAE,GAAG,CAAC;oBACtE,cAAc,EAAE,IAAI;oBACpB,aAAa,EAAE,SAAS,CAAC,GAAG,EAAE,IAAI,iBAAiB,CAAC,iBAAiB,CAAC,CAAC;iBAC1E;aACJ;SACJ;KACJ,CAAC;IAEF,gBAAgB,CAAkB,kBAAkB,CAAU;IAE9D,YAAmB,EACf,iBAAiB,GAAG,KAAK,KAMzB,EAAE;QACF,KAAK,EAAE,CAAC;QAER,IAAI,CAAC,kBAAkB,GAAG,iBAAiB,CAAC;IAChD,CAAC;IAEe,2CAA2C,CAAC,OAAmC;QAC3F,OAAO,OAAO,CAAC;IACnB,CAAC;IAEe,oBAAoB,CAAC,EACjC,WAAW,EAAE,kBAAkB,EAAE,sBAAsB,EAClB;QACrC,MAAM,SAAS,GAAG,IAAI,CAAC,2BAA2B,CAAC,EAAC,kBAAkB,EAAE,sBAAsB,EAAC,CAAC,CAAC;QACjG,MAAM,EAAC,aAAa,EAAE,WAAW,EAAE,+BAA+B,EAAC,GAAG,IAAI,CAAC,kCAAkC,CAAC,WAAW,CAAC,CAAC;QAC3H,MAAM,EAAC,eAAe,EAAE,WAAW,EAAE,gBAAgB,EAAC,GAAG,IAAI,CAAC,oCAAoC,CAAC,+BAA+B,CAAC,CAAC;QAEpI,MAAM,kBAAkB,GAAG,cAAc,CAAC,gBAAgB,EAAE;YACxD,yBAAyB,EAAE,IAAI,CAAC,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC;SACvE,CAAC,CAAC;QACH,MAAM,sBAAsB,GAAG,cAAc,CAAC,eAAe,EAAE;YAC3D,yBAAyB,EAAE,IAAI,CAAC,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC;SACvE,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,SAAS,CACzB,IAAI,YAAY,CAAC,KAAK,CAAC,EACvB,kBAAkB,CAAC,GAAG,CAAC,CAAC,EAAC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAC,EAAE,EAAE;YAC7C,OAAO,SAAS,CAAC;gBACb,IAAI,iBAAiB,CAAC,QAAQ,CAAC;gBAC/B,SAAS,CAAC,UAAU,CAAC,MAAM,EACvB;oBACI,MAAM;oBACN,IAAI;iBACP,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAC7C;gBACD,IAAI,iBAAiB,CAAC,SAAS,CAAC;gBAChC,KAAK;gBACL,IAAI,CAAC,kBAAkB;oBACnB,CAAC,CAAC,GAAG;oBACL,CAAC,CAAC,EAAE;gBACR,IAAI,YAAY,CAAC,KAAK,CAAC;aAC1B,CAAC,CAAC;QACP,CAAC,CAAC,EACF,SAAS,KAAK,EAAE;YACZ,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC;gBACE,IAAI,iBAAiB,CAAC,mBAAmB,CAAC;gBAC1C,SAAS;gBACT,IAAI,iBAAiB,CAAC,oBAAoB,CAAC;aAC9C,EACL,sBAAsB,CAAC,GAAG,CAAC,CAAC,EAAC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAC,EAAE,KAAK,EAAE,EAAE;YACxD,MAAM,UAAU,GAAG,KAAK,KAAK,sBAAsB,CAAC,MAAM,GAAG,CAAC,CAAC;YAE/D,OAAO,SAAS,CAAC;gBACb,IAAI,iBAAiB,CAAC,QAAQ,CAAC;gBAC/B,CAAC,UAAU,IAAI,SAAS,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;oBACtD,CAAC,CAAC,CAAC,aAAa,EAAE,MAAM,CAAC;oBACzB,CAAC,CAAC,EAAE;gBACR,SAAS,CAAC,UAAU,CAAC,MAAM,EACvB;oBACI,MAAM;oBACN,IAAI;iBACP,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAC7C;gBACD,IAAI,iBAAiB,CAAC,SAAS,CAAC;gBAChC,KAAK;gBACL,IAAI,CAAC,kBAAkB;oBACnB,CAAC,CAAC,GAAG;oBACL,CAAC,CAAC,EAAE;gBACR,UAAU;oBACN,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;oBACf,CAAC,CAAC,IAAI,YAAY,CAAC,KAAK,CAAC;aAChC,CAAC,CAAC;QACP,CAAC,CAAC,CACL,CAAC;QAEF,OAAO;YACH,WAAW;YACX,sBAAsB,EAAE;gBACpB,SAAS,CAAC,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;gBAClC,SAAS,CAAC,MAAM,CAAC;aACpB;SACJ,CAAC;IACN,CAAC;IAEe,0BAA0B,CAAC,EACvC,YAAY,KAC8B,EAAE;QAC5C,IAAI,YAAY,IAAI,IAAI,IAAI,YAAY,CAAC,IAAI,EAAE,KAAK,EAAE;YAClD,OAAO,EAAE,CAAC;QAEd,OAAO,CAAC;gBACJ,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,SAAS,CAAC,YAAY,CAAC,CAAC,MAAM,EAAE;aACzC,CAAC,CAAC;IACP,CAAC;IAED,gBAAgB;IACR,2BAA2B,CAAC,EAChC,kBAAkB,EAClB,sBAAsB,GAAG,IAAI,EAIhC;QACG,MAAM,sBAAsB,GAAG,MAAM,CAAC,IAAI,CAAC,kBAAkB,IAAI,EAAE,CAAC,CAAC;QAErE,IAAI,kBAAkB,IAAI,IAAI,IAAI,sBAAsB,CAAC,MAAM,KAAK,CAAC;YACjE,OAAO,EAAE,CAAC;QAEd,MAAM,cAAc,GAAG,sBAAsB,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,EAAE;YAC/D,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,YAAY,CAAC,CAAC;YAE5D,OAAO;gBACH,IAAI,EAAE,UAAU;gBAChB,QAAQ,EAAE;oBACN,IAAI,EAAE,YAAY;oBAClB,WAAW,EAAE,kBAAkB,EAAE,WAAW,IAAI,IAAI,IAAI,kBAAkB,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE;wBAChG,CAAC,CAAC,kBAAkB,CAAC,WAAW;wBAChC,CAAC,CAAC,SAAS;oBACf,UAAU,EAAE,sBAAsB,IAAI,kBAAkB,EAAE,MAAM,IAAI,IAAI;wBACpE,CAAC,CAAC,kBAAkB,CAAC,MAAM;wBAC3B,CAAC,CAAC,SAAS;iBAClB;aACJ,CAAC;QACN,CAAC,CAAC,CAAC;QAEH,OAAO,SAAS,CAAC,cAAc,CAAC,CAAC;IACrC,CAAC;IAED,gBAAgB;IACR,kCAAkC,CAAC,OAAmC;QAC1E,MAAM,cAAc,GAAgB,EAAE,CAAC;QACvC,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;QAEnC,OAAO,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,CAAC,CAAC,CAAE,CAAC,IAAI,KAAK,QAAQ;YAC5D,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAE,UAAU,CAAC,KAAK,EAAyB,CAAC,IAAI,CAAC,CAAC,CAAC;QAE7F,OAAO;YACH,aAAa,EAAE,SAAS,CAAC,UAAU,CAAC,MAAM,EAAE,cAAc,CAAC;YAC3D,WAAW,EAAE,UAAU;SAC1B,CAAC;IACN,CAAC;IAED,gBAAgB;IACR,oCAAoC,CAAC,OAAmC;QAC5E,MAAM,eAAe,GAAsB,EAAE,CAAC;QAC9C,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;QAEnC,OAAO,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,EAAG,CAAC;YAC/B,eAAe,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAE9B,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM;gBACpB,MAAM;QACd,CAAC;QAED,OAAO;YACH,eAAe;YACf,WAAW,EAAE,UAAU;SAC1B,CAAC;IACN,CAAC;IAED,gBAAgB;IACT,MAAM,CAAU,yDAAyD;QAC5E,OAAO;YACH,EAAC,iBAAiB,EAAE,KAAK,EAAC;YAC1B,EAAC,iBAAiB,EAAE,IAAI,EAAC;SAC8B,CAAC;IAChE,CAAC;CACJ"}

package/dist/chatWrappers/generic/JinjaTemplateChatWrapper.d.ts

@@ -0,0 +1,100 @@
+import { ChatWrapperGenerateContextStateOptions, ChatWrapperGeneratedContextState, ChatWrapperSettings } from "../../types.js";
+import { ChatWrapper } from "../../ChatWrapper.js";
+import { ChatHistoryFunctionCallMessageTemplate } from "./utils/chatHistoryFunctionCallMessageTemplate.js";
+export type JinjaTemplateChatWrapperOptions = {
+    template: string;
+    /**
+     * Defaults to `"assistant"`.
+     */
+    modelRoleName?: string;
+    /**
+     * Defaults to `"user"`.
+     */
+    userRoleName?: string;
+    /**
+     * Defaults to `"system"`.
+     */
+    systemRoleName?: string;
+    /**
+     * Some Jinja templates may not support system messages, and in such cases,
+     * it'll be detected and system messages can be converted to user messages.
+     *
+     * You can specify the format of the converted user message.
+     * - **"auto"**: Convert system messages to user messages only if the template does not support system messages.
+     * - **`true`**: Always convert system messages to user messages.
+     * - **`false`**: Never convert system messages to user messages.
+     * May throw an error if some system messages don't appear in the template.
+     * - **`{use: "ifNeeded", format: "..."}`**: Convert system messages to user messages only if the template does not support system
+     * messages with the specified format.
+     * - **`{use: "always", format: "..."}`**: Always convert system messages to user messages with the specified format.
+     *
+     * Defaults to `"auto"`.
+     */
+    convertUnsupportedSystemMessagesToUserMessages?: "auto" | boolean | JinjaTemplateChatWrapperOptionsConvertMessageFormat;
+    functionCallMessageTemplate?: ChatHistoryFunctionCallMessageTemplate;
+    /**
+     * Whether to join adjacent messages of the same type.
+     * Some Jinja templates may throw an error if this is not set to `true`.
+     *
+     * Defaults to `true`.
+     */
+    joinAdjacentMessagesOfTheSameType?: boolean;
+    /**
+     * Whether to trim leading whitespace in responses.
+     *
+     * Defaults to `true`.
+     */
+    trimLeadingWhitespaceInResponses?: boolean;
+    /**
+     * Additional parameters to use for rendering the Jinja template.
+     */
+    additionalRenderParameters?: Record<string, any>;
+};
+export type JinjaTemplateChatWrapperOptionsConvertMessageFormat = {
+    use?: "always" | "ifNeeded";
+    format: `${string}{{message}}${string}`;
+};
+/**
+ * A chat wrapper based on a Jinja template.
+ * Useful for using the original model's Jinja template as-is without any additional conversion work to chat with a model.
+ *
+ * If you want to create a new chat wrapper from scratch, using this chat wrapper is not recommended, and instead you better inherit
+ * from the `ChatWrapper` class and implement a custom chat wrapper of your own in TypeScript.
+ *
+ * For a simpler way to create a chat wrapper, see the `TemplateChatWrapper` class.
+ * @example
+ * <span v-pre>
+ *
+ * ```ts
+ * import {JinjaTemplateChatWrapper} from "node-llama-cpp";
+ *
+ * const chatWrapper = new JinjaTemplateChatWrapper({
+ *     template: "<Jinja template here>",
+ *     // functionCallMessageTemplate: { // optional
+ *     //     call: "[[call: {{functionName}}({{functionParams}})]]",
+ *     //     result: " [[result: {{functionCallResult}}]]"
+ *     // }
+ * });
+ * ```
+ *
+ * </span>
+ */
+export declare class JinjaTemplateChatWrapper extends ChatWrapper {
+    readonly wrapperName = "JinjaTemplate";
+    readonly settings: ChatWrapperSettings;
+    readonly template: string;
+    readonly modelRoleName: string;
+    readonly userRoleName: string;
+    readonly systemRoleName: string;
+    readonly convertUnsupportedSystemMessagesToUserMessages?: JinjaTemplateChatWrapperOptionsConvertMessageFormat;
+    readonly joinAdjacentMessagesOfTheSameType: boolean;
+    readonly trimLeadingWhitespaceInResponses: boolean;
+    readonly additionalRenderParameters?: Record<string, any>;
+    /**
+     * @param options
+     */
+    constructor({ template, modelRoleName, userRoleName, systemRoleName, convertUnsupportedSystemMessagesToUserMessages, functionCallMessageTemplate, joinAdjacentMessagesOfTheSameType, trimLeadingWhitespaceInResponses, additionalRenderParameters }: JinjaTemplateChatWrapperOptions);
+    generateContextState({ chatHistory, availableFunctions, documentFunctionParams }: ChatWrapperGenerateContextStateOptions): ChatWrapperGeneratedContextState & {
+        transformedSystemMessagesToUserMessages: boolean;
+    };
+}