node-llama-cpp 3.0.0-beta.13 → 3.0.0-beta.15

package/dist/chatWrappers/utils/resolveChatWrapper.js

@@ -0,0 +1,206 @@
+import { parseModelFileName } from "../../utils/parseModelFileName.js";
+import { LlamaChatWrapper } from "../LlamaChatWrapper.js";
+import { ChatMLChatWrapper } from "../ChatMLChatWrapper.js";
+import { GeneralChatWrapper } from "../GeneralChatWrapper.js";
+import { FalconChatWrapper } from "../FalconChatWrapper.js";
+import { FunctionaryChatWrapper } from "../FunctionaryChatWrapper.js";
+import { AlpacaChatWrapper } from "../AlpacaChatWrapper.js";
+import { GemmaChatWrapper } from "../GemmaChatWrapper.js";
+import { JinjaTemplateChatWrapper } from "../generic/JinjaTemplateChatWrapper.js";
+import { TemplateChatWrapper } from "../generic/TemplateChatWrapper.js";
+import { getConsoleLogPrefix } from "../../utils/getConsoleLogPrefix.js";
+import { isJinjaTemplateEquivalentToSpecializedChatWrapper } from "./isJinjaTemplateEquivalentToSpecializedChatWrapper.js";
+export const specializedChatWrapperTypeNames = Object.freeze([
+    "general", "llamaChat", "alpacaChat", "functionary", "chatML", "falconChat", "gemma"
+]);
+export const templateChatWrapperTypeNames = Object.freeze([
+    "template", "jinjaTemplate"
+]);
+export const resolvableChatWrapperTypeNames = Object.freeze([
+    "auto",
+    ...specializedChatWrapperTypeNames,
+    ...templateChatWrapperTypeNames
+]);
+const chatWrappers = {
+    "general": GeneralChatWrapper,
+    "llamaChat": LlamaChatWrapper,
+    "alpacaChat": AlpacaChatWrapper,
+    "functionary": FunctionaryChatWrapper,
+    "chatML": ChatMLChatWrapper,
+    "falconChat": FalconChatWrapper,
+    "gemma": GemmaChatWrapper,
+    "template": TemplateChatWrapper,
+    "jinjaTemplate": JinjaTemplateChatWrapper
+};
+const chatWrapperToConfigType = new Map(Object.entries(chatWrappers)
+    .map(([configType, Wrapper]) => ([Wrapper, configType])));
+/**
+ * Resolve to a chat wrapper instance based on the provided information.
+ * The more information provided, the better the resolution will be (except for `type`).
+ *
+ * It's recommended to not set `type` to a specific chat wrapper in order for the resolution to be more flexible, but it is useful for when
+ * you need to provide the ability to force a specific chat wrapper type.
+ * Note that when setting `type` to a generic chat wrapper type (such as `"template"` or `"jinjaTemplate"`), the `customWrapperSettings`
+ * must contain the necessary settings for that chat wrapper to be created.
+ *
+ * When loading a Jinja chat template from either `fileInfo` or `customWrapperSettings.jinjaTemplate.template`,
+ * if the chat template format is invalid, it fallbacks to resolve other chat wrappers,
+ * unless `fallbackToOtherWrappersOnJinjaError` is set to `false` (in which case, it will throw an error).
+ */
+export function resolveChatWrapper({ type = "auto", bosString, filename, fileInfo, tokenizer, customWrapperSettings, warningLogs = true, fallbackToOtherWrappersOnJinjaError = true, noJinja = false }) {
+    function createSpecializedChatWrapper(specializedChatWrapper, defaultSettings = {}) {
+        const chatWrapperConfigType = chatWrapperToConfigType.get(specializedChatWrapper);
+        const chatWrapperSettings = customWrapperSettings?.[chatWrapperConfigType];
+        return new specializedChatWrapper({
+            ...(defaultSettings ?? {}),
+            ...(chatWrapperSettings ?? {})
+        });
+    }
+    if (type !== "auto" && type != null) {
+        if (isTemplateChatWrapperType(type)) {
+            const Wrapper = chatWrappers[type];
+            if (isClassReference(Wrapper, TemplateChatWrapper)) {
+                const wrapperSettings = customWrapperSettings?.template;
+                if (wrapperSettings == null || wrapperSettings?.template == null || wrapperSettings?.historyTemplate == null ||
+                    wrapperSettings?.modelRoleName == null || wrapperSettings?.userRoleName == null) {
+                    if (warningLogs)
+                        console.warn(getConsoleLogPrefix() + "Template chat wrapper settings must have a template, historyTemplate, modelRoleName, and userRoleName. Falling back to resolve other chat wrapper types.");
+                }
+                else
+                    return new TemplateChatWrapper(wrapperSettings);
+            }
+            else if (isClassReference(Wrapper, JinjaTemplateChatWrapper)) {
+                const jinjaTemplate = customWrapperSettings?.jinjaTemplate?.template ?? fileInfo?.metadata?.tokenizer?.chat_template;
+                if (jinjaTemplate == null) {
+                    if (warningLogs)
+                        console.warn(getConsoleLogPrefix() + "Jinja template chat wrapper received no template. Falling back to resolve other chat wrapper types.");
+                }
+                else {
+                    try {
+                        return new JinjaTemplateChatWrapper({
+                            ...(customWrapperSettings?.jinjaTemplate ?? {}),
+                            template: jinjaTemplate
+                        });
+                    }
+                    catch (err) {
+                        if (!fallbackToOtherWrappersOnJinjaError)
+                            throw err;
+                        else if (warningLogs)
+                            console.error(getConsoleLogPrefix() + "Error creating Jinja template chat wrapper. Falling back to resolve other chat wrappers. Error:", err);
+                    }
+                }
+            }
+            else
+                void (Wrapper);
+        }
+        else if (Object.hasOwn(chatWrappers, type)) {
+            const Wrapper = chatWrappers[type];
+            const wrapperSettings = customWrapperSettings?.[type];
+            return new Wrapper(wrapperSettings);
+        }
+    }
+    const modelJinjaTemplate = customWrapperSettings?.jinjaTemplate?.template ?? fileInfo?.metadata?.tokenizer?.chat_template;
+    if (!noJinja && modelJinjaTemplate != null && modelJinjaTemplate.trim() !== "") {
+        const jinjaTemplateChatWrapperOptions = {
+            ...(customWrapperSettings?.jinjaTemplate ?? {}),
+            template: modelJinjaTemplate
+        };
+        for (const specializedChatWrapperTypeName of specializedChatWrapperTypeNames) {
+            const Wrapper = chatWrappers[specializedChatWrapperTypeName];
+            const wrapperSettings = customWrapperSettings?.[specializedChatWrapperTypeName];
+            const testOptionConfigurations = Wrapper._getOptionConfigurationsToTestIfCanSupersedeJinjaTemplate?.() ?? [];
+            if (testOptionConfigurations.length === 0)
+                testOptionConfigurations.push({});
+            for (const testConfiguration of testOptionConfigurations) {
+                const testChatWrapperSettings = {
+                    ...(wrapperSettings ?? {}),
+                    ...(testConfiguration ?? {})
+                };
+                const chatWrapper = new Wrapper(testChatWrapperSettings);
+                if (isJinjaTemplateEquivalentToSpecializedChatWrapper(jinjaTemplateChatWrapperOptions, chatWrapper, tokenizer))
+                    return new Wrapper(testChatWrapperSettings);
+            }
+        }
+        if (!fallbackToOtherWrappersOnJinjaError)
+            return new JinjaTemplateChatWrapper(jinjaTemplateChatWrapperOptions);
+        try {
+            return new JinjaTemplateChatWrapper(jinjaTemplateChatWrapperOptions);
+        }
+        catch (err) {
+            console.error(getConsoleLogPrefix() + "Error creating Jinja template chat wrapper. Falling back to resolve other chat wrappers. Error:", err);
+        }
+    }
+    // try to find a pattern in the Jinja template to resolve to a specialized chat wrapper,
+    // with a logic similar to `llama.cpp`'s `llama_chat_apply_template_internal` function
+    if (modelJinjaTemplate != null && modelJinjaTemplate.trim() !== "") {
+        if (modelJinjaTemplate.includes("<|im_start|>"))
+            return createSpecializedChatWrapper(ChatMLChatWrapper);
+        else if (modelJinjaTemplate.includes("[INST]"))
+            return createSpecializedChatWrapper(LlamaChatWrapper, {
+                addSpaceBeforeEos: modelJinjaTemplate.includes("' ' + eos_token")
+            });
+        else if (modelJinjaTemplate.includes("<start_of_turn>"))
+            return createSpecializedChatWrapper(GemmaChatWrapper);
+    }
+    if (filename != null) {
+        const { name, subType, fileType, otherInfo } = parseModelFileName(filename);
+        if (fileType?.toLowerCase() === "gguf") {
+            const lowercaseName = name?.toLowerCase();
+            const lowercaseSubType = subType?.toLowerCase();
+            const splitLowercaseSubType = (lowercaseSubType?.split("-") ?? []).concat(otherInfo.map(info => info.toLowerCase()));
+            const firstSplitLowercaseSubType = splitLowercaseSubType[0];
+            if (lowercaseName === "llama") {
+                if (splitLowercaseSubType.includes("chat"))
+                    return createSpecializedChatWrapper(LlamaChatWrapper);
+                return createSpecializedChatWrapper(GeneralChatWrapper);
+            }
+            else if (lowercaseName === "yarn" && firstSplitLowercaseSubType === "llama")
+                return createSpecializedChatWrapper(LlamaChatWrapper);
+            else if (lowercaseName === "orca")
+                return createSpecializedChatWrapper(ChatMLChatWrapper);
+            else if (lowercaseName === "phind" && lowercaseSubType === "codellama")
+                return createSpecializedChatWrapper(LlamaChatWrapper);
+            else if (lowercaseName === "mistral")
+                return createSpecializedChatWrapper(GeneralChatWrapper);
+            else if (firstSplitLowercaseSubType === "llama")
+                return createSpecializedChatWrapper(LlamaChatWrapper);
+            else if (lowercaseSubType === "alpaca")
+                return createSpecializedChatWrapper(AlpacaChatWrapper);
+            else if (lowercaseName === "functionary")
+                return createSpecializedChatWrapper(FunctionaryChatWrapper);
+            else if (lowercaseName === "dolphin" && splitLowercaseSubType.includes("mistral"))
+                return createSpecializedChatWrapper(ChatMLChatWrapper);
+            else if (lowercaseName === "gemma")
+                return createSpecializedChatWrapper(GemmaChatWrapper);
+            else if (splitLowercaseSubType.includes("chatml"))
+                return createSpecializedChatWrapper(ChatMLChatWrapper);
+        }
+    }
+    if (fileInfo != null) {
+        const arch = fileInfo.metadata.general?.architecture;
+        if (arch === "llama")
+            return createSpecializedChatWrapper(LlamaChatWrapper);
+        else if (arch === "falcon")
+            return createSpecializedChatWrapper(FalconChatWrapper);
+    }
+    if (bosString === "" || bosString == null)
+        return null;
+    if ("<s>[INST] <<SYS>>\n".startsWith(bosString)) {
+        return createSpecializedChatWrapper(LlamaChatWrapper);
+    }
+    else if ("<|im_start|>system\n".startsWith(bosString)) {
+        return createSpecializedChatWrapper(ChatMLChatWrapper);
+    }
+    return null;
+}
+export function isSpecializedChatWrapperType(type) {
+    return specializedChatWrapperTypeNames.includes(type);
+}
+export function isTemplateChatWrapperType(type) {
+    return templateChatWrapperTypeNames.includes(type);
+}
+// this is needed because TypeScript guards don't work automatically with class references
+function isClassReference(value, classReference) {
+    return value === classReference;
+}
+//# sourceMappingURL=resolveChatWrapper.js.map

package/dist/chatWrappers/utils/resolveChatWrapper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolveChatWrapper.js","sourceRoot":"","sources":["../../../src/chatWrappers/utils/resolveChatWrapper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,kBAAkB,EAAC,MAAM,mCAAmC,CAAC;AACrE,OAAO,EAAC,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAC,iBAAiB,EAAC,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAC,kBAAkB,EAAC,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EAAC,iBAAiB,EAAC,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAC,sBAAsB,EAAC,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAC,iBAAiB,EAAC,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAC,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAC,wBAAwB,EAAkC,MAAM,wCAAwC,CAAC;AACjH,OAAO,EAAC,mBAAmB,EAAC,MAAM,mCAAmC,CAAC;AACtE,OAAO,EAAC,mBAAmB,EAAC,MAAM,oCAAoC,CAAC;AAEvE,OAAO,EAAC,iDAAiD,EAAC,MAAM,wDAAwD,CAAC;AAIzH,MAAM,CAAC,MAAM,+BAA+B,GAAG,MAAM,CAAC,MAAM,CAAC;IACzD,SAAS,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO;CAC9E,CAAC,CAAC;AAGZ,MAAM,CAAC,MAAM,4BAA4B,GAAG,MAAM,CAAC,MAAM,CAAC;IACtD,UAAU,EAAE,eAAe;CACrB,CAAC,CAAC;AAGZ,MAAM,CAAC,MAAM,8BAA8B,GAAG,MAAM,CAAC,MAAM,CAAC;IACxD,MAAM;IACN,GAAG,+BAA+B;IAClC,GAAG,4BAA4B;CACzB,CAAC,CAAC;AAGZ,MAAM,YAAY,GAAG;IACjB,SAAS,EAAE,kBAAkB;IAC7B,WAAW,EAAE,gBAAgB;IAC7B,YAAY,EAAE,iBAAiB;IAC/B,aAAa,EAAE,sBAAsB;IACrC,QAAQ,EAAE,iBAAiB;IAC3B,YAAY,EAAE,iBAAiB;IAC/B,OAAO,EAAE,gBAAgB;IACzB,UAAU,EAAE,mBAAmB;IAC/B,eAAe,EAAE,wBAAwB;CACiD,CAAC;AAC/F,MAAM,uBAAuB,GAAG,IAAI,GAAG,CACnC,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC;KACvB,GAAG,CAAC,CAAC,CAAC,UAAU,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAC5B,CAAC,OAAO,EAAE,UAAuC,CAAC,CACrD,CAAC,CACT,CAAC;AA2BF;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,kBAAkB,CAAC,EAC/B,IAAI,GAAG,MAAM,EACb,SAAS,EACT,QAAQ,EACR,QAAQ,EACR,SAAS,EACT,qBAAqB,EACrB,WAAW,GAAG,IAAI,EAClB,mCAAmC,GAAG,IAAI,EAC1C,OAAO,GAAG,KAAK,EACS;IACxB,SAAS,4BAA4B,CACjC,sBAAyB,EACzB,kBAA+C,EAAE;QAEjD,MAAM,qBAAqB,GAAG,uBAAuB,CAAC,GAAG,CAAC,sBAAsB,CAAmC,CAAC;QACpH,MAAM,mBAAmB,GAAG,qBAAqB,EAAE,CAAC,qBAAqB,CAAC,CAAC;QAE3E,OAAO,IAAK,sBAA8B,CAAC;YACvC,GAAG,CAAC,eAAe,IAAI,EAAE,CAAC;YAC1B,GAAG,CAAC,mBAAmB,IAAI,EAAE,CAAC;SACjC,CAAC,CAAC;IACP,CAAC;IAED,IAAI,IAAI,KAAK,MAAM,IAAI,IAAI,IAAI,IAAI,EAAE;QACjC,IAAI,yBAAyB,CAAC,IAAI,CAAC,EAAE;YACjC,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;YAEnC,IAAI,gBAAgB,CAAC,OAAO,EAAE,mBAAmB,CAAC,EAAE;gBAChD,MAAM,eAAe,GAAG,qBAAqB,EAAE,QAAQ,CAAC;gBACxD,IAAI,eAAe,IAAI,IAAI,IAAI,eAAe,EAAE,QAAQ,IAAI,IAAI,IAAI,eAAe,EAAE,eAAe,IAAI,IAAI;oBACxG,eAAe,EAAE,aAAa,IAAI,IAAI,IAAI,eAAe,EAAE,YAAY,IAAI,IAAI,EACjF;oBACE,IAAI,WAAW;wBACX,OAAO,CAAC,IAAI,CAAC,mBAAmB,EAAE,GAAG,0JAA0J,CAAC,CAAC;iBACxM;;oBACG,OAAO,IAAI,mBAAmB,CAAC,eAAe,CAAC,CAAC;aACvD;iBAAM,IAAI,gBAAgB,CAAC,OAAO,EAAE,wBAAwB,CAAC,EAAE;gBAC5D,MAAM,aAAa,GAAG,qBAAqB,EAAE,aAAa,EAAE,QAAQ,IAAI,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,aAAa,CAAC;gBAErH,IAAI,aAAa,IAAI,IAAI,EAAE;oBACvB,IAAI,WAAW;wBACX,OAAO,CAAC,IAAI,CAAC,mBAAmB,EAAE,GAAG,qGAAqG,CAAC,CAAC;iBACnJ;qBAAM;oBACH,IAAI;wBACA,OAAO,IAAI,wBAAwB,CAAC;4BAChC,GAAG,CAAC,qBAAqB,EAAE,aAAa,IAAI,EAAE,CAAC;4BAC/C,QAAQ,EAAE,aAAa;yBAC1B,CAAC,CAAC;qBACN;oBAAC,OAAO,GAAG,EAAE;wBACV,IAAI,CAAC,mCAAmC;4BACpC,MAAM,GAAG,CAAC;6BACT,IAAI,WAAW;4BAChB,OAAO,CAAC,KAAK,CAAC,mBAAmB,EAAE,GAAG,iGAAiG,EAAE,GAAG,CAAC,CAAC;qBACrJ;iBACJ;aACJ;;gBACG,KAAK,CAAC,OAAuB,CAAC,CAAC;SACtC;aAAM,IAAI,MAAM,CAAC,MAAM,CAAC,YAAY,EAAE,IAAI,CAAC,EAAE;YAC1C,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;YACnC,MAAM,eAAe,GACjB,qBAAqB,EAAE,CAAC,IAAI,CAAC,CAAC;YAElC,OAAO,IAAK,OAAe,CAAC,eAAe,CAAC,CAAC;SAChD;KACJ;IAED,MAAM,kBAAkB,GAAG,qBAAqB,EAAE,aAAa,EAAE,QAAQ,IAAI,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,aAAa,CAAC;IAE1H,IAAI,CAAC,OAAO,IAAI,kBAAkB,IAAI,IAAI,IAAI,kBAAkB,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC5E,MAAM,+BAA+B,GAAoC;YACrE,GAAG,CAAC,qBAAqB,EAAE,aAAa,IAAI,EAAE,CAAC;YAC/C,QAAQ,EAAE,kBAAkB;SAC/B,CAAC;QAEF,KAAK,MAAM,8BAA8B,IAAI,+BAA+B,EAAE;YAC1E,MAAM,OAAO,GAAG,YAAY,CAAC,8BAA8B,CAAC,CAAC;YAC7D,MAAM,eAAe,GAAG,qBAAqB,EAAE,CAAC,8BAA8B,CAAC,CAAC;YAEhF,MAAM,wBAAwB,GAAG,OAAO,CAAC,yDAAyD,EAAE,EAAE,IAAI,EAAE,CAAC;YAC7G,IAAI,wBAAwB,CAAC,MAAM,KAAK,CAAC;gBACrC,wBAAwB,CAAC,IAAI,CAAC,EAAS,CAAC,CAAC;YAE7C,KAAK,MAAM,iBAAiB,IAAI,wBAAwB,EAAE;gBACtD,MAAM,uBAAuB,GAAG;oBAC5B,GAAG,CAAC,eAAe,IAAI,EAAE,CAAC;oBAC1B,GAAG,CAAC,iBAAiB,IAAI,EAAE,CAAC;iBAC/B,CAAC;gBACF,MAAM,WAAW,GAAG,IAAK,OAAe,CAAC,uBAAuB,CAAC,CAAC;gBAElE,IAAI,iDAAiD,CAAC,+BAA+B,EAAE,WAAW,EAAE,SAAS,CAAC;oBAC1G,OAAO,IAAK,OAAe,CAAC,uBAAuB,CAAC,CAAC;aAC5D;SACJ;QAED,IAAI,CAAC,mCAAmC;YACpC,OAAO,IAAI,wBAAwB,CAAC,+BAA+B,CAAC,CAAC;QAEzE,IAAI;YACA,OAAO,IAAI,wBAAwB,CAAC,+BAA+B,CAAC,CAAC;SACxE;QAAC,OAAO,GAAG,EAAE;YACV,OAAO,CAAC,KAAK,CAAC,mBAAmB,EAAE,GAAG,iGAAiG,EAAE,GAAG,CAAC,CAAC;SACjJ;KACJ;IAED,wFAAwF;IACxF,sFAAsF;IACtF,IAAI,kBAAkB,IAAI,IAAI,IAAI,kBAAkB,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAChE,IAAI,kBAAkB,CAAC,QAAQ,CAAC,cAAc,CAAC;YAC3C,OAAO,4BAA4B,CAAC,iBAAiB,CAAC,CAAC;aACtD,IAAI,kBAAkB,CAAC,QAAQ,CAAC,QAAQ,CAAC;YAC1C,OAAO,4BAA4B,CAAC,gBAAgB,EAAE;gBAClD,iBAAiB,EAAE,kBAAkB,CAAC,QAAQ,CAAC,iBAAiB,CAAC;aACpE,CAAC,CAAC;aACF,IAAI,kBAAkB,CAAC,QAAQ,CAAC,iBAAiB,CAAC;YACnD,OAAO,4BAA4B,CAAC,gBAAgB,CAAC,CAAC;KAC7D;IAED,IAAI,QAAQ,IAAI,IAAI,EAAE;QAClB,MAAM,EAAC,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAC,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QAE1E,IAAI,QAAQ,EAAE,WAAW,EAAE,KAAK,MAAM,EAAE;YACpC,MAAM,aAAa,GAAG,IAAI,EAAE,WAAW,EAAE,CAAC;YAC1C,MAAM,gBAAgB,GAAG,OAAO,EAAE,WAAW,EAAE,CAAC;YAChD,MAAM,qBAAqB,GAAG,CAAC,gBAAgB,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CACrE,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAC5C,CAAC;YACF,MAAM,0BAA0B,GAAG,qBAAqB,CAAC,CAAC,CAAC,CAAC;YAE5D,IAAI,aAAa,KAAK,OAAO,EAAE;gBAC3B,IAAI,qBAAqB,CAAC,QAAQ,CAAC,MAAM,CAAC;oBACtC,OAAO,4BAA4B,CAAC,gBAAgB,CAAC,CAAC;gBAE1D,OAAO,4BAA4B,CAAC,kBAAkB,CAAC,CAAC;aAC3D;iBAAM,IAAI,aAAa,KAAK,MAAM,IAAI,0BAA0B,KAAK,OAAO;gBACzE,OAAO,4BAA4B,CAAC,gBAAgB,CAAC,CAAC;iBACrD,IAAI,aAAa,KAAK,MAAM;gBAC7B,OAAO,4BAA4B,CAAC,iBAAiB,CAAC,CAAC;iBACtD,IAAI,aAAa,KAAK,OAAO,IAAI,gBAAgB,KAAK,WAAW;gBAClE,OAAO,4BAA4B,CAAC,gBAAgB,CAAC,CAAC;iBACrD,IAAI,aAAa,KAAK,SAAS;gBAChC,OAAO,4BAA4B,CAAC,kBAAkB,CAAC,CAAC;iBACvD,IAAI,0BAA0B,KAAK,OAAO;gBAC3C,OAAO,4BAA4B,CAAC,gBAAgB,CAAC,CAAC;iBACrD,IAAI,gBAAgB,KAAK,QAAQ;gBAClC,OAAO,4BAA4B,CAAC,iBAAiB,CAAC,CAAC;iBACtD,IAAI,aAAa,KAAK,aAAa;gBACpC,OAAO,4BAA4B,CAAC,sBAAsB,CAAC,CAAC;iBAC3D,IAAI,aAAa,KAAK,SAAS,IAAI,qBAAqB,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAC7E,OAAO,4BAA4B,CAAC,iBAAiB,CAAC,CAAC;iBACtD,IAAI,aAAa,KAAK,OAAO;gBAC9B,OAAO,4BAA4B,CAAC,gBAAgB,CAAC,CAAC;iBACrD,IAAI,qBAAqB,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAC7C,OAAO,4BAA4B,CAAC,iBAAiB,CAAC,CAAC;SAC9D;KACJ;IAED,IAAI,QAAQ,IAAI,IAAI,EAAE;QAClB,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;QAErD,IAAI,IAAI,KAAK,OAAO;YAChB,OAAO,4BAA4B,CAAC,gBAAgB,CAAC,CAAC;aACrD,IAAI,IAAI,KAAK,QAAQ;YACtB,OAAO,4BAA4B,CAAC,iBAAiB,CAAC,CAAC;KAC9D;IAED,IAAI,SAAS,KAAK,EAAE,IAAI,SAAS,IAAI,IAAI;QACrC,OAAO,IAAI,CAAC;IAEhB,IAAI,qBAAqB,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;QAC7C,OAAO,4BAA4B,CAAC,gBAAgB,CAAC,CAAC;KACzD;SAAM,IAAI,sBAAsB,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;QACrD,OAAO,4BAA4B,CAAC,iBAAiB,CAAC,CAAC;KAC1D;IAED,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,4BAA4B,CAAC,IAAY;IACrD,OAAO,+BAA+B,CAAC,QAAQ,CAAC,IAAW,CAAC,CAAC;AACjE,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,IAAY;IAClD,OAAO,4BAA4B,CAAC,QAAQ,CAAC,IAAW,CAAC,CAAC;AAC9D,CAAC;AAED,0FAA0F;AAC1F,SAAS,gBAAgB,CAAI,KAAU,EAAE,cAAiB;IACtD,OAAO,KAAK,KAAK,cAAc,CAAC;AACpC,CAAC"}

package/dist/cli/cli.js

@@ -14,7 +14,7 @@ import { ClearCommand } from "./commands/ClearCommand.js";
 import { ChatCommand } from "./commands/ChatCommand.js";
 import { CompleteCommand } from "./commands/CompleteCommand.js";
 import { InfillCommand } from "./commands/InfillCommand.js";
-import { InspectCommand } from "./commands/InspectCommand.js";
+import { InspectCommand } from "./commands/inspect/InspectCommand.js";
 import { DebugCommand } from "./commands/DebugCommand.js";
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const packageJson = fs.readJSONSync(path.join(__dirname, "..", "..", "package.json"));

package/dist/cli/cli.js.map

@@ -1 +1 @@
-{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/cli/cli.ts"],"names":[],"mappings":";AAEA,OAAO,EAAC,aAAa,EAAC,MAAM,KAAK,CAAC;AAClC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,sDAAsD;AACtD,OAAO,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,UAAU,EAAC,MAAM,cAAc,CAAC;AACxC,OAAO,EAAC,mBAAmB,EAAC,MAAM,aAAa,CAAC;AAChD,OAAO,EAAC,eAAe,EAAC,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAC,YAAY,EAAC,MAAM,4BAA4B,CAAC;AACxD,OAAO,EAAC,oBAAoB,EAAC,MAAM,oCAAoC,CAAC;AACxE,OAAO,EAAC,YAAY,EAAC,MAAM,4BAA4B,CAAC;AACxD,OAAO,EAAC,WAAW,EAAC,MAAM,2BAA2B,CAAC;AACtD,OAAO,EAAC,eAAe,EAAC,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAC,aAAa,EAAC,MAAM,6BAA6B,CAAC;AAC1D,OAAO,EAAC,cAAc,EAAC,MAAM,8BAA8B,CAAC;AAC5D,OAAO,EAAC,YAAY,EAAC,MAAM,4BAA4B,CAAC;AAExD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE/D,MAAM,WAAW,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC;AAEtF,mBAAmB,CAAC,IAAI,CAAC,CAAC;AAE1B,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;AAE1C,IAAI;KACC,UAAU,CAAC,UAAU,CAAC;KACtB,KAAK,CAAC,+BAA+B,CAAC;KACtC,OAAO,CAAC,eAAe,CAAC;KACxB,OAAO,CAAC,YAAY,CAAC;KACrB,OAAO,CAAC,YAAY,CAAC;KACrB,OAAO,CAAC,WAAW,CAAC;KACpB,OAAO,CAAC,eAAe,CAAC;KACxB,OAAO,CAAC,aAAa,CAAC;KACtB,OAAO,CAAC,cAAc,CAAC;KACvB,OAAO,CAAC,oBAAoB,CAAC;KAC7B,OAAO,CAAC,YAAY,CAAC;KACrB,iBAAiB,EAAE;KACnB,aAAa,CAAC,CAAC,CAAC;KAChB,MAAM,EAAE;KACR,cAAc,EAAE;KAChB,KAAK,CAAC,GAAG,EAAE,SAAS,CAAC;KACrB,IAAI,CAAC,GAAG,CAAC;KACT,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC;KAClB,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC;KAC5B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;KACzC,KAAK,EAAE,CAAC"}
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/cli/cli.ts"],"names":[],"mappings":";AAEA,OAAO,EAAC,aAAa,EAAC,MAAM,KAAK,CAAC;AAClC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,sDAAsD;AACtD,OAAO,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,UAAU,EAAC,MAAM,cAAc,CAAC;AACxC,OAAO,EAAC,mBAAmB,EAAC,MAAM,aAAa,CAAC;AAChD,OAAO,EAAC,eAAe,EAAC,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAC,YAAY,EAAC,MAAM,4BAA4B,CAAC;AACxD,OAAO,EAAC,oBAAoB,EAAC,MAAM,oCAAoC,CAAC;AACxE,OAAO,EAAC,YAAY,EAAC,MAAM,4BAA4B,CAAC;AACxD,OAAO,EAAC,WAAW,EAAC,MAAM,2BAA2B,CAAC;AACtD,OAAO,EAAC,eAAe,EAAC,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAC,aAAa,EAAC,MAAM,6BAA6B,CAAC;AAC1D,OAAO,EAAC,cAAc,EAAC,MAAM,sCAAsC,CAAC;AACpE,OAAO,EAAC,YAAY,EAAC,MAAM,4BAA4B,CAAC;AAExD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE/D,MAAM,WAAW,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC;AAEtF,mBAAmB,CAAC,IAAI,CAAC,CAAC;AAE1B,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;AAE1C,IAAI;KACC,UAAU,CAAC,UAAU,CAAC;KACtB,KAAK,CAAC,+BAA+B,CAAC;KACtC,OAAO,CAAC,eAAe,CAAC;KACxB,OAAO,CAAC,YAAY,CAAC;KACrB,OAAO,CAAC,YAAY,CAAC;KACrB,OAAO,CAAC,WAAW,CAAC;KACpB,OAAO,CAAC,eAAe,CAAC;KACxB,OAAO,CAAC,aAAa,CAAC;KACtB,OAAO,CAAC,cAAc,CAAC;KACvB,OAAO,CAAC,oBAAoB,CAAC;KAC7B,OAAO,CAAC,YAAY,CAAC;KACrB,iBAAiB,EAAE;KACnB,aAAa,CAAC,CAAC,CAAC;KAChB,MAAM,EAAE;KACR,cAAc,EAAE;KAChB,KAAK,CAAC,GAAG,EAAE,SAAS,CAAC;KACrB,IAAI,CAAC,GAAG,CAAC;KACT,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC;KAClB,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC;KAC5B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;KACzC,KAAK,EAAE,CAAC"}

package/dist/cli/commands/ChatCommand.d.ts

@@ -1,6 +1,6 @@
 import { CommandModule } from "yargs";
 import { LlamaGrammar } from "../../evaluator/LlamaGrammar.js";
-import { ChatWrapperTypeName } from "../../bindings/utils/resolveChatWrapperBasedOnWrapperTypeName.js";
+import { SpecializedChatWrapperTypeName } from "../../chatWrappers/utils/resolveChatWrapper.js";
 type ChatCommand = {
     model: string;
     systemInfo: boolean;
@@ -8,9 +8,11 @@ type ChatCommand = {
     systemPromptFile?: string;
     prompt?: string;
     promptFile?: string;
-    wrapper: ChatWrapperTypeName;
-    contextSize: number;
+    wrapper: SpecializedChatWrapperTypeName | "auto";
+    noJinja?: boolean;
+    contextSize?: number;
     batchSize?: number;
+    noTrimWhitespace: boolean;
     grammar: "text" | Parameters<typeof LlamaGrammar.getFor>[1];
     jsonSchemaGrammarFile?: string;
     threads: number;
@@ -27,7 +29,8 @@ type ChatCommand = {
     maxTokens: number;
     noHistory: boolean;
     environmentFunctions: boolean;
-    noInfoLog: boolean;
+    debug: boolean;
+    meter: boolean;
     printTimings: boolean;
 };
 export declare const ChatCommand: CommandModule<object, ChatCommand>;

package/dist/cli/commands/ChatCommand.js

@@ -6,24 +6,27 @@ import fs from "fs-extra";
 import { chatCommandHistoryFilePath, defaultChatSystemPrompt } from "../../config.js";
 import { getIsInDocumentationMode } from "../../state.js";
 import { ReplHistory } from "../../utils/ReplHistory.js";
-import withStatusLogs from "../../utils/withStatusLogs.js";
 import { defineChatSessionFunction } from "../../evaluator/LlamaChatSession/utils/defineChatSessionFunction.js";
 import { getLlama } from "../../bindings/getLlama.js";
 import { LlamaGrammar } from "../../evaluator/LlamaGrammar.js";
 import { LlamaChatSession } from "../../evaluator/LlamaChatSession/LlamaChatSession.js";
-import { LlamaModel } from "../../evaluator/LlamaModel.js";
-import { LlamaContext } from "../../evaluator/LlamaContext/LlamaContext.js";
 import { LlamaJsonSchemaGrammar } from "../../evaluator/LlamaJsonSchemaGrammar.js";
-import { LlamaLogLevel } from "../../bindings/types.js";
-import { resolveChatWrapperBasedOnWrapperTypeName, chatWrapperTypeNames } from "../../bindings/utils/resolveChatWrapperBasedOnWrapperTypeName.js";
+import { LlamaLogLevel, LlamaLogLevelGreaterThan } from "../../bindings/types.js";
+import withOra from "../../utils/withOra.js";
+import { TokenMeter } from "../../evaluator/TokenMeter.js";
+import { printInfoLine } from "../utils/printInfoLine.js";
+import { resolveChatWrapper, specializedChatWrapperTypeNames } from "../../chatWrappers/utils/resolveChatWrapper.js";
+import { GeneralChatWrapper } from "../../chatWrappers/GeneralChatWrapper.js";
+import { printCommonInfoLines } from "../utils/printCommonInfoLines.js";
+import { resolveCommandGgufPath } from "../utils/resolveCommandGgufPath.js";
 export const ChatCommand = {
-    command: "chat",
+    command: "chat [modelPath]",
     describe: "Chat with a Llama model",
     builder(yargs) {
         const isInDocumentationMode = getIsInDocumentationMode();
         return yargs
             .option("model", {
-            alias: "m",
+            alias: ["m", "modelPath"],
             type: "string",
             demandOption: true,
             description: "Llama model file to use for the chat",
@@ -64,15 +67,22 @@ export const ChatCommand = {
             alias: "w",
             type: "string",
             default: "auto",
-            choices: chatWrapperTypeNames,
+            choices: ["auto", ...specializedChatWrapperTypeNames],
             description: "Chat wrapper to use. Use `auto` to automatically select a wrapper based on the model's BOS token",
             group: "Optional:"
+        })
+            .option("noJinja", {
+            type: "boolean",
+            default: false,
+            description: "Don't use a Jinja wrapper, even if it's the best option for the model",
+            group: "Optional:"
         })
             .option("contextSize", {
             alias: "c",
             type: "number",
-            default: 1024 * 4,
             description: "Context size to use for the model context",
+            default: -1,
+            defaultDescription: "Automatically determined based on the available VRAM",
             group: "Optional:"
         })
             .option("batchSize", {
@@ -80,6 +90,13 @@ export const ChatCommand = {
             type: "number",
             description: "Batch size to use for the model context. The default value is the context size",
             group: "Optional:"
+        })
+            .option("noTrimWhitespace", {
+            type: "boolean",
+            alias: ["noTrim"],
+            default: false,
+            description: "Don't trim whitespaces from the model response",
+            group: "Optional:"
         })
             .option("grammar", {
             alias: "g",
@@ -133,6 +150,8 @@ export const ChatCommand = {
             alias: "gl",
             type: "number",
             description: "number of layers to store in VRAM",
+            default: -1,
+            defaultDescription: "Automatically determined based on the available VRAM",
             group: "Optional:"
         })
             .option("repeatPenalty", {
@@ -189,11 +208,17 @@ export const ChatCommand = {
             description: "Provide access to environment functions like `getDate` and `getTime`",
             group: "Optional:"
         })
-            .option("noInfoLog", {
-            alias: "nl",
+            .option("debug", {
+            alias: "d",
             type: "boolean",
             default: false,
-            description: "Disable llama.cpp info logs",
+            description: "Print llama.cpp info and debug logs",
+            group: "Optional:"
+        })
+            .option("meter", {
+            type: "boolean",
+            default: false,
+            description: "Log how many tokens were used as input and output for each response",
             group: "Optional:"
         })
             .option("printTimings", {
@@ -204,28 +229,36 @@ export const ChatCommand = {
             group: "Optional:"
         });
     },
-    async handler({ model, systemInfo, systemPrompt, systemPromptFile, prompt, promptFile, wrapper, contextSize, batchSize, grammar, jsonSchemaGrammarFile, threads, temperature, minP, topK, topP, gpuLayers, repeatPenalty, lastTokensRepeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, maxTokens, noHistory, environmentFunctions, noInfoLog, printTimings }) {
+    async handler({ model, systemInfo, systemPrompt, systemPromptFile, prompt, promptFile, wrapper, noJinja, contextSize, batchSize, noTrimWhitespace, grammar, jsonSchemaGrammarFile, threads, temperature, minP, topK, topP, gpuLayers, repeatPenalty, lastTokensRepeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, maxTokens, noHistory, environmentFunctions, debug, meter, printTimings }) {
         try {
             await RunChat({
-                model, systemInfo, systemPrompt, systemPromptFile, prompt, promptFile, wrapper, contextSize, batchSize,
-                grammar, jsonSchemaGrammarFile, threads, temperature, minP, topK, topP, gpuLayers, lastTokensRepeatPenalty,
-                repeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, maxTokens,
-                noHistory, environmentFunctions, noInfoLog, printTimings
+                model, systemInfo, systemPrompt, systemPromptFile, prompt, promptFile, wrapper, noJinja, contextSize, batchSize,
+                noTrimWhitespace, grammar, jsonSchemaGrammarFile, threads, temperature, minP, topK, topP, gpuLayers,
+                lastTokensRepeatPenalty, repeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, maxTokens,
+                noHistory, environmentFunctions, debug, meter, printTimings
             });
         }
         catch (err) {
+            await new Promise((accept) => setTimeout(accept, 0)); // wait for logs to finish printing
             console.error(err);
             process.exit(1);
         }
     }
 };
-async function RunChat({ model: modelArg, systemInfo, systemPrompt, systemPromptFile, prompt, promptFile, wrapper, contextSize, batchSize, grammar: grammarArg, jsonSchemaGrammarFile: jsonSchemaGrammarFilePath, threads, temperature, minP, topK, topP, gpuLayers, lastTokensRepeatPenalty, repeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, maxTokens, noHistory, environmentFunctions, noInfoLog, printTimings }) {
-    if (noInfoLog)
-        console.info(`${chalk.yellow("Log level:")} warn`);
+async function RunChat({ model: modelArg, systemInfo, systemPrompt, systemPromptFile, prompt, promptFile, wrapper, noJinja, contextSize, batchSize, noTrimWhitespace, grammar: grammarArg, jsonSchemaGrammarFile: jsonSchemaGrammarFilePath, threads, temperature, minP, topK, topP, gpuLayers, lastTokensRepeatPenalty, repeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, maxTokens, noHistory, environmentFunctions, debug, meter, printTimings }) {
+    if (contextSize === -1)
+        contextSize = undefined;
+    if (gpuLayers === -1)
+        gpuLayers = undefined;
+    const trimWhitespace = !noTrimWhitespace;
+    if (debug)
+        console.info(`${chalk.yellow("Log level:")} debug`);
+    const resolvedModelPath = await resolveCommandGgufPath(modelArg);
+    const llamaLogLevel = debug
+        ? LlamaLogLevel.debug
+        : LlamaLogLevel.warn;
     const llama = await getLlama("lastBuild", {
-        logLevel: noInfoLog
-            ? LlamaLogLevel.warn
-            : LlamaLogLevel.debug
+        logLevel: llamaLogLevel
     });
     const logBatchSize = batchSize != null;
     if (systemInfo)
@@ -240,75 +273,121 @@ async function RunChat({ model: modelArg, systemInfo, systemPrompt, systemPrompt
             console.warn(chalk.yellow("Both `prompt` and `promptFile` were specified. `promptFile` will be used."));
         prompt = await fs.readFile(path.resolve(process.cwd(), promptFile), "utf8");
     }
-    if (batchSize == null)
-        batchSize = contextSize;
-    else if (batchSize > contextSize) {
+    if (batchSize != null && contextSize != null && batchSize > contextSize) {
         console.warn(chalk.yellow("Batch size is greater than the context size. Batch size will be set to the context size."));
         batchSize = contextSize;
     }
     let initialPrompt = prompt ?? null;
-    const model = await withStatusLogs({
+    const model = await withOra({
         loading: chalk.blue("Loading model"),
         success: chalk.blue("Model loaded"),
-        fail: chalk.blue("Failed to load model")
-    }, async () => new LlamaModel({
-        llama,
-        modelPath: path.resolve(process.cwd(), modelArg),
-        gpuLayers: gpuLayers != null ? gpuLayers : undefined
-    }));
-    const context = await withStatusLogs({
+        fail: chalk.blue("Failed to load model"),
+        useStatusLogs: debug
+    }, async () => {
+        try {
+            return await llama.loadModel({
+                modelPath: resolvedModelPath,
+                gpuLayers: gpuLayers != null ? gpuLayers : undefined
+            });
+        }
+        finally {
+            if (llama.logLevel === LlamaLogLevel.debug) {
+                await new Promise((accept) => setTimeout(accept, 0)); // wait for logs to finish printing
+                console.info();
+            }
+        }
+    });
+    const context = await withOra({
         loading: chalk.blue("Creating context"),
         success: chalk.blue("Context created"),
-        fail: chalk.blue("Failed to create context")
-    }, async () => new LlamaContext({
-        model,
-        contextSize,
-        batchSize,
-        threads
-    }));
+        fail: chalk.blue("Failed to create context"),
+        useStatusLogs: debug
+    }, async () => {
+        try {
+            return await model.createContext({
+                contextSize: contextSize != null ? contextSize : undefined,
+                batchSize: batchSize != null ? batchSize : undefined,
+                threads
+            });
+        }
+        finally {
+            if (llama.logLevel === LlamaLogLevel.debug) {
+                await new Promise((accept) => setTimeout(accept, 0)); // wait for logs to finish printing
+                console.info();
+            }
+        }
+    });
     const grammar = jsonSchemaGrammarFilePath != null
         ? new LlamaJsonSchemaGrammar(llama, await fs.readJson(path.resolve(process.cwd(), jsonSchemaGrammarFilePath)))
         : grammarArg !== "text"
             ? await LlamaGrammar.getFor(llama, grammarArg)
             : undefined;
-    const bos = model.tokens.bosString; // bos = beginning of sequence
-    const eos = model.tokens.bosString; // eos = end of sequence
-    const chatWrapper = resolveChatWrapperBasedOnWrapperTypeName(wrapper, {
-        bosString: bos,
+    const chatWrapper = resolveChatWrapper({
+        type: wrapper,
+        bosString: model.tokens.bosString,
         filename: model.filename,
-        typeDescription: model.typeDescription
-    });
+        fileInfo: model.fileInfo,
+        tokenizer: model.tokenize,
+        noJinja
+    }) ?? new GeneralChatWrapper();
+    const contextSequence = context.getSequence();
     const session = new LlamaChatSession({
-        contextSequence: context.getSequence(),
+        contextSequence,
         systemPrompt,
         chatWrapper: chatWrapper
     });
+    let lastTokenMeterState = contextSequence.tokenMeter.getState();
     await new Promise((accept) => setTimeout(accept, 0)); // wait for logs to finish printing
     if (grammarArg != "text" && jsonSchemaGrammarFilePath != null)
         console.warn(chalk.yellow("Both `grammar` and `jsonSchemaGrammarFile` were specified. `jsonSchemaGrammarFile` will be used."));
-    console.info(`${chalk.yellow("Context size:")} ${context.contextSize}`);
-    if (logBatchSize)
-        console.info(`${chalk.yellow("Batch size:")} ${context.batchSize}`);
-    console.info(`${chalk.yellow("Train context size:")} ${model.trainContextSize}`);
-    console.info(`${chalk.yellow("Model type:")} ${model.typeDescription}`);
-    console.info(`${chalk.yellow("BOS:")} ${bos}`);
-    console.info(`${chalk.yellow("EOS:")} ${eos}`);
-    console.info(`${chalk.yellow("Chat wrapper:")} ${chatWrapper.wrapperName}`);
-    console.info(`${chalk.yellow("Repeat penalty:")} ${repeatPenalty} (apply to last ${lastTokensRepeatPenalty} tokens)`);
-    if (repeatFrequencyPenalty != null)
-        console.info(`${chalk.yellow("Repeat frequency penalty:")} ${repeatFrequencyPenalty}`);
-    if (repeatPresencePenalty != null)
-        console.info(`${chalk.yellow("Repeat presence penalty:")} ${repeatPresencePenalty}`);
-    if (!penalizeRepeatingNewLine)
-        console.info(`${chalk.yellow("Penalize repeating new line:")} disabled`);
-    if (jsonSchemaGrammarFilePath != null)
-        console.info(`${chalk.yellow("JSON schema grammar file:")} ${path.relative(process.cwd(), path.resolve(process.cwd(), jsonSchemaGrammarFilePath))}`);
-    else if (grammarArg !== "text")
-        console.info(`${chalk.yellow("Grammar:")} ${grammarArg}`);
     if (environmentFunctions && grammar != null) {
         console.warn(chalk.yellow("Environment functions are disabled since a grammar is already specified"));
         environmentFunctions = false;
     }
+    const padTitle = "Context".length + 1;
+    printCommonInfoLines({
+        context,
+        minTitleLength: padTitle,
+        printBos: true,
+        printEos: true,
+        logBatchSize,
+        tokenMeterEnabled: meter
+    });
+    printInfoLine({
+        title: "Chat",
+        padTitle: padTitle,
+        info: [{
+                title: "Wrapper",
+                value: chatWrapper.wrapperName
+            }, {
+                title: "Repeat penalty",
+                value: `${repeatPenalty} (apply to last ${lastTokensRepeatPenalty} tokens)`
+            }, {
+                show: repeatFrequencyPenalty != null,
+                title: "Repeat frequency penalty",
+                value: String(repeatFrequencyPenalty)
+            }, {
+                show: repeatPresencePenalty != null,
+                title: "Repeat presence penalty",
+                value: String(repeatPresencePenalty)
+            }, {
+                show: !penalizeRepeatingNewLine,
+                title: "Penalize repeating new line",
+                value: "disabled"
+            }, {
+                show: jsonSchemaGrammarFilePath != null,
+                title: "JSON schema grammar file",
+                value: () => path.relative(process.cwd(), path.resolve(process.cwd(), jsonSchemaGrammarFilePath ?? ""))
+            }, {
+                show: jsonSchemaGrammarFilePath == null && grammarArg !== "text",
+                title: "Grammar",
+                value: grammarArg
+            }, {
+                show: environmentFunctions,
+                title: "Environment functions",
+                value: "enabled"
+            }]
+    });
     // this is for ora to not interfere with readline
     await new Promise(resolve => setTimeout(resolve, 1));
     const replHistory = await ReplHistory.load(chatCommandHistoryFilePath, !noHistory);
@@ -324,6 +403,8 @@ async function RunChat({ model: modelArg, systemInfo, systemPrompt, systemPrompt
     }
     // eslint-disable-next-line no-constant-condition
     while (true) {
+        let hadNoWhitespaceTextInThisIteration = false;
+        let nextPrintLeftovers = "";
         const input = initialPrompt != null
             ? initialPrompt
             : await getPrompt();
@@ -357,16 +438,42 @@ async function RunChat({ model: modelArg, systemInfo, systemPrompt, systemPrompt
                     ? undefined
                     : maxTokens,
             onToken(chunk) {
-                process.stdout.write(model.detokenize(chunk));
+                let text = nextPrintLeftovers + model.detokenize(chunk);
+                nextPrintLeftovers = "";
+                if (trimWhitespace) {
+                    if (!hadNoWhitespaceTextInThisIteration) {
+                        text = text.trimStart();
+                        if (text.length > 0)
+                            hadNoWhitespaceTextInThisIteration = true;
+                    }
+                    const textWithTrimmedEnd = text.trimEnd();
+                    if (textWithTrimmedEnd.length < text.length) {
+                        nextPrintLeftovers = text.slice(textWithTrimmedEnd.length);
+                        text = textWithTrimmedEnd;
+                    }
+                }
+                process.stdout.write(text);
             },
             functions: (grammar == null && environmentFunctions)
                 ? defaultEnvironmentFunctions
-                : undefined
+                : undefined,
+            trimWhitespaceSuffix: trimWhitespace
         });
         process.stdout.write(endColor);
         console.log();
-        if (printTimings)
+        if (printTimings) {
+            if (LlamaLogLevelGreaterThan(llama.logLevel, LlamaLogLevel.info))
+                llama.logLevel = LlamaLogLevel.info;
             await context.printTimings();
+            await new Promise((accept) => setTimeout(accept, 0)); // wait for logs to finish printing
+            llama.logLevel = llamaLogLevel;
+        }
+        if (meter) {
+            const newTokenMeterState = contextSequence.tokenMeter.getState();
+            const tokenMeterDiff = TokenMeter.diff(newTokenMeterState, lastTokenMeterState);
+            lastTokenMeterState = newTokenMeterState;
+            console.info(`${chalk.dim("Input tokens:")} ${String(tokenMeterDiff.usedInputTokens).padEnd(5, " ")}  ${chalk.dim("Output tokens:")} ${tokenMeterDiff.usedOutputTokens}`);
+        }
     }
 }
 const defaultEnvironmentFunctions = {