node-llama-cpp 2.8.6 → 3.0.0-beta.10

package/dist/chatWrappers/resolveChatWrapperBasedOnModel.js

@@ -0,0 +1,60 @@
+import { parseModelFileName } from "../utils/parseModelFileName.js";
+import { parseModelTypeDescription } from "../utils/parseModelTypeDescription.js";
+import { LlamaChatWrapper } from "./LlamaChatWrapper.js";
+import { ChatMLChatWrapper } from "./ChatMLChatWrapper.js";
+import { GeneralChatWrapper } from "./GeneralChatWrapper.js";
+import { FalconChatWrapper } from "./FalconChatWrapper.js";
+import { FunctionaryChatWrapper } from "./FunctionaryChatWrapper.js";
+import { AlpacaChatWrapper } from "./AlpacaChatWrapper.js";
+/**
+ * @param options
+ */
+export function resolveChatWrapperBasedOnModel({ bosString, filename, typeDescription }) {
+    if (filename != null) {
+        const { name, subType, fileType } = parseModelFileName(filename);
+        if (fileType?.toLowerCase() === "gguf") {
+            const lowercaseName = name?.toLowerCase();
+            const lowercaseSubType = subType?.toLowerCase();
+            const splitLowercaseSubType = lowercaseSubType?.split("-") ?? [];
+            const firstSplitLowercaseSubType = splitLowercaseSubType[0];
+            if (lowercaseName === "llama") {
+                if (splitLowercaseSubType.includes("chat"))
+                    return LlamaChatWrapper;
+                return GeneralChatWrapper;
+            }
+            else if (lowercaseName === "yarn" && firstSplitLowercaseSubType === "llama")
+                return LlamaChatWrapper;
+            else if (lowercaseName === "orca")
+                return ChatMLChatWrapper;
+            else if (lowercaseName === "phind" && lowercaseSubType === "codellama")
+                return LlamaChatWrapper;
+            else if (lowercaseName === "mistral")
+                return GeneralChatWrapper;
+            else if (firstSplitLowercaseSubType === "llama")
+                return LlamaChatWrapper;
+            else if (lowercaseSubType === "alpaca")
+                return AlpacaChatWrapper;
+            else if (lowercaseName === "functionary")
+                return FunctionaryChatWrapper;
+            else if (lowercaseName === "dolphin" && splitLowercaseSubType.includes("mistral"))
+                return ChatMLChatWrapper;
+        }
+    }
+    if (typeDescription != null) {
+        const { arch } = parseModelTypeDescription(typeDescription);
+        if (arch === "llama")
+            return LlamaChatWrapper;
+        else if (arch === "falcon")
+            return FalconChatWrapper;
+    }
+    if (bosString === "" || bosString == null)
+        return null;
+    if ("<s>[INST] <<SYS>>\n".startsWith(bosString)) {
+        return LlamaChatWrapper;
+    }
+    else if ("<|im_start|>system\n".startsWith(bosString)) {
+        return ChatMLChatWrapper;
+    }
+    return null;
+}
+//# sourceMappingURL=resolveChatWrapperBasedOnModel.js.map

package/dist/chatWrappers/resolveChatWrapperBasedOnModel.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolveChatWrapperBasedOnModel.js","sourceRoot":"","sources":["../../src/chatWrappers/resolveChatWrapperBasedOnModel.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,kBAAkB,EAAC,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAC,yBAAyB,EAAC,MAAM,uCAAuC,CAAC;AAEhF,OAAO,EAAC,gBAAgB,EAAC,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAC,iBAAiB,EAAC,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAC,kBAAkB,EAAC,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAC,iBAAiB,EAAC,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAC,sBAAsB,EAAC,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAC,iBAAiB,EAAC,MAAM,wBAAwB,CAAC;AAGzD;;GAEG;AACH,MAAM,UAAU,8BAA8B,CAAC,EAC3C,SAAS,EACT,QAAQ,EACR,eAAe,EAOlB;IACG,IAAI,QAAQ,IAAI,IAAI,EAAE;QAClB,MAAM,EAAC,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAC,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QAE/D,IAAI,QAAQ,EAAE,WAAW,EAAE,KAAK,MAAM,EAAE;YACpC,MAAM,aAAa,GAAG,IAAI,EAAE,WAAW,EAAE,CAAC;YAC1C,MAAM,gBAAgB,GAAG,OAAO,EAAE,WAAW,EAAE,CAAC;YAChD,MAAM,qBAAqB,GAAG,gBAAgB,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;YACjE,MAAM,0BAA0B,GAAG,qBAAqB,CAAC,CAAC,CAAC,CAAC;YAE5D,IAAI,aAAa,KAAK,OAAO,EAAE;gBAC3B,IAAI,qBAAqB,CAAC,QAAQ,CAAC,MAAM,CAAC;oBACtC,OAAO,gBAAgB,CAAC;gBAE5B,OAAO,kBAAkB,CAAC;aAC7B;iBAAM,IAAI,aAAa,KAAK,MAAM,IAAI,0BAA0B,KAAK,OAAO;gBACzE,OAAO,gBAAgB,CAAC;iBACvB,IAAI,aAAa,KAAK,MAAM;gBAC7B,OAAO,iBAAiB,CAAC;iBACxB,IAAI,aAAa,KAAK,OAAO,IAAI,gBAAgB,KAAK,WAAW;gBAClE,OAAO,gBAAgB,CAAC;iBACvB,IAAI,aAAa,KAAK,SAAS;gBAChC,OAAO,kBAAkB,CAAC;iBACzB,IAAI,0BAA0B,KAAK,OAAO;gBAC3C,OAAO,gBAAgB,CAAC;iBACvB,IAAI,gBAAgB,KAAK,QAAQ;gBAClC,OAAO,iBAAiB,CAAC;iBACxB,IAAI,aAAa,KAAK,aAAa;gBACpC,OAAO,sBAAsB,CAAC;iBAC7B,IAAI,aAAa,KAAK,SAAS,IAAI,qBAAqB,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAC7E,OAAO,iBAAiB,CAAC;SAChC;KACJ;IAED,IAAI,eAAe,IAAI,IAAI,EAAE;QACzB,MAAM,EAAC,IAAI,EAAC,GAAG,yBAAyB,CAAC,eAAe,CAAC,CAAC;QAE1D,IAAI,IAAI,KAAK,OAAO;YAChB,OAAO,gBAAgB,CAAC;aACvB,IAAI,IAAI,KAAK,QAAQ;YACtB,OAAO,iBAAiB,CAAC;KAChC;IAED,IAAI,SAAS,KAAK,EAAE,IAAI,SAAS,IAAI,IAAI;QACrC,OAAO,IAAI,CAAC;IAEhB,IAAI,qBAAqB,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;QAC7C,OAAO,gBAAgB,CAAC;KAC3B;SAAM,IAAI,sBAAsB,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;QACrD,OAAO,iBAAiB,CAAC;KAC5B;IAED,OAAO,IAAI,CAAC;AAChB,CAAC"}

package/dist/cli/cli.js

@@ -2,17 +2,20 @@
 import { fileURLToPath } from "url";
 import path from "path";
 import yargs from "yargs";
-// eslint-disable-next-line node/file-extension-in-import
+// eslint-disable-next-line n/file-extension-in-import
 import { hideBin } from "yargs/helpers";
 import fs from "fs-extra";
 import { cliBinName } from "../config.js";
+import { setIsRunningFromCLI } from "../state.js";
 import { DownloadCommand } from "./commands/DownloadCommand.js";
 import { BuildCommand } from "./commands/BuildCommand.js";
 import { OnPostInstallCommand } from "./commands/OnPostInstallCommand.js";
 import { ClearCommand } from "./commands/ClearCommand.js";
 import { ChatCommand } from "./commands/ChatCommand.js";
+import { DebugCommand } from "./commands/DebugCommand.js";
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const packageJson = fs.readJSONSync(path.join(__dirname, "..", "..", "package.json"));
+setIsRunningFromCLI(true);
 const yarg = yargs(hideBin(process.argv));
 yarg
     .scriptName(cliBinName)
@@ -22,6 +25,7 @@ yarg
     .command(ClearCommand)
     .command(ChatCommand)
     .command(OnPostInstallCommand)
+    .command(DebugCommand)
     .recommendCommands()
     .demandCommand(1)
     .strict()

package/dist/cli/cli.js.map

@@ -1 +1 @@
-{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/cli/cli.ts"],"names":[],"mappings":";AAEA,OAAO,EAAC,aAAa,EAAC,MAAM,KAAK,CAAC;AAClC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,yDAAyD;AACzD,OAAO,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,UAAU,EAAC,MAAM,cAAc,CAAC;AACxC,OAAO,EAAC,eAAe,EAAC,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAC,YAAY,EAAC,MAAM,4BAA4B,CAAC;AACxD,OAAO,EAAC,oBAAoB,EAAC,MAAM,oCAAoC,CAAC;AACxE,OAAO,EAAC,YAAY,EAAC,MAAM,4BAA4B,CAAC;AACxD,OAAO,EAAC,WAAW,EAAC,MAAM,2BAA2B,CAAC;AAEtD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE/D,MAAM,WAAW,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC;AAEtF,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;AAE1C,IAAI;KACC,UAAU,CAAC,UAAU,CAAC;KACtB,KAAK,CAAC,+BAA+B,CAAC;KACtC,OAAO,CAAC,eAAe,CAAC;KACxB,OAAO,CAAC,YAAY,CAAC;KACrB,OAAO,CAAC,YAAY,CAAC;KACrB,OAAO,CAAC,WAAW,CAAC;KACpB,OAAO,CAAC,oBAAoB,CAAC;KAC7B,iBAAiB,EAAE;KACnB,aAAa,CAAC,CAAC,CAAC;KAChB,MAAM,EAAE;KACR,cAAc,EAAE;KAChB,KAAK,CAAC,GAAG,EAAE,SAAS,CAAC;KACrB,IAAI,CAAC,GAAG,CAAC;KACT,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC;KAClB,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC;KAC5B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;KACzC,KAAK,EAAE,CAAC"}
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/cli/cli.ts"],"names":[],"mappings":";AAEA,OAAO,EAAC,aAAa,EAAC,MAAM,KAAK,CAAC;AAClC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,sDAAsD;AACtD,OAAO,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,UAAU,EAAC,MAAM,cAAc,CAAC;AACxC,OAAO,EAAC,mBAAmB,EAAC,MAAM,aAAa,CAAC;AAChD,OAAO,EAAC,eAAe,EAAC,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAC,YAAY,EAAC,MAAM,4BAA4B,CAAC;AACxD,OAAO,EAAC,oBAAoB,EAAC,MAAM,oCAAoC,CAAC;AACxE,OAAO,EAAC,YAAY,EAAC,MAAM,4BAA4B,CAAC;AACxD,OAAO,EAAC,WAAW,EAAC,MAAM,2BAA2B,CAAC;AACtD,OAAO,EAAC,YAAY,EAAC,MAAM,4BAA4B,CAAC;AAExD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE/D,MAAM,WAAW,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC;AAEtF,mBAAmB,CAAC,IAAI,CAAC,CAAC;AAE1B,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;AAE1C,IAAI;KACC,UAAU,CAAC,UAAU,CAAC;KACtB,KAAK,CAAC,+BAA+B,CAAC;KACtC,OAAO,CAAC,eAAe,CAAC;KACxB,OAAO,CAAC,YAAY,CAAC;KACrB,OAAO,CAAC,YAAY,CAAC;KACrB,OAAO,CAAC,WAAW,CAAC;KACpB,OAAO,CAAC,oBAAoB,CAAC;KAC7B,OAAO,CAAC,YAAY,CAAC;KACrB,iBAAiB,EAAE;KACnB,aAAa,CAAC,CAAC,CAAC;KAChB,MAAM,EAAE;KACR,cAAc,EAAE;KAChB,KAAK,CAAC,GAAG,EAAE,SAAS,CAAC;KACrB,IAAI,CAAC,GAAG,CAAC;KACT,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC;KAClB,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC;KAC5B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;KACzC,KAAK,EAAE,CAAC"}

package/dist/cli/commands/BuildCommand.d.ts

@@ -4,7 +4,8 @@ type BuildCommand = {
     nodeTarget?: string;
     metal?: boolean;
     cuda?: boolean;
+    noUsageExample?: boolean;
 };
 export declare const BuildCommand: CommandModule<object, BuildCommand>;
-export declare function BuildLlamaCppCommand({ arch, nodeTarget, metal, cuda }: BuildCommand): Promise<void>;
+export declare function BuildLlamaCppCommand({ arch, nodeTarget, metal, cuda, noUsageExample, noCustomCmakeBuildOptionsInBinaryFolderName }: BuildCommand): Promise<void>;
 export {};

package/dist/cli/commands/BuildCommand.js

@@ -1,13 +1,16 @@
 import process from "process";
 import chalk from "chalk";
-import fs from "fs-extra";
-import { compileLlamaCpp } from "../../utils/compileLLamaCpp.js";
+import { compileLlamaCpp } from "../../bindings/utils/compileLLamaCpp.js";
 import withOra from "../../utils/withOra.js";
 import { clearTempFolder } from "../../utils/clearTempFolder.js";
-import { defaultLlamaCppCudaSupport, defaultLlamaCppMetalSupport, llamaCppDirectory } from "../../config.js";
+import { builtinLlamaCppGitHubRepo, builtinLlamaCppRelease, defaultLlamaCppCudaSupport, defaultLlamaCppMetalSupport, isCI } from "../../config.js";
 import { downloadCmakeIfNeeded } from "../../utils/cmake.js";
 import withStatusLogs from "../../utils/withStatusLogs.js";
 import { getIsInDocumentationMode } from "../../state.js";
+import { logBinaryUsageExampleToConsole } from "../../bindings/utils/logBinaryUsageExampleToConsole.js";
+import { getPlatform } from "../../bindings/utils/getPlatform.js";
+import { resolveCustomCmakeOptions } from "../../bindings/utils/resolveCustomCmakeOptions.js";
+import { getClonedLlamaCppRepoReleaseInfo, isLlamaCppRepoCloned } from "../../bindings/utils/cloneLlamaCppRepo.js";
 export const BuildCommand = {
     command: "build",
     describe: "Compile the currently downloaded llama.cpp",
@@ -33,15 +36,31 @@ export const BuildCommand = {
             type: "boolean",
             default: defaultLlamaCppCudaSupport,
             description: "Compile llama.cpp with CUDA support. Can also be set via the NODE_LLAMA_CPP_CUDA environment variable"
+        })
+            .option("noUsageExample", {
+            alias: "nu",
+            type: "boolean",
+            default: false,
+            description: "Don't print code usage example after building"
+        })
+            .option("noCustomCmakeBuildOptionsInBinaryFolderName", {
+            type: "boolean",
+            hidden: true,
+            default: false,
+            description: "Don't include custom CMake build options in build folder name"
         });
     },
     handler: BuildLlamaCppCommand
 };
-export async function BuildLlamaCppCommand({ arch = undefined, nodeTarget = undefined, metal = defaultLlamaCppMetalSupport, cuda = defaultLlamaCppCudaSupport }) {
-    if (!(await fs.pathExists(llamaCppDirectory))) {
+export async function BuildLlamaCppCommand({ arch = undefined, nodeTarget = undefined, metal = defaultLlamaCppMetalSupport, cuda = defaultLlamaCppCudaSupport, noUsageExample = false, noCustomCmakeBuildOptionsInBinaryFolderName = false }) {
+    if (!(await isLlamaCppRepoCloned())) {
         console.log(chalk.red('llama.cpp is not downloaded. Please run "node-llama-cpp download" first'));
         process.exit(1);
     }
+    const includeBuildOptionsInBinaryFolderName = !noCustomCmakeBuildOptionsInBinaryFolderName || !isCI;
+    const clonedLlamaCppRepoReleaseInfo = await getClonedLlamaCppRepoReleaseInfo();
+    const platform = getPlatform();
+    const customCmakeOptions = resolveCustomCmakeOptions();
     if (metal && process.platform === "darwin") {
         console.log(`${chalk.yellow("Metal:")} enabled`);
     }
@@ -49,17 +68,33 @@ export async function BuildLlamaCppCommand({ arch = undefined, nodeTarget = unde
         console.log(`${chalk.yellow("CUDA:")} enabled`);
     }
     await downloadCmakeIfNeeded(true);
+    const buildOptions = {
+        customCmakeOptions,
+        progressLogs: true,
+        platform,
+        arch: arch
+            ? arch
+            : process.arch,
+        computeLayers: {
+            metal,
+            cuda
+        },
+        llamaCpp: {
+            repo: clonedLlamaCppRepoReleaseInfo?.llamaCppGithubRepo ?? builtinLlamaCppGitHubRepo,
+            release: clonedLlamaCppRepoReleaseInfo?.tag ?? builtinLlamaCppRelease
+        }
+    };
     await withStatusLogs({
         loading: chalk.blue("Compiling llama.cpp"),
         success: chalk.blue("Compiled llama.cpp"),
         fail: chalk.blue("Failed to compile llama.cpp")
     }, async () => {
-        await compileLlamaCpp({
-            arch: arch ? arch : undefined,
+        await compileLlamaCpp(buildOptions, {
             nodeTarget: nodeTarget ? nodeTarget : undefined,
-            setUsedBinFlag: true,
-            metal,
-            cuda
+            updateLastBuildInfo: true,
+            downloadCmakeIfNeeded: false,
+            ensureLlamaCppRepoIsCloned: false,
+            includeBuildOptionsInBinaryFolderName
         });
     });
     await withOra({
@@ -69,5 +104,10 @@ export async function BuildLlamaCppCommand({ arch = undefined, nodeTarget = unde
     }, async () => {
         await clearTempFolder();
     });
+    if (!noUsageExample) {
+        console.log();
+        logBinaryUsageExampleToConsole(buildOptions, true);
+        console.log();
+    }
 }
 //# sourceMappingURL=BuildCommand.js.map

package/dist/cli/commands/BuildCommand.js.map

@@ -1 +1 @@
-{"version":3,"file":"BuildCommand.js","sourceRoot":"","sources":["../../../src/cli/commands/BuildCommand.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,eAAe,EAAC,MAAM,gCAAgC,CAAC;AAC/D,OAAO,OAAO,MAAM,wBAAwB,CAAC;AAC7C,OAAO,EAAC,eAAe,EAAC,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAC,0BAA0B,EAAE,2BAA2B,EAAE,iBAAiB,EAAC,MAAM,iBAAiB,CAAC;AAC3G,OAAO,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAC3D,OAAO,cAAc,MAAM,+BAA+B,CAAC;AAC3D,OAAO,EAAC,wBAAwB,EAAC,MAAM,gBAAgB,CAAC;AASxD,MAAM,CAAC,MAAM,YAAY,GAAwC;IAC7D,OAAO,EAAE,OAAO;IAChB,QAAQ,EAAE,4CAA4C;IACtD,OAAO,CAAC,KAAK;QACT,MAAM,qBAAqB,GAAG,wBAAwB,EAAE,CAAC;QAEzD,OAAO,KAAK;aACP,MAAM,CAAC,MAAM,EAAE;YACZ,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,2CAA2C;SAC3D,CAAC;aACD,MAAM,CAAC,YAAY,EAAE;YAClB,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,gEAAgE;SAChF,CAAC;aACD,MAAM,CAAC,OAAO,EAAE;YACb,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,2BAA2B,IAAI,qBAAqB;YAC7D,WAAW,EAAE,2KAA2K;SAC3L,CAAC;aACD,MAAM,CAAC,MAAM,EAAE;YACZ,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,0BAA0B;YACnC,WAAW,EAAE,uGAAuG;SACvH,CAAC,CAAC;IACX,CAAC;IACD,OAAO,EAAE,oBAAoB;CAChC,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,EACvC,IAAI,GAAG,SAAS,EAChB,UAAU,GAAG,SAAS,EACtB,KAAK,GAAG,2BAA2B,EACnC,IAAI,GAAG,0BAA0B,EACtB;IACX,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,EAAE;QAC3C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,yEAAyE,CAAC,CAAC,CAAC;QAClG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACnB;IAED,IAAI,KAAK,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE;QACxC,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;KACpD;IAED,IAAI,IAAI,EAAE;QACN,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;KACnD;IAED,MAAM,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAElC,MAAM,cAAc,CAAC;QACjB,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC;QAC1C,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC;QACzC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,6BAA6B,CAAC;KAClD,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,eAAe,CAAC;YAClB,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;YAC7B,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;YAC/C,cAAc,EAAE,IAAI;YACpB,KAAK;YACL,IAAI;SACP,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC;QAC/C,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC;QAC9C,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,kCAAkC,CAAC;KACvD,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,eAAe,EAAE,CAAC;IAC5B,CAAC,CAAC,CAAC;AACP,CAAC"}
+{"version":3,"file":"BuildCommand.js","sourceRoot":"","sources":["../../../src/cli/commands/BuildCommand.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAC,eAAe,EAAC,MAAM,yCAAyC,CAAC;AACxE,OAAO,OAAO,MAAM,wBAAwB,CAAC;AAC7C,OAAO,EAAC,eAAe,EAAC,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EACH,yBAAyB,EAAE,sBAAsB,EAAE,0BAA0B,EAAE,2BAA2B,EAAE,IAAI,EACnH,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAC3D,OAAO,cAAc,MAAM,+BAA+B,CAAC;AAC3D,OAAO,EAAC,wBAAwB,EAAC,MAAM,gBAAgB,CAAC;AACxD,OAAO,EAAC,8BAA8B,EAAC,MAAM,wDAAwD,CAAC;AACtG,OAAO,EAAC,WAAW,EAAC,MAAM,qCAAqC,CAAC;AAChE,OAAO,EAAC,yBAAyB,EAAC,MAAM,mDAAmD,CAAC;AAC5F,OAAO,EAAC,gCAAgC,EAAE,oBAAoB,EAAC,MAAM,2CAA2C,CAAC;AAcjH,MAAM,CAAC,MAAM,YAAY,GAAwC;IAC7D,OAAO,EAAE,OAAO;IAChB,QAAQ,EAAE,4CAA4C;IACtD,OAAO,CAAC,KAAK;QACT,MAAM,qBAAqB,GAAG,wBAAwB,EAAE,CAAC;QAEzD,OAAO,KAAK;aACP,MAAM,CAAC,MAAM,EAAE;YACZ,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,2CAA2C;SAC3D,CAAC;aACD,MAAM,CAAC,YAAY,EAAE;YAClB,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,gEAAgE;SAChF,CAAC;aACD,MAAM,CAAC,OAAO,EAAE;YACb,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,2BAA2B,IAAI,qBAAqB;YAC7D,WAAW,EAAE,2KAA2K;SAC3L,CAAC;aACD,MAAM,CAAC,MAAM,EAAE;YACZ,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,0BAA0B;YACnC,WAAW,EAAE,uGAAuG;SACvH,CAAC;aACD,MAAM,CAAC,gBAAgB,EAAE;YACtB,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,KAAK;YACd,WAAW,EAAE,+CAA+C;SAC/D,CAAC;aACD,MAAM,CAAC,6CAA6C,EAAE;YACnD,IAAI,EAAE,SAAS;YACf,MAAM,EAAE,IAAI;YACZ,OAAO,EAAE,KAAK;YACd,WAAW,EAAE,+DAA+D;SAC/E,CAAC,CAAC;IACX,CAAC;IACD,OAAO,EAAE,oBAAoB;CAChC,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,EACvC,IAAI,GAAG,SAAS,EAChB,UAAU,GAAG,SAAS,EACtB,KAAK,GAAG,2BAA2B,EACnC,IAAI,GAAG,0BAA0B,EACjC,cAAc,GAAG,KAAK,EACtB,2CAA2C,GAAG,KAAK,EACxC;IACX,IAAI,CAAC,CAAC,MAAM,oBAAoB,EAAE,CAAC,EAAE;QACjC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,yEAAyE,CAAC,CAAC,CAAC;QAClG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACnB;IAED,MAAM,qCAAqC,GAAG,CAAC,2CAA2C,IAAI,CAAC,IAAI,CAAC;IAEpG,MAAM,6BAA6B,GAAG,MAAM,gCAAgC,EAAE,CAAC;IAE/E,MAAM,QAAQ,GAAG,WAAW,EAAE,CAAC;IAC/B,MAAM,kBAAkB,GAAG,yBAAyB,EAAE,CAAC;IAEvD,IAAI,KAAK,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE;QACxC,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;KACpD;IAED,IAAI,IAAI,EAAE;QACN,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;KACnD;IAED,MAAM,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAElC,MAAM,YAAY,GAAiB;QAC/B,kBAAkB;QAClB,YAAY,EAAE,IAAI;QAClB,QAAQ;QACR,IAAI,EAAE,IAAI;YACN,CAAC,CAAC,IAA2B;YAC7B,CAAC,CAAC,OAAO,CAAC,IAAI;QAClB,aAAa,EAAE;YACX,KAAK;YACL,IAAI;SACP;QACD,QAAQ,EAAE;YACN,IAAI,EAAE,6BAA6B,EAAE,kBAAkB,IAAI,yBAAyB;YACpF,OAAO,EAAE,6BAA6B,EAAE,GAAG,IAAI,sBAAsB;SACxE;KACJ,CAAC;IAEF,MAAM,cAAc,CAAC;QACjB,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC;QAC1C,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC;QACzC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,6BAA6B,CAAC;KAClD,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,eAAe,CAAC,YAAY,EAAE;YAChC,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;YAC/C,mBAAmB,EAAE,IAAI;YACzB,qBAAqB,EAAE,KAAK;YAC5B,0BAA0B,EAAE,KAAK;YACjC,qCAAqC;SACxC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC;QAC/C,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC;QAC9C,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,kCAAkC,CAAC;KACvD,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,eAAe,EAAE,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,cAAc,EAAE;QACjB,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,8BAA8B,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;QACnD,OAAO,CAAC,GAAG,EAAE,CAAC;KACjB;AACL,CAAC"}

package/dist/cli/commands/ChatCommand.d.ts

@@ -1,18 +1,21 @@
 import { CommandModule } from "yargs";
-import type { LlamaGrammar } from "../../llamaEvaluator/LlamaGrammar.js";
-declare const modelWrappers: readonly ["auto", "general", "llamaChat", "chatML", "falconChat"];
+import { LlamaGrammar } from "../../evaluator/LlamaGrammar.js";
+declare const modelWrappers: readonly ["auto", "general", "llamaChat", "alpacaChat", "functionary", "chatML", "falconChat"];
 type ChatCommand = {
     model: string;
     systemInfo: boolean;
-    printTimings: boolean;
     systemPrompt: string;
+    systemPromptFile?: string;
     prompt?: string;
+    promptFile?: string;
     wrapper: (typeof modelWrappers)[number];
     contextSize: number;
-    grammar: "text" | Parameters<typeof LlamaGrammar.getFor>[0];
+    batchSize?: number;
+    grammar: "text" | Parameters<typeof LlamaGrammar.getFor>[1];
     jsonSchemaGrammarFile?: string;
     threads: number;
     temperature: number;
+    minP: number;
     topK: number;
     topP: number;
     gpuLayers?: number;
@@ -23,6 +26,9 @@ type ChatCommand = {
     repeatPresencePenalty?: number;
     maxTokens: number;
     noHistory: boolean;
+    environmentFunctions: boolean;
+    noInfoLog: boolean;
+    printTimings: boolean;
 };
 export declare const ChatCommand: CommandModule<object, ChatCommand>;
 export {};

package/dist/cli/commands/ChatCommand.js

@@ -3,16 +3,26 @@ import process from "process";
 import path from "path";
 import chalk from "chalk";
 import fs from "fs-extra";
-import withOra from "../../utils/withOra.js";
 import { chatCommandHistoryFilePath, defaultChatSystemPrompt } from "../../config.js";
-import { LlamaChatPromptWrapper } from "../../chatWrappers/LlamaChatPromptWrapper.js";
-import { GeneralChatPromptWrapper } from "../../chatWrappers/GeneralChatPromptWrapper.js";
-import { ChatMLChatPromptWrapper } from "../../chatWrappers/ChatMLChatPromptWrapper.js";
-import { getChatWrapperByBos } from "../../chatWrappers/createChatWrapperByBos.js";
-import { FalconChatPromptWrapper } from "../../chatWrappers/FalconChatPromptWrapper.js";
+import { LlamaChatWrapper } from "../../chatWrappers/LlamaChatWrapper.js";
+import { GeneralChatWrapper } from "../../chatWrappers/GeneralChatWrapper.js";
+import { ChatMLChatWrapper } from "../../chatWrappers/ChatMLChatWrapper.js";
+import { resolveChatWrapperBasedOnModel } from "../../chatWrappers/resolveChatWrapperBasedOnModel.js";
+import { FalconChatWrapper } from "../../chatWrappers/FalconChatWrapper.js";
 import { getIsInDocumentationMode } from "../../state.js";
 import { ReplHistory } from "../../utils/ReplHistory.js";
-const modelWrappers = ["auto", "general", "llamaChat", "chatML", "falconChat"];
+import withStatusLogs from "../../utils/withStatusLogs.js";
+import { AlpacaChatWrapper } from "../../chatWrappers/AlpacaChatWrapper.js";
+import { FunctionaryChatWrapper } from "../../chatWrappers/FunctionaryChatWrapper.js";
+import { defineChatSessionFunction } from "../../evaluator/LlamaChatSession/utils/defineChatSessionFunction.js";
+import { getLlama } from "../../bindings/getLlama.js";
+import { LlamaGrammar } from "../../evaluator/LlamaGrammar.js";
+import { LlamaChatSession } from "../../evaluator/LlamaChatSession/LlamaChatSession.js";
+import { LlamaModel } from "../../evaluator/LlamaModel.js";
+import { LlamaContext } from "../../evaluator/LlamaContext/LlamaContext.js";
+import { LlamaJsonSchemaGrammar } from "../../evaluator/LlamaJsonSchemaGrammar.js";
+import { LlamaLogLevel } from "../../bindings/types.js";
+const modelWrappers = ["auto", "general", "llamaChat", "alpacaChat", "functionary", "chatML", "falconChat"];
 export const ChatCommand = {
     command: "chat",
     describe: "Chat with a Llama model",
@@ -32,12 +42,6 @@ export const ChatCommand = {
             default: false,
             description: "Print llama.cpp system info",
             group: "Optional:"
-        })
-            .option("printTimings", {
-            type: "boolean",
-            default: false,
-            description: "Print llama.cpp timings",
-            group: "Optional:"
         })
             .option("systemPrompt", {
             alias: "s",
@@ -47,16 +51,26 @@ export const ChatCommand = {
             description: "System prompt to use against the model" +
                 (isInDocumentationMode ? "" : (". [default value: " + defaultChatSystemPrompt.split("\n").join(" ") + "]")),
             group: "Optional:"
+        })
+            .option("systemPromptFile", {
+            type: "string",
+            description: "Path to a file to load text from and use as as the model system prompt",
+            group: "Optional:"
         })
             .option("prompt", {
             type: "string",
             description: "First prompt to automatically send to the model when starting the chat",
             group: "Optional:"
+        })
+            .option("promptFile", {
+            type: "string",
+            description: "Path to a file to load text from and use as a first prompt to automatically send to the model when starting the chat",
+            group: "Optional:"
         })
             .option("wrapper", {
             alias: "w",
             type: "string",
-            default: "general",
+            default: "auto",
             choices: modelWrappers,
             description: "Chat wrapper to use. Use `auto` to automatically select a wrapper based on the model's BOS token",
             group: "Optional:"
@@ -65,7 +79,13 @@ export const ChatCommand = {
             alias: "c",
             type: "number",
             default: 1024 * 4,
-            description: "Context size to use for the model",
+            description: "Context size to use for the model context",
+            group: "Optional:"
+        })
+            .option("batchSize", {
+            alias: "b",
+            type: "number",
+            description: "Batch size to use for the model context. The default value is the context size",
             group: "Optional:"
         })
             .option("grammar", {
@@ -94,6 +114,13 @@ export const ChatCommand = {
             default: 0,
             description: "Temperature is a hyperparameter that controls the randomness of the generated text. It affects the probability distribution of the model's output tokens. A higher temperature (e.g., 1.5) makes the output more random and creative, while a lower temperature (e.g., 0.5) makes the output more focused, deterministic, and conservative. The suggested temperature is 0.8, which provides a balance between randomness and determinism. At the extreme, a temperature of 0 will always pick the most likely next token, leading to identical outputs in each run. Set to `0` to disable.",
             group: "Optional:"
+        })
+            .option("minP", {
+            alias: "mp",
+            type: "number",
+            default: 0,
+            description: "From the next token candidates, discard the percentage of tokens with the lowest probability. For example, if set to `0.05`, 5% of the lowest probability tokens will be discarded. This is useful for generating more high-quality results when using a high temperature. Set to a value between `0` and `1` to enable. Only relevant when `temperature` is set to a value greater than `0`.",
+            group: "Optional:"
         })
             .option("topK", {
             alias: "k",
@@ -161,14 +188,36 @@ export const ChatCommand = {
             default: false,
             description: "Don't load or save chat history",
             group: "Optional:"
+        })
+            .option("environmentFunctions", {
+            alias: "ef",
+            type: "boolean",
+            default: false,
+            description: "Provide access to environment functions like `getDate` and `getTime`",
+            group: "Optional:"
+        })
+            .option("noInfoLog", {
+            alias: "nl",
+            type: "boolean",
+            default: false,
+            description: "Disable llama.cpp info logs",
+            group: "Optional:"
+        })
+            .option("printTimings", {
+            alias: "pt",
+            type: "boolean",
+            default: false,
+            description: "Print llama.cpp timings after each response",
+            group: "Optional:"
         });
     },
-    async handler({ model, systemInfo, systemPrompt, prompt, wrapper, contextSize, grammar, jsonSchemaGrammarFile, threads, temperature, topK, topP, gpuLayers, repeatPenalty, lastTokensRepeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, maxTokens, noHistory, printTimings }) {
+    async handler({ model, systemInfo, systemPrompt, systemPromptFile, prompt, promptFile, wrapper, contextSize, batchSize, grammar, jsonSchemaGrammarFile, threads, temperature, minP, topK, topP, gpuLayers, repeatPenalty, lastTokensRepeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, maxTokens, noHistory, environmentFunctions, noInfoLog, printTimings }) {
         try {
             await RunChat({
-                model, systemInfo, systemPrompt, prompt, wrapper, contextSize, grammar, jsonSchemaGrammarFile, threads, temperature, topK,
-                topP, gpuLayers, lastTokensRepeatPenalty, repeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty,
-                repeatPresencePenalty, maxTokens, noHistory, printTimings
+                model, systemInfo, systemPrompt, systemPromptFile, prompt, promptFile, wrapper, contextSize, batchSize,
+                grammar, jsonSchemaGrammarFile, threads, temperature, minP, topK, topP, gpuLayers, lastTokensRepeatPenalty,
+                repeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, maxTokens,
+                noHistory, environmentFunctions, noInfoLog, printTimings
             });
         }
         catch (err) {
@@ -177,41 +226,81 @@ export const ChatCommand = {
         }
     }
 };
-async function RunChat({ model: modelArg, systemInfo, systemPrompt, prompt, wrapper, contextSize, grammar: grammarArg, jsonSchemaGrammarFile: jsonSchemaGrammarFilePath, threads, temperature, topK, topP, gpuLayers, lastTokensRepeatPenalty, repeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, maxTokens, noHistory, printTimings }) {
-    const { LlamaChatSession } = await import("../../llamaEvaluator/LlamaChatSession.js");
-    const { LlamaModel } = await import("../../llamaEvaluator/LlamaModel.js");
-    const { LlamaContext } = await import("../../llamaEvaluator/LlamaContext.js");
-    const { LlamaGrammar } = await import("../../llamaEvaluator/LlamaGrammar.js");
-    const { LlamaJsonSchemaGrammar } = await import("../../llamaEvaluator/LlamaJsonSchemaGrammar.js");
+async function RunChat({ model: modelArg, systemInfo, systemPrompt, systemPromptFile, prompt, promptFile, wrapper, contextSize, batchSize, grammar: grammarArg, jsonSchemaGrammarFile: jsonSchemaGrammarFilePath, threads, temperature, minP, topK, topP, gpuLayers, lastTokensRepeatPenalty, repeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, maxTokens, noHistory, environmentFunctions, noInfoLog, printTimings }) {
+    if (noInfoLog)
+        console.info(`${chalk.yellow("Log level:")} warn`);
+    const llama = await getLlama("lastBuild", {
+        logLevel: noInfoLog
+            ? LlamaLogLevel.warn
+            : LlamaLogLevel.debug
+    });
+    const logBatchSize = batchSize != null;
+    if (systemInfo)
+        console.log(llama.systemInfo);
+    if (systemPromptFile != null && systemPromptFile !== "") {
+        if (systemPrompt != null && systemPrompt !== "" && systemPrompt !== defaultChatSystemPrompt)
+            console.warn(chalk.yellow("Both `systemPrompt` and `systemPromptFile` were specified. `systemPromptFile` will be used."));
+        systemPrompt = await fs.readFile(path.resolve(process.cwd(), systemPromptFile), "utf8");
+    }
+    if (promptFile != null && promptFile !== "") {
+        if (prompt != null && prompt !== "")
+            console.warn(chalk.yellow("Both `prompt` and `promptFile` were specified. `promptFile` will be used."));
+        prompt = await fs.readFile(path.resolve(process.cwd(), promptFile), "utf8");
+    }
+    if (batchSize == null)
+        batchSize = contextSize;
+    else if (batchSize > contextSize) {
+        console.warn(chalk.yellow("Batch size is greater than the context size. Batch size will be set to the context size."));
+        batchSize = contextSize;
+    }
     let initialPrompt = prompt ?? null;
-    const model = new LlamaModel({
+    const model = await withStatusLogs({
+        loading: chalk.blue("Loading model"),
+        success: chalk.blue("Model loaded"),
+        fail: chalk.blue("Failed to load model")
+    }, async () => new LlamaModel({
+        llama,
         modelPath: path.resolve(process.cwd(), modelArg),
         gpuLayers: gpuLayers != null ? gpuLayers : undefined
-    });
-    const context = new LlamaContext({
+    }));
+    const context = await withStatusLogs({
+        loading: chalk.blue("Creating context"),
+        success: chalk.blue("Context created"),
+        fail: chalk.blue("Failed to create context")
+    }, async () => new LlamaContext({
         model,
         contextSize,
+        batchSize,
         threads
-    });
+    }));
     const grammar = jsonSchemaGrammarFilePath != null
-        ? new LlamaJsonSchemaGrammar(await fs.readJson(path.resolve(process.cwd(), jsonSchemaGrammarFilePath)))
+        ? new LlamaJsonSchemaGrammar(llama, await fs.readJson(path.resolve(process.cwd(), jsonSchemaGrammarFilePath)))
         : grammarArg !== "text"
-            ? await LlamaGrammar.getFor(grammarArg)
+            ? await LlamaGrammar.getFor(llama, grammarArg)
             : undefined;
-    const bos = context.getBosString(); // bos = beginning of sequence
-    const eos = context.getEosString(); // eos = end of sequence
-    const promptWrapper = getChatWrapper(wrapper, bos);
+    const bos = model.tokens.bosString; // bos = beginning of sequence
+    const eos = model.tokens.bosString; // eos = end of sequence
+    const chatWrapper = getChatWrapper(wrapper, {
+        bosString: bos,
+        filename: model.filename,
+        typeDescription: model.typeDescription
+    });
     const session = new LlamaChatSession({
-        context,
-        printLLamaSystemInfo: systemInfo,
+        contextSequence: context.getSequence(),
         systemPrompt,
-        promptWrapper
+        chatWrapper: chatWrapper
     });
+    await new Promise((accept) => setTimeout(accept, 0)); // wait for logs to finish printing
     if (grammarArg != "text" && jsonSchemaGrammarFilePath != null)
         console.warn(chalk.yellow("Both `grammar` and `jsonSchemaGrammarFile` were specified. `jsonSchemaGrammarFile` will be used."));
+    console.info(`${chalk.yellow("Context size:")} ${context.contextSize}`);
+    if (logBatchSize)
+        console.info(`${chalk.yellow("Batch size:")} ${context.batchSize}`);
+    console.info(`${chalk.yellow("Train context size:")} ${model.trainContextSize}`);
+    console.info(`${chalk.yellow("Model type:")} ${model.typeDescription}`);
     console.info(`${chalk.yellow("BOS:")} ${bos}`);
     console.info(`${chalk.yellow("EOS:")} ${eos}`);
-    console.info(`${chalk.yellow("Chat wrapper:")} ${promptWrapper.wrapperName}`);
+    console.info(`${chalk.yellow("Chat wrapper:")} ${chatWrapper.wrapperName}`);
     console.info(`${chalk.yellow("Repeat penalty:")} ${repeatPenalty} (apply to last ${lastTokensRepeatPenalty} tokens)`);
     if (repeatFrequencyPenalty != null)
         console.info(`${chalk.yellow("Repeat frequency penalty:")} ${repeatFrequencyPenalty}`);
@@ -223,13 +312,10 @@ async function RunChat({ model: modelArg, systemInfo, systemPrompt, prompt, wrap
         console.info(`${chalk.yellow("JSON schema grammar file:")} ${path.relative(process.cwd(), path.resolve(process.cwd(), jsonSchemaGrammarFilePath))}`);
     else if (grammarArg !== "text")
         console.info(`${chalk.yellow("Grammar:")} ${grammarArg}`);
-    await withOra({
-        loading: chalk.blue("Loading model"),
-        success: chalk.blue("Model loaded"),
-        fail: chalk.blue("Failed to load model")
-    }, async () => {
-        await session.init();
-    });
+    if (environmentFunctions && grammar != null) {
+        console.warn(chalk.yellow("Environment functions are disabled since a grammar is already specified"));
+        environmentFunctions = false;
+    }
     // this is for ora to not interfere with readline
     await new Promise(resolve => setTimeout(resolve, 1));
     const replHistory = await ReplHistory.load(chatCommandHistoryFilePath, !noHistory);
@@ -260,8 +346,9 @@ async function RunChat({ model: modelArg, systemInfo, systemPrompt, prompt, wrap
         const [startColor, endColor] = chalk.blue("MIDDLE").split("MIDDLE");
         process.stdout.write(startColor);
         await session.prompt(input, {
-            grammar,
+            grammar: grammar,
             temperature,
+            minP,
             topK,
             topP,
             repeatPenalty: {
@@ -272,37 +359,62 @@ async function RunChat({ model: modelArg, systemInfo, systemPrompt, prompt, wrap
                 lastTokens: lastTokensRepeatPenalty
             },
             maxTokens: maxTokens === -1
-                ? context.getContextSize()
+                ? context.contextSize
                 : maxTokens <= 0
                     ? undefined
                     : maxTokens,
             onToken(chunk) {
-                process.stdout.write(session.context.decode(chunk));
-            }
+                process.stdout.write(model.detokenize(chunk));
+            },
+            functions: (grammar == null && environmentFunctions)
+                ? defaultEnvironmentFunctions
+                : undefined
         });
         process.stdout.write(endColor);
         console.log();
         if (printTimings)
-            context.printTimings();
+            await context.printTimings();
     }
 }
-function getChatWrapper(wrapper, bos) {
+const defaultEnvironmentFunctions = {
+    getDate: defineChatSessionFunction({
+        description: "Retrieve the current date",
+        handler() {
+            return new Date().toLocaleDateString();
+        }
+    }),
+    getTime: defineChatSessionFunction({
+        description: "Retrieve the current time",
+        handler() {
+            return new Date().toLocaleTimeString();
+        }
+    })
+};
+function getChatWrapper(wrapper, { bosString, filename, typeDescription }) {
     switch (wrapper) {
         case "general":
-            return new GeneralChatPromptWrapper();
+            return new GeneralChatWrapper();
         case "llamaChat":
-            return new LlamaChatPromptWrapper();
+            return new LlamaChatWrapper();
+        case "alpacaChat":
+            return new AlpacaChatWrapper();
+        case "functionary":
+            return new FunctionaryChatWrapper();
         case "chatML":
-            return new ChatMLChatPromptWrapper();
+            return new ChatMLChatWrapper();
         case "falconChat":
-            return new FalconChatPromptWrapper();
+            return new FalconChatWrapper();
         default:
     }
     if (wrapper === "auto") {
-        const chatWrapper = getChatWrapperByBos(bos);
+        const chatWrapper = resolveChatWrapperBasedOnModel({
+            bosString,
+            filename,
+            typeDescription
+        });
         if (chatWrapper != null)
             return new chatWrapper();
-        return new GeneralChatPromptWrapper();
+        return new GeneralChatWrapper();
     }
     void (wrapper);
     throw new Error("Unknown wrapper: " + wrapper);