node-llama-cpp 3.0.0-beta.10 → 3.0.0-beta.12

package/dist/cli/commands/InfillCommand.d.ts

@@ -0,0 +1,27 @@
+import { CommandModule } from "yargs";
+type InfillCommand = {
+    model: string;
+    systemInfo: boolean;
+    prefix?: string;
+    prefixFile?: string;
+    suffix?: string;
+    suffixFile?: string;
+    contextSize: number;
+    batchSize?: number;
+    threads: number;
+    temperature: number;
+    minP: number;
+    topK: number;
+    topP: number;
+    gpuLayers?: number;
+    repeatPenalty: number;
+    lastTokensRepeatPenalty: number;
+    penalizeRepeatingNewLine: boolean;
+    repeatFrequencyPenalty?: number;
+    repeatPresencePenalty?: number;
+    maxTokens: number;
+    noInfoLog: boolean;
+    printTimings: boolean;
+};
+export declare const InfillCommand: CommandModule<object, InfillCommand>;
+export {};

package/dist/cli/commands/InfillCommand.js

@@ -0,0 +1,316 @@
+import * as readline from "readline";
+import process from "process";
+import path from "path";
+import chalk from "chalk";
+import fs from "fs-extra";
+import withStatusLogs from "../../utils/withStatusLogs.js";
+import { getLlama } from "../../bindings/getLlama.js";
+import { LlamaModel } from "../../evaluator/LlamaModel.js";
+import { LlamaContext } from "../../evaluator/LlamaContext/LlamaContext.js";
+import { LlamaLogLevel } from "../../bindings/types.js";
+import { LlamaCompletion } from "../../evaluator/LlamaCompletion.js";
+export const InfillCommand = {
+    command: "infill",
+    describe: "Generate an infill completion for a given suffix and prefix texts",
+    builder(yargs) {
+        return yargs
+            .option("model", {
+            alias: "m",
+            type: "string",
+            demandOption: true,
+            description: "Llama model file to use for the chat",
+            group: "Required:"
+        })
+            .option("systemInfo", {
+            alias: "i",
+            type: "boolean",
+            default: false,
+            description: "Print llama.cpp system info",
+            group: "Optional:"
+        })
+            .option("prefix", {
+            type: "string",
+            description: "First prefix text to automatically load",
+            group: "Optional:"
+        })
+            .option("prefixFile", {
+            type: "string",
+            description: "Path to a file to load prefix text from automatically",
+            group: "Optional:"
+        })
+            .option("suffix", {
+            type: "string",
+            description: "First suffix text to automatically load. Requires `prefix` or `prefixFile` to be set",
+            group: "Optional:"
+        })
+            .option("suffixFile", {
+            type: "string",
+            description: "Path to a file to load suffix text from automatically. Requires `prefix` or `prefixFile` to be set",
+            group: "Optional:"
+        })
+            .option("contextSize", {
+            alias: "c",
+            type: "number",
+            default: 1024 * 4,
+            description: "Context size to use for the model context",
+            group: "Optional:"
+        })
+            .option("batchSize", {
+            alias: "b",
+            type: "number",
+            description: "Batch size to use for the model context. The default value is the context size",
+            group: "Optional:"
+        })
+            .option("threads", {
+            type: "number",
+            default: 6,
+            description: "Number of threads to use for the evaluation of tokens",
+            group: "Optional:"
+        })
+            .option("temperature", {
+            alias: "t",
+            type: "number",
+            default: 0,
+            description: "Temperature is a hyperparameter that controls the randomness of the generated text. It affects the probability distribution of the model's output tokens. A higher temperature (e.g., 1.5) makes the output more random and creative, while a lower temperature (e.g., 0.5) makes the output more focused, deterministic, and conservative. The suggested temperature is 0.8, which provides a balance between randomness and determinism. At the extreme, a temperature of 0 will always pick the most likely next token, leading to identical outputs in each run. Set to `0` to disable.",
+            group: "Optional:"
+        })
+            .option("minP", {
+            alias: "mp",
+            type: "number",
+            default: 0,
+            description: "From the next token candidates, discard the percentage of tokens with the lowest probability. For example, if set to `0.05`, 5% of the lowest probability tokens will be discarded. This is useful for generating more high-quality results when using a high temperature. Set to a value between `0` and `1` to enable. Only relevant when `temperature` is set to a value greater than `0`.",
+            group: "Optional:"
+        })
+            .option("topK", {
+            alias: "k",
+            type: "number",
+            default: 40,
+            description: "Limits the model to consider only the K most likely next tokens for sampling at each step of sequence generation. An integer number between `1` and the size of the vocabulary. Set to `0` to disable (which uses the full vocabulary). Only relevant when `temperature` is set to a value greater than 0.",
+            group: "Optional:"
+        })
+            .option("topP", {
+            alias: "p",
+            type: "number",
+            default: 0.95,
+            description: "Dynamically selects the smallest set of tokens whose cumulative probability exceeds the threshold P, and samples the next token only from this set. A float number between `0` and `1`. Set to `1` to disable. Only relevant when `temperature` is set to a value greater than `0`.",
+            group: "Optional:"
+        })
+            .option("gpuLayers", {
+            alias: "gl",
+            type: "number",
+            description: "number of layers to store in VRAM",
+            group: "Optional:"
+        })
+            .option("repeatPenalty", {
+            alias: "rp",
+            type: "number",
+            default: 1.1,
+            description: "Prevent the model from repeating the same token too much. Set to `1` to disable.",
+            group: "Optional:"
+        })
+            .option("lastTokensRepeatPenalty", {
+            alias: "rpn",
+            type: "number",
+            default: 64,
+            description: "Number of recent tokens generated by the model to apply penalties to repetition of",
+            group: "Optional:"
+        })
+            .option("penalizeRepeatingNewLine", {
+            alias: "rpnl",
+            type: "boolean",
+            default: true,
+            description: "Penalize new line tokens. set \"--no-penalizeRepeatingNewLine\" or \"--no-rpnl\" to disable",
+            group: "Optional:"
+        })
+            .option("repeatFrequencyPenalty", {
+            alias: "rfp",
+            type: "number",
+            description: "For n time a token is in the `punishTokens` array, lower its probability by `n * repeatFrequencyPenalty`. Set to a value between `0` and `1` to enable.",
+            group: "Optional:"
+        })
+            .option("repeatPresencePenalty", {
+            alias: "rpp",
+            type: "number",
+            description: "Lower the probability of all the tokens in the `punishTokens` array by `repeatPresencePenalty`. Set to a value between `0` and `1` to enable.",
+            group: "Optional:"
+        })
+            .option("maxTokens", {
+            alias: "mt",
+            type: "number",
+            default: 0,
+            description: "Maximum number of tokens to generate in responses. Set to `0` to disable. Set to `-1` to set to the context size",
+            group: "Optional:"
+        })
+            .option("noInfoLog", {
+            alias: "nl",
+            type: "boolean",
+            default: false,
+            description: "Disable llama.cpp info logs",
+            group: "Optional:"
+        })
+            .option("printTimings", {
+            alias: "pt",
+            type: "boolean",
+            default: false,
+            description: "Print llama.cpp timings after each response",
+            group: "Optional:"
+        });
+    },
+    async handler({ model, systemInfo, prefix, prefixFile, suffix, suffixFile, contextSize, batchSize, threads, temperature, minP, topK, topP, gpuLayers, repeatPenalty, lastTokensRepeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, maxTokens, noInfoLog, printTimings }) {
+        try {
+            await RunInfill({
+                model, systemInfo, prefix, prefixFile, suffix, suffixFile, contextSize, batchSize,
+                threads, temperature, minP, topK, topP, gpuLayers, lastTokensRepeatPenalty,
+                repeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, maxTokens,
+                noInfoLog, printTimings
+            });
+        }
+        catch (err) {
+            console.error(err);
+            process.exit(1);
+        }
+    }
+};
+async function RunInfill({ model: modelArg, systemInfo, prefix, prefixFile, suffix, suffixFile, contextSize, batchSize, threads, temperature, minP, topK, topP, gpuLayers, lastTokensRepeatPenalty, repeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, maxTokens, noInfoLog, printTimings }) {
+    if (noInfoLog)
+        console.info(`${chalk.yellow("Log level:")} warn`);
+    const llama = await getLlama("lastBuild", {
+        logLevel: noInfoLog
+            ? LlamaLogLevel.warn
+            : LlamaLogLevel.debug
+    });
+    const logBatchSize = batchSize != null;
+    if (systemInfo)
+        console.log(llama.systemInfo);
+    if (prefixFile != null && prefixFile !== "") {
+        if (prefix != null && prefix !== "")
+            console.warn(chalk.yellow("Both `prefix` and `prefixFile` were specified. `prefixFile` will be used."));
+        prefix = await fs.readFile(path.resolve(process.cwd(), prefixFile), "utf8");
+    }
+    if (suffixFile != null && suffixFile !== "") {
+        if (suffix != null && suffix !== "")
+            console.warn(chalk.yellow("Both `suffix` and `suffixFile` were specified. `suffixFile` will be used."));
+        suffix = await fs.readFile(path.resolve(process.cwd(), suffixFile), "utf8");
+    }
+    if (suffix != null && prefix == null) {
+        console.warn(chalk.yellow("Suffix was specified but no prefix was specified. Suffix will be ignored."));
+        suffix = undefined;
+    }
+    if (batchSize == null)
+        batchSize = contextSize;
+    else if (batchSize > contextSize) {
+        console.warn(chalk.yellow("Batch size is greater than the context size. Batch size will be set to the context size."));
+        batchSize = contextSize;
+    }
+    let initialPrefix = prefix ?? null;
+    let initialSuffix = suffix ?? null;
+    const model = await withStatusLogs({
+        loading: chalk.blue("Loading model"),
+        success: chalk.blue("Model loaded"),
+        fail: chalk.blue("Failed to load model")
+    }, async () => new LlamaModel({
+        llama,
+        modelPath: path.resolve(process.cwd(), modelArg),
+        gpuLayers: gpuLayers != null ? gpuLayers : undefined
+    }));
+    const context = await withStatusLogs({
+        loading: chalk.blue("Creating context"),
+        success: chalk.blue("Context created"),
+        fail: chalk.blue("Failed to create context")
+    }, async () => new LlamaContext({
+        model,
+        contextSize,
+        batchSize,
+        threads
+    }));
+    const completion = new LlamaCompletion({
+        contextSequence: context.getSequence()
+    });
+    await new Promise((accept) => setTimeout(accept, 0)); // wait for logs to finish printing
+    console.info(`${chalk.yellow("Context size:")} ${context.contextSize}`);
+    if (logBatchSize)
+        console.info(`${chalk.yellow("Batch size:")} ${context.batchSize}`);
+    console.info(`${chalk.yellow("Train context size:")} ${model.trainContextSize}`);
+    console.info(`${chalk.yellow("Model type:")} ${model.typeDescription}`);
+    console.info(`${chalk.yellow("Repeat penalty:")} ${repeatPenalty} (apply to last ${lastTokensRepeatPenalty} tokens)`);
+    if (repeatFrequencyPenalty != null)
+        console.info(`${chalk.yellow("Repeat frequency penalty:")} ${repeatFrequencyPenalty}`);
+    if (repeatPresencePenalty != null)
+        console.info(`${chalk.yellow("Repeat presence penalty:")} ${repeatPresencePenalty}`);
+    if (!penalizeRepeatingNewLine)
+        console.info(`${chalk.yellow("Penalize repeating new line:")} disabled`);
+    // this is for ora to not interfere with readline
+    await new Promise(resolve => setTimeout(resolve, 1));
+    if (!completion.infillSupported) {
+        console.log(chalk.red("Infill is not supported for this model"));
+        process.exit(1);
+    }
+    const replPrefixHistory = [];
+    const replSuffixHistory = [];
+    async function getInput(name) {
+        const rl = readline.createInterface({
+            input: process.stdin,
+            output: process.stdout,
+            history: name === "Prefix"
+                ? replPrefixHistory.slice()
+                : replSuffixHistory.slice()
+        });
+        const res = await new Promise((accept) => rl.question(chalk.yellow(name + "> "), accept));
+        rl.close();
+        return res;
+    }
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        const prefixInput = initialPrefix != null
+            ? initialPrefix
+            : await getInput("Prefix");
+        if (initialPrefix != null) {
+            console.log(chalk.green("Prefix> ") + initialPrefix);
+            initialPrefix = null;
+        }
+        else
+            await replPrefixHistory.push(prefixInput);
+        if (prefixInput === ".exit")
+            break;
+        const suffixInput = initialSuffix != null
+            ? initialSuffix
+            : await getInput("Suffix");
+        if (initialSuffix != null) {
+            console.log(chalk.green("Suffix> ") + initialSuffix);
+            initialSuffix = null;
+        }
+        else
+            await replSuffixHistory.push(suffixInput);
+        if (suffixInput === ".exit")
+            break;
+        process.stdout.write(chalk.yellow("Infill: "));
+        const [startColor, endColor] = chalk.blue("MIDDLE").split("MIDDLE");
+        process.stdout.write(startColor);
+        await completion.generateInfillCompletion(prefixInput, suffixInput, {
+            temperature,
+            minP,
+            topK,
+            topP,
+            repeatPenalty: {
+                penalty: repeatPenalty,
+                frequencyPenalty: repeatFrequencyPenalty != null ? repeatFrequencyPenalty : undefined,
+                presencePenalty: repeatPresencePenalty != null ? repeatPresencePenalty : undefined,
+                penalizeNewLine: penalizeRepeatingNewLine,
+                lastTokens: lastTokensRepeatPenalty
+            },
+            maxTokens: maxTokens === -1
+                ? context.contextSize
+                : maxTokens <= 0
+                    ? undefined
+                    : maxTokens,
+            onToken(chunk) {
+                process.stdout.write(model.detokenize(chunk));
+            }
+        });
+        process.stdout.write(endColor);
+        console.log();
+        if (printTimings)
+            await context.printTimings();
+    }
+}
+//# sourceMappingURL=InfillCommand.js.map

package/dist/cli/commands/InfillCommand.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"InfillCommand.js","sourceRoot":"","sources":["../../../src/cli/commands/InfillCommand.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AACrC,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,cAAc,MAAM,+BAA+B,CAAC;AAC3D,OAAO,EAAC,QAAQ,EAAC,MAAM,4BAA4B,CAAC;AACpD,OAAO,EAAC,UAAU,EAAC,MAAM,+BAA+B,CAAC;AACzD,OAAO,EAAC,YAAY,EAAC,MAAM,8CAA8C,CAAC;AAC1E,OAAO,EAAC,aAAa,EAAC,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAC,eAAe,EAAC,MAAM,oCAAoC,CAAC;AA2BnE,MAAM,CAAC,MAAM,aAAa,GAAyC;IAC/D,OAAO,EAAE,QAAQ;IACjB,QAAQ,EAAE,mEAAmE;IAC7E,OAAO,CAAC,KAAK;QACT,OAAO,KAAK;aACP,MAAM,CAAC,OAAO,EAAE;YACb,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,YAAY,EAAE,IAAI;YAClB,WAAW,EAAE,sCAAsC;YACnD,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,YAAY,EAAE;YAClB,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,KAAK;YACd,WAAW,EAAE,6BAA6B;YAC1C,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,QAAQ,EAAE;YACd,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,yCAAyC;YACtD,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,YAAY,EAAE;YAClB,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,uDAAuD;YACpE,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,QAAQ,EAAE;YACd,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,sFAAsF;YACnG,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,YAAY,EAAE;YAClB,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,oGAAoG;YACjH,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,aAAa,EAAE;YACnB,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,IAAI,GAAG,CAAC;YACjB,WAAW,EAAE,2CAA2C;YACxD,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,WAAW,EAAE;YACjB,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,gFAAgF;YAC7F,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,SAAS,EAAE;YACf,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,CAAC;YACV,WAAW,EAAE,uDAAuD;YACpE,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,aAAa,EAAE;YACnB,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,CAAC;YACV,WAAW,EAAE,6jBAA6jB;YAC1kB,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,MAAM,EAAE;YACZ,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,CAAC;YACV,WAAW,EAAE,+XAA+X;YAC5Y,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,MAAM,EAAE;YACZ,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,EAAE;YACX,WAAW,EAAE,4SAA4S;YACzT,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,MAAM,EAAE;YACZ,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,IAAI;YACb,WAAW,EAAE,qRAAqR;YAClS,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,WAAW,EAAE;YACjB,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,mCAAmC;YAChD,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,eAAe,EAAE;YACrB,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,GAAG;YACZ,WAAW,EAAE,kFAAkF;YAC/F,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,yBAAyB,EAAE;YAC/B,KAAK,EAAE,KAAK;YACZ,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,EAAE;YACX,WAAW,EAAE,oFAAoF;YACjG,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,0BAA0B,EAAE;YAChC,KAAK,EAAE,MAAM;YACb,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,IAAI;YACb,WAAW,EAAE,6FAA6F;YAC1G,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,wBAAwB,EAAE;YAC9B,KAAK,EAAE,KAAK;YACZ,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,yJAAyJ;YACtK,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,uBAAuB,EAAE;YAC7B,KAAK,EAAE,KAAK;YACZ,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,+IAA+I;YAC5J,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,WAAW,EAAE;YACjB,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,CAAC;YACV,WAAW,EAAE,kHAAkH;YAC/H,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,WAAW,EAAE;YACjB,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,KAAK;YACd,WAAW,EAAE,6BAA6B;YAC1C,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,cAAc,EAAE;YACpB,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,KAAK;YACd,WAAW,EAAE,6CAA6C;YAC1D,KAAK,EAAE,WAAW;SACrB,CAAC,CAAC;IACX,CAAC;IACD,KAAK,CAAC,OAAO,CAAC,EACV,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EACjF,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAChC,IAAI,EAAE,SAAS,EAAE,aAAa,EAAE,uBAAuB,EAAE,wBAAwB,EACjF,sBAAsB,EAAE,qBAAqB,EAAE,SAAS,EACxD,SAAS,EAAE,YAAY,EAC1B;QACG,IAAI;YACA,MAAM,SAAS,CAAC;gBACZ,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS;gBACjF,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,uBAAuB;gBAC1E,aAAa,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,qBAAqB,EAAE,SAAS;gBACjG,SAAS,EAAE,YAAY;aAC1B,CAAC,CAAC;SACN;QAAC,OAAO,GAAG,EAAE;YACV,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACnB;IACL,CAAC;CACJ,CAAC;AAGF,KAAK,UAAU,SAAS,CAAC,EACrB,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAC3F,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EACjD,uBAAuB,EAAE,aAAa,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,qBAAqB,EAC/G,SAAS,EAAE,SAAS,EAAE,YAAY,EACtB;IACZ,IAAI,SAAS;QACT,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;IAEvD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE;QACtC,QAAQ,EAAE,SAAS;YACf,CAAC,CAAC,aAAa,CAAC,IAAI;YACpB,CAAC,CAAC,aAAa,CAAC,KAAK;KAC5B,CAAC,CAAC;IACH,MAAM,YAAY,GAAG,SAAS,IAAI,IAAI,CAAC;IAEvC,IAAI,UAAU;QACV,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAElC,IAAI,UAAU,IAAI,IAAI,IAAI,UAAU,KAAK,EAAE,EAAE;QACzC,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,KAAK,EAAE;YAC/B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,2EAA2E,CAAC,CAAC,CAAC;QAE5G,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,UAAU,CAAC,EAAE,MAAM,CAAC,CAAC;KAC/E;IAED,IAAI,UAAU,IAAI,IAAI,IAAI,UAAU,KAAK,EAAE,EAAE;QACzC,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,KAAK,EAAE;YAC/B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,2EAA2E,CAAC,CAAC,CAAC;QAE5G,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,UAAU,CAAC,EAAE,MAAM,CAAC,CAAC;KAC/E;IAED,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,IAAI,EAAE;QAClC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,2EAA2E,CAAC,CAAC,CAAC;QACxG,MAAM,GAAG,SAAS,CAAC;KACtB;IAED,IAAI,SAAS,IAAI,IAAI;QACjB,SAAS,GAAG,WAAW,CAAC;SACvB,IAAI,SAAS,GAAG,WAAW,EAAE;QAC9B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,0FAA0F,CAAC,CAAC,CAAC;QACvH,SAAS,GAAG,WAAW,CAAC;KAC3B;IAED,IAAI,aAAa,GAAG,MAAM,IAAI,IAAI,CAAC;IACnC,IAAI,aAAa,GAAG,MAAM,IAAI,IAAI,CAAC;IAEnC,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC;QAC/B,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC;QACpC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC;QACnC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC;KAC3C,EAAE,KAAK,IAAI,EAAE,CAAC,IAAI,UAAU,CAAC;QAC1B,KAAK;QACL,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,QAAQ,CAAC;QAChD,SAAS,EAAE,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;KACvD,CAAC,CAAC,CAAC;IACJ,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC;QACjC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC;QACvC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC;QACtC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC;KAC/C,EAAE,KAAK,IAAI,EAAE,CAAC,IAAI,YAAY,CAAC;QAC5B,KAAK;QACL,WAAW;QACX,SAAS;QACT,OAAO;KACV,CAAC,CAAC,CAAC;IAEJ,MAAM,UAAU,GAAG,IAAI,eAAe,CAAC;QACnC,eAAe,EAAE,OAAO,CAAC,WAAW,EAAE;KACzC,CAAC,CAAC;IAEH,MAAM,IAAI,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,mCAAmC;IAEzF,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IAExE,IAAI,YAAY;QACZ,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IAExE,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,qBAAqB,CAAC,IAAI,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC;IACjF,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC;IACxE,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI,aAAa,mBAAmB,uBAAuB,UAAU,CAAC,CAAC;IAEtH,IAAI,sBAAsB,IAAI,IAAI;QAC9B,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,2BAA2B,CAAC,IAAI,sBAAsB,EAAE,CAAC,CAAC;IAE3F,IAAI,qBAAqB,IAAI,IAAI;QAC7B,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,0BAA0B,CAAC,IAAI,qBAAqB,EAAE,CAAC,CAAC;IAEzF,IAAI,CAAC,wBAAwB;QACzB,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,8BAA8B,CAAC,WAAW,CAAC,CAAC;IAE7E,iDAAiD;IACjD,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;IAErD,IAAI,CAAC,UAAU,CAAC,eAAe,EAAE;QAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC,CAAC;QACjE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACnB;IAED,MAAM,iBAAiB,GAAa,EAAE,CAAC;IACvC,MAAM,iBAAiB,GAAa,EAAE,CAAC;IAEvC,KAAK,UAAU,QAAQ,CAAC,IAAyB;QAC7C,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;YAChC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,OAAO,EAAE,IAAI,KAAK,QAAQ;gBACtB,CAAC,CAAC,iBAAiB,CAAC,KAAK,EAAE;gBAC3B,CAAC,CAAC,iBAAiB,CAAC,KAAK,EAAE;SAClC,CAAC,CAAC;QAEH,MAAM,GAAG,GAAW,MAAM,IAAI,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;QAClG,EAAE,CAAC,KAAK,EAAE,CAAC;QAEX,OAAO,GAAG,CAAC;IACf,CAAC;IAED,iDAAiD;IACjD,OAAO,IAAI,EAAE;QACT,MAAM,WAAW,GAAG,aAAa,IAAI,IAAI;YACrC,CAAC,CAAC,aAAa;YACf,CAAC,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAE/B,IAAI,aAAa,IAAI,IAAI,EAAE;YACvB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,CAAC;YACrD,aAAa,GAAG,IAAI,CAAC;SACxB;;YACG,MAAM,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAE9C,IAAI,WAAW,KAAK,OAAO;YACvB,MAAM;QAEV,MAAM,WAAW,GAAG,aAAa,IAAI,IAAI;YACrC,CAAC,CAAC,aAAa;YACf,CAAC,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAE/B,IAAI,aAAa,IAAI,IAAI,EAAE;YACvB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,CAAC;YACrD,aAAa,GAAG,IAAI,CAAC;SACxB;;YACG,MAAM,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAE9C,IAAI,WAAW,KAAK,OAAO;YACvB,MAAM;QAEV,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;QAE/C,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAEpE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACjC,MAAM,UAAU,CAAC,wBAAwB,CAAC,WAAW,EAAE,WAAW,EAAE;YAChE,WAAW;YACX,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,aAAa,EAAE;gBACX,OAAO,EAAE,aAAa;gBACtB,gBAAgB,EAAE,sBAAsB,IAAI,IAAI,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,SAAS;gBACrF,eAAe,EAAE,qBAAqB,IAAI,IAAI,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,SAAS;gBAClF,eAAe,EAAE,wBAAwB;gBACzC,UAAU,EAAE,uBAAuB;aACtC;YACD,SAAS,EAAE,SAAS,KAAK,CAAC,CAAC;gBACvB,CAAC,CAAC,OAAO,CAAC,WAAW;gBACrB,CAAC,CAAC,SAAS,IAAI,CAAC;oBACZ,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,SAAS;YACnB,OAAO,CAAC,KAAK;gBACT,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;YAClD,CAAC;SACJ,CAAC,CAAC;QACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC/B,OAAO,CAAC,GAAG,EAAE,CAAC;QAEd,IAAI,YAAY;YACZ,MAAM,OAAO,CAAC,YAAY,EAAE,CAAC;KACpC;AACL,CAAC"}

package/dist/cli/utils/logEnabledComputeLayers.d.ts

@@ -0,0 +1,8 @@
+import { BinaryPlatform } from "../../bindings/utils/getPlatform.js";
+export declare function logEnabledComputeLayers({ metal, cuda, vulkan }: {
+    metal: boolean;
+    cuda: boolean;
+    vulkan: boolean;
+}, { platform }?: {
+    platform?: BinaryPlatform;
+}): void;

package/dist/cli/utils/logEnabledComputeLayers.js

@@ -0,0 +1,11 @@
+import chalk from "chalk";
+import { getPlatform } from "../../bindings/utils/getPlatform.js";
+export function logEnabledComputeLayers({ metal, cuda, vulkan }, { platform = getPlatform() } = {}) {
+    if (metal && platform === "mac")
+        console.log(`${chalk.yellow("Metal:")} enabled`);
+    if (cuda)
+        console.log(`${chalk.yellow("CUDA:")} enabled`);
+    if (vulkan)
+        console.log(`${chalk.yellow("Vulkan:")} enabled`);
+}
+//# sourceMappingURL=logEnabledComputeLayers.js.map

package/dist/cli/utils/logEnabledComputeLayers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"logEnabledComputeLayers.js","sourceRoot":"","sources":["../../../src/cli/utils/logEnabledComputeLayers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAiB,WAAW,EAAC,MAAM,qCAAqC,CAAC;AAEhF,MAAM,UAAU,uBAAuB,CAAC,EACpC,KAAK,EAAE,IAAI,EAAE,MAAM,EAGtB,EAAE,EACC,QAAQ,GAAG,WAAW,EAAE,KAGxB,EAAE;IACF,IAAI,KAAK,IAAI,QAAQ,KAAK,KAAK;QAC3B,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;IAErD,IAAI,IAAI;QACJ,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAEpD,IAAI,MAAM;QACN,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;AAC1D,CAAC"}

package/dist/config.d.ts

@@ -25,6 +25,7 @@ export declare const defaultLlamaCppGitHubRepo: string;
 export declare const defaultLlamaCppRelease: string;
 export declare const defaultLlamaCppMetalSupport: boolean;
 export declare const defaultLlamaCppCudaSupport: boolean;
+export declare const defaultLlamaCppVulkanSupport: boolean;
 export declare const defaultLlamaCppDebugLogs: LlamaLogLevel;
 export declare const defaultSkipDownload: boolean;
 export declare const defaultXpacksStoreDirectory: string;

package/dist/config.js

@@ -46,6 +46,9 @@ export const defaultLlamaCppMetalSupport = env.get("NODE_LLAMA_CPP_METAL")
 export const defaultLlamaCppCudaSupport = env.get("NODE_LLAMA_CPP_CUDA")
     .default("false")
     .asBool();
+export const defaultLlamaCppVulkanSupport = env.get("NODE_LLAMA_CPP_VULKAN")
+    .default("false")
+    .asBool();
 export const defaultLlamaCppDebugLogs = env.get("NODE_LLAMA_CPP_LOG_LEVEL")
     .default(LlamaLogLevel.debug)
     .asEnum(LlamaLogLevelValues);
@@ -60,8 +63,8 @@ export const defaultXpacksCacheDirectory = env.get("NODE_LLAMA_CPP_XPACKS_CACHE_
     .asString();
 export const customCmakeOptionsEnvVarPrefix = "NODE_LLAMA_CPP_CMAKE_OPTION_";
 export const defaultChatSystemPrompt = "You are a helpful, respectful and honest assistant. Always answer as helpfully as possible.\n" +
-    "If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. " +
-    "If you don't know the answer to a question, please don't share false information.";
+    "If a question does not make any sense, or is not factually coherent, explain why instead of answering something incorrectly. " +
+    "If you don't know the answer to a question, don't share false information.";
 export const cliBinName = "node-llama-cpp";
 export const npxRunPrefix = "npx --no ";
 const documentationUrl = "https://withcatai.github.io/node-llama-cpp";

package/dist/config.js.map

@@ -1 +1 @@
-{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,KAAK,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,MAAM,MAAM,SAAS,CAAC;AAC7B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAC,wBAAwB,EAAC,MAAM,2CAA2C,CAAC;AACnF,OAAO,EAAC,aAAa,EAAE,mBAAmB,EAAC,MAAM,qBAAqB,CAAC;AAEvE,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE/D,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;AAGrC,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;AAClE,MAAM,CAAC,MAAM,wBAAwB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;AAChF,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;AAClF,MAAM,CAAC,MAAM,4BAA4B,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,aAAa,CAAC,CAAC;AACrF,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;AAC1F,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;AACxE,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;AAC5F,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,gBAAgB,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;AACzF,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,mCAAmC,CAAC,CAAC;AACvG,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;AACjF,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,4BAA4B,CAAC,CAAC;AACjG,MAAM,CAAC,MAAM,6BAA6B,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,qBAAqB,CAAC,CAAC;AAC9F,MAAM,CAAC,MAAM,2BAA2B,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,mBAAmB,CAAC,CAAC;AAC1F,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;AACjE,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;AAC5E,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;AAC5E,MAAM,CAAC,MAAM,qBAAqB,GAAG,qBAAqB,CAAC;AAC3D,MAAM,CAAC,MAAM,UAAU,GAAG,SAAS,CAAC;AACpC,MAAM,CAAC,MAAM,yBAAyB,GAAG,qBAAqB,CAAC;AAC/D,MAAM,CAAC,MAAM,sBAAsB,GAAG,MAAM,wBAAwB,EAAE,CAAC;AAEvE,MAAM,CAAC,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC;KAC5B,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,0BAA0B,GAAG,GAAG,CAAC,GAAG,CAAC,mBAAmB,CAAC;KACjE,OAAO,CAAC,EAAE,CAAC;KACX,QAAQ,EAAE,KAAK,EAAE,CAAC;AACvB,MAAM,CAAC,MAAM,yBAAyB,GAAG,GAAG,CAAC,GAAG,CAAC,qBAAqB,CAAC;KAClE,OAAO,CAAC,yBAAyB,CAAC;KAClC,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC,GAAG,CAAC,6BAA6B,CAAC;KACvE,OAAO,CAAC,sBAAsB,CAAC;KAC/B,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC,GAAG,CAAC,sBAAsB,CAAC;KACrE,OAAO,CAAC,CAAC,OAAO,CAAC,QAAQ,KAAK,QAAQ,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;KACrF,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,0BAA0B,GAAG,GAAG,CAAC,GAAG,CAAC,qBAAqB,CAAC;KACnE,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,wBAAwB,GAAG,GAAG,CAAC,GAAG,CAAC,0BAA0B,CAAC;KACtE,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC;KAC5B,MAAM,CAAC,mBAAmB,CAAC,CAAC;AACjC,MAAM,CAAC,MAAM,mBAAmB,GAAG,GAAG,CAAC,GAAG,CAAC,8BAA8B,CAAC;KACrE,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC,GAAG,CAAC,oCAAoC,CAAC;KACnF,OAAO,CAAC,yBAAyB,CAAC;KAClC,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC,GAAG,CAAC,oCAAoC,CAAC;KACnF,OAAO,CAAC,yBAAyB,CAAC;KAClC,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,8BAA8B,GAAG,8BAA8B,CAAC;AAC7E,MAAM,CAAC,MAAM,uBAAuB,GAAG,+FAA+F;IAClI,+HAA+H;IAC/H,mFAAmF,CAAC;AACxF,MAAM,CAAC,MAAM,UAAU,GAAG,gBAAgB,CAAC;AAC3C,MAAM,CAAC,MAAM,YAAY,GAAG,WAAW,CAAC;AAExC,MAAM,gBAAgB,GAAG,4CAA4C,CAAC;AACtE,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACjC,IAAI,EAAE,gBAAgB,GAAG,aAAa;CAChC,CAAC"}
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,KAAK,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,MAAM,MAAM,SAAS,CAAC;AAC7B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAC,wBAAwB,EAAC,MAAM,2CAA2C,CAAC;AACnF,OAAO,EAAC,aAAa,EAAE,mBAAmB,EAAC,MAAM,qBAAqB,CAAC;AAEvE,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE/D,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;AAGrC,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;AAClE,MAAM,CAAC,MAAM,wBAAwB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;AAChF,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;AAClF,MAAM,CAAC,MAAM,4BAA4B,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,aAAa,CAAC,CAAC;AACrF,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;AAC1F,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;AACxE,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;AAC5F,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,gBAAgB,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;AACzF,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,mCAAmC,CAAC,CAAC;AACvG,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;AACjF,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,4BAA4B,CAAC,CAAC;AACjG,MAAM,CAAC,MAAM,6BAA6B,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,qBAAqB,CAAC,CAAC;AAC9F,MAAM,CAAC,MAAM,2BAA2B,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,mBAAmB,CAAC,CAAC;AAC1F,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;AACjE,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;AAC5E,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;AAC5E,MAAM,CAAC,MAAM,qBAAqB,GAAG,qBAAqB,CAAC;AAC3D,MAAM,CAAC,MAAM,UAAU,GAAG,SAAS,CAAC;AACpC,MAAM,CAAC,MAAM,yBAAyB,GAAG,qBAAqB,CAAC;AAC/D,MAAM,CAAC,MAAM,sBAAsB,GAAG,MAAM,wBAAwB,EAAE,CAAC;AAEvE,MAAM,CAAC,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC;KAC5B,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,0BAA0B,GAAG,GAAG,CAAC,GAAG,CAAC,mBAAmB,CAAC;KACjE,OAAO,CAAC,EAAE,CAAC;KACX,QAAQ,EAAE,KAAK,EAAE,CAAC;AACvB,MAAM,CAAC,MAAM,yBAAyB,GAAG,GAAG,CAAC,GAAG,CAAC,qBAAqB,CAAC;KAClE,OAAO,CAAC,yBAAyB,CAAC;KAClC,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC,GAAG,CAAC,6BAA6B,CAAC;KACvE,OAAO,CAAC,sBAAsB,CAAC;KAC/B,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC,GAAG,CAAC,sBAAsB,CAAC;KACrE,OAAO,CAAC,CAAC,OAAO,CAAC,QAAQ,KAAK,QAAQ,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;KACrF,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,0BAA0B,GAAG,GAAG,CAAC,GAAG,CAAC,qBAAqB,CAAC;KACnE,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,4BAA4B,GAAG,GAAG,CAAC,GAAG,CAAC,uBAAuB,CAAC;KACvE,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,wBAAwB,GAAG,GAAG,CAAC,GAAG,CAAC,0BAA0B,CAAC;KACtE,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC;KAC5B,MAAM,CAAC,mBAAmB,CAAC,CAAC;AACjC,MAAM,CAAC,MAAM,mBAAmB,GAAG,GAAG,CAAC,GAAG,CAAC,8BAA8B,CAAC;KACrE,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC,GAAG,CAAC,oCAAoC,CAAC;KACnF,OAAO,CAAC,yBAAyB,CAAC;KAClC,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC,GAAG,CAAC,oCAAoC,CAAC;KACnF,OAAO,CAAC,yBAAyB,CAAC;KAClC,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,8BAA8B,GAAG,8BAA8B,CAAC;AAC7E,MAAM,CAAC,MAAM,uBAAuB,GAAG,+FAA+F;IAClI,+HAA+H;IAC/H,4EAA4E,CAAC;AACjF,MAAM,CAAC,MAAM,UAAU,GAAG,gBAAgB,CAAC;AAC3C,MAAM,CAAC,MAAM,YAAY,GAAG,WAAW,CAAC;AAExC,MAAM,gBAAgB,GAAG,4CAA4C,CAAC;AACtE,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACjC,IAAI,EAAE,gBAAgB,GAAG,aAAa;CAChC,CAAC"}

package/dist/consts.d.ts

@@ -0,0 +1 @@
+export declare const UNKNOWN_UNICODE_CHAR = "\uFFFD";

package/dist/consts.js

@@ -0,0 +1,2 @@
+export const UNKNOWN_UNICODE_CHAR = "\ufffd";
+//# sourceMappingURL=consts.js.map

package/dist/consts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"consts.js","sourceRoot":"","sources":["../src/consts.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,oBAAoB,GAAG,QAAQ,CAAC"}

package/dist/evaluator/LlamaChat/LlamaChat.d.ts

@@ -2,7 +2,7 @@
 import { EventRelay } from "lifecycle-utils";
 import { ChatWrapper } from "../../ChatWrapper.js";
 import { LlamaContextSequence } from "../LlamaContext/LlamaContext.js";
-import { ChatHistoryItem, ChatModelFunctions, Token } from "../../types.js";
+import { ChatHistoryItem, ChatModelFunctions, LLamaContextualRepeatPenalty, Token } from "../../types.js";
 import { GbnfJsonSchemaToType } from "../../utils/gbnfJson/types.js";
 import { LlamaGrammar } from "../LlamaGrammar.js";
 import { EvaluationPriority } from "../LlamaContext/types.js";
@@ -61,7 +61,7 @@ export type LLamaChatGenerateResponseOptions<Functions extends ChatModelFunction
      * Disabled by default.
      */
     trimWhitespaceSuffix?: boolean;
-    repeatPenalty?: false | LLamaChatRepeatPenalty;
+    repeatPenalty?: false | LLamaContextualRepeatPenalty;
     /**
      * See the parameter `evaluationPriority` on the `LlamaContextSequence.evaluate()` function for more information.
      */
@@ -119,37 +119,6 @@ export type LLamaChatContextShiftOptions = {
      */
     lastEvaluationMetadata?: object | undefined | null;
 };
-export type LLamaChatRepeatPenalty = {
-    /**
-     * Number of recent tokens generated by the model to apply penalties to repetition of.
-     * Defaults to `64`.
-     */
-    lastTokens?: number;
-    punishTokensFilter?: (tokens: Token[]) => Token[];
-    /**
-     * Penalize new line tokens.
-     * Enabled by default.
-     */
-    penalizeNewLine?: boolean;
-    /**
-     * The relative amount to lower the probability of the tokens in `punishTokens` by
-     * Defaults to `1.1`.
-     * Set to `1` to disable.
-     */
-    penalty?: number;
-    /**
-     * For n time a token is in the `punishTokens` array, lower its probability by `n * frequencyPenalty`
-     * Disabled by default (`0`).
-     * Set to a value between `0` and `1` to enable.
-     */
-    frequencyPenalty?: number;
-    /**
-     * Lower the probability of all the tokens in the `punishTokens` array by `presencePenalty`
-     * Disabled by default (`0`).
-     * Set to a value between `0` and `1` to enable.
-     */
-    presencePenalty?: number;
-};
 export declare class LlamaChat {
     readonly onDispose: EventRelay<void>;
     constructor({ contextSequence, chatWrapper, autoDisposeSequence }: LlamaChatOptions);

package/dist/evaluator/LlamaChat/LlamaChat.js

@@ -2,9 +2,10 @@ import { DisposeAggregator, DisposedError, EventRelay } from "lifecycle-utils";
 import { resolveChatWrapper } from "../../utils/resolveChatWrapper.js";
 import { removeNullFields } from "../../utils/removeNullFields.js";
 import { LlamaGrammarEvaluationState } from "../LlamaGrammarEvaluationState.js";
-import { AbortError } from "../../AbortError.js";
 import { StopGenerationDetector } from "../../utils/StopGenerationDetector.js";
 import { TokenStreamRegulator } from "../../utils/TokenStreamRegulator.js";
+import { UNKNOWN_UNICODE_CHAR } from "../../consts.js";
+import { getQueuedTokensBeforeStopTrigger } from "../../utils/getQueuedTokensBeforeStopTrigger.js";
 import { eraseFirstResponseAndKeepFirstSystemChatContextShiftStrategy } from "./utils/contextShiftStrategies/eraseFirstResponseAndKeepFirstSystemChatContextShiftStrategy.js";
 import { FunctionCallGrammar, LlamaFunctionCallValidationError } from "./utils/FunctionCallGrammar.js";
 const defaultContextShiftOptions = {
@@ -12,7 +13,6 @@ const defaultContextShiftOptions = {
     strategy: "eraseFirstResponseAndKeepFirstSystem",
     lastEvaluationMetadata: null
 };
-const UNKNOWN_UNICODE_CHAR = "\ufffd";
 export class LlamaChat {
     /** @internal */ _chatWrapper;
     /** @internal */ _disposeAggregator = new DisposeAggregator();
@@ -68,7 +68,7 @@ export class LlamaChat {
         if (grammar != null && functionsEnabled)
             throw new Error("Using both grammar and functions is not supported yet");
         if (signal?.aborted)
-            throw new AbortError();
+            throw signal.reason;
         if (this._sequence == null)
             throw new DisposedError();
         let resolvedHistory = this._sequence.isLoadedToMemory
@@ -122,7 +122,7 @@ export class LlamaChat {
         let lastHistoryCompressionMetadata = resolvedContextShift.lastEvaluationMetadata;
         const ensureNotAborted = () => {
             if (signal?.aborted)
-                throw new AbortError();
+                throw signal.reason;
             if (this._sequence == null)
                 throw new DisposedError();
         };
@@ -132,7 +132,7 @@ export class LlamaChat {
             let punishTokens = res.slice(-repeatPenaltyLastTokens);
             if (punishTokensFilter != null)
                 punishTokens = punishTokensFilter(punishTokens);
-            if (!penalizeNewLine) {
+            if (penalizeNewLine == null || !penalizeNewLine) {
                 const nlToken = model.tokens.nl;
                 if (nlToken != null)
                     punishTokens = punishTokens.filter(token => token !== nlToken);
@@ -537,27 +537,6 @@ async function compressHistoryToFitContextSize({ history, contextShiftSize, cont
         metadata
     };
 }
-function getQueuedTokensBeforeStopTrigger(triggeredStops, partiallyFreeTokens, tokenizer) {
-    if (partiallyFreeTokens.tokens.length === 0 && partiallyFreeTokens.text.length === 0)
-        return [];
-    else if (partiallyFreeTokens.tokens.length !== 0 && partiallyFreeTokens.text.length === 0)
-        return partiallyFreeTokens.tokens;
-    else if (partiallyFreeTokens.tokens.length === 0 && partiallyFreeTokens.text.length !== 0)
-        return tokenizer(partiallyFreeTokens.text);
-    const triggerThatStartsWithStringIndex = triggeredStops.findIndex((trigger) => trigger.stopTrigger.length > 0 && typeof trigger.stopTrigger[0] === "string");
-    const triggerThatStartsWithTokenIndex = triggeredStops.findIndex((trigger) => trigger.stopTrigger.length > 0 && typeof trigger.stopTrigger[0] !== "string");
-    if (triggerThatStartsWithTokenIndex > 0 && triggerThatStartsWithStringIndex < 0)
-        return partiallyFreeTokens.tokens;
-    else if (triggerThatStartsWithStringIndex > 0 && triggerThatStartsWithTokenIndex < 0)
-        return tokenizer(partiallyFreeTokens.text);
-    const stringTokens = tokenizer(partiallyFreeTokens.text);
-    if (stringTokens.length === partiallyFreeTokens.tokens.length &&
-        stringTokens.every((value, index) => value === partiallyFreeTokens.tokens[index]))
-        return stringTokens;
-    else if (triggerThatStartsWithStringIndex < triggerThatStartsWithTokenIndex)
-        return stringTokens;
-    return partiallyFreeTokens.tokens;
-}
 function getLastTextModelResponseFromChatHistory(chatHistory) {
     if (chatHistory.length === 0 || chatHistory[chatHistory.length - 1].type !== "model")
         return "";
@@ -673,9 +652,9 @@ async function getContextWindow({ resolvedHistory, resolvedContextShift, lastHis
                 disengageInitiallyEngagedFunctionCall: functionCall?.disengageInitiallyEngaged ?? []
             };
     }
-    const contextShiftSize = resolvedContextShift.size instanceof Function
+    const contextShiftSize = Math.min(context.contextSize, Math.max(1, Math.floor(resolvedContextShift.size instanceof Function
         ? await resolvedContextShift.size(sequence)
-        : resolvedContextShift.size;
+        : resolvedContextShift.size)));
     const { compressedHistory, metadata } = await compressHistoryToFitContextSize({
         history: resolvedHistory,
         contextShiftSize: Math.max(contextShiftSize, minFreeContextTokens) + pendingTokensCount,