node-llama-cpp 3.0.0-beta.1 → 3.0.0-beta.11

package/dist/cli/commands/InfillCommand.js

@@ -0,0 +1,316 @@
+import * as readline from "readline";
+import process from "process";
+import path from "path";
+import chalk from "chalk";
+import fs from "fs-extra";
+import withStatusLogs from "../../utils/withStatusLogs.js";
+import { getLlama } from "../../bindings/getLlama.js";
+import { LlamaModel } from "../../evaluator/LlamaModel.js";
+import { LlamaContext } from "../../evaluator/LlamaContext/LlamaContext.js";
+import { LlamaLogLevel } from "../../bindings/types.js";
+import { LlamaCompletion } from "../../evaluator/LlamaCompletion.js";
+export const InfillCommand = {
+    command: "infill",
+    describe: "Generate an infill completion for a given suffix and prefix texts",
+    builder(yargs) {
+        return yargs
+            .option("model", {
+            alias: "m",
+            type: "string",
+            demandOption: true,
+            description: "Llama model file to use for the chat",
+            group: "Required:"
+        })
+            .option("systemInfo", {
+            alias: "i",
+            type: "boolean",
+            default: false,
+            description: "Print llama.cpp system info",
+            group: "Optional:"
+        })
+            .option("prefix", {
+            type: "string",
+            description: "First prefix text to automatically load",
+            group: "Optional:"
+        })
+            .option("prefixFile", {
+            type: "string",
+            description: "Path to a file to load prefix text from automatically",
+            group: "Optional:"
+        })
+            .option("suffix", {
+            type: "string",
+            description: "First suffix text to automatically load. Requires `prefix` or `prefixFile` to be set",
+            group: "Optional:"
+        })
+            .option("suffixFile", {
+            type: "string",
+            description: "Path to a file to load suffix text from automatically. Requires `prefix` or `prefixFile` to be set",
+            group: "Optional:"
+        })
+            .option("contextSize", {
+            alias: "c",
+            type: "number",
+            default: 1024 * 4,
+            description: "Context size to use for the model context",
+            group: "Optional:"
+        })
+            .option("batchSize", {
+            alias: "b",
+            type: "number",
+            description: "Batch size to use for the model context. The default value is the context size",
+            group: "Optional:"
+        })
+            .option("threads", {
+            type: "number",
+            default: 6,
+            description: "Number of threads to use for the evaluation of tokens",
+            group: "Optional:"
+        })
+            .option("temperature", {
+            alias: "t",
+            type: "number",
+            default: 0,
+            description: "Temperature is a hyperparameter that controls the randomness of the generated text. It affects the probability distribution of the model's output tokens. A higher temperature (e.g., 1.5) makes the output more random and creative, while a lower temperature (e.g., 0.5) makes the output more focused, deterministic, and conservative. The suggested temperature is 0.8, which provides a balance between randomness and determinism. At the extreme, a temperature of 0 will always pick the most likely next token, leading to identical outputs in each run. Set to `0` to disable.",
+            group: "Optional:"
+        })
+            .option("minP", {
+            alias: "mp",
+            type: "number",
+            default: 0,
+            description: "From the next token candidates, discard the percentage of tokens with the lowest probability. For example, if set to `0.05`, 5% of the lowest probability tokens will be discarded. This is useful for generating more high-quality results when using a high temperature. Set to a value between `0` and `1` to enable. Only relevant when `temperature` is set to a value greater than `0`.",
+            group: "Optional:"
+        })
+            .option("topK", {
+            alias: "k",
+            type: "number",
+            default: 40,
+            description: "Limits the model to consider only the K most likely next tokens for sampling at each step of sequence generation. An integer number between `1` and the size of the vocabulary. Set to `0` to disable (which uses the full vocabulary). Only relevant when `temperature` is set to a value greater than 0.",
+            group: "Optional:"
+        })
+            .option("topP", {
+            alias: "p",
+            type: "number",
+            default: 0.95,
+            description: "Dynamically selects the smallest set of tokens whose cumulative probability exceeds the threshold P, and samples the next token only from this set. A float number between `0` and `1`. Set to `1` to disable. Only relevant when `temperature` is set to a value greater than `0`.",
+            group: "Optional:"
+        })
+            .option("gpuLayers", {
+            alias: "gl",
+            type: "number",
+            description: "number of layers to store in VRAM",
+            group: "Optional:"
+        })
+            .option("repeatPenalty", {
+            alias: "rp",
+            type: "number",
+            default: 1.1,
+            description: "Prevent the model from repeating the same token too much. Set to `1` to disable.",
+            group: "Optional:"
+        })
+            .option("lastTokensRepeatPenalty", {
+            alias: "rpn",
+            type: "number",
+            default: 64,
+            description: "Number of recent tokens generated by the model to apply penalties to repetition of",
+            group: "Optional:"
+        })
+            .option("penalizeRepeatingNewLine", {
+            alias: "rpnl",
+            type: "boolean",
+            default: true,
+            description: "Penalize new line tokens. set \"--no-penalizeRepeatingNewLine\" or \"--no-rpnl\" to disable",
+            group: "Optional:"
+        })
+            .option("repeatFrequencyPenalty", {
+            alias: "rfp",
+            type: "number",
+            description: "For n time a token is in the `punishTokens` array, lower its probability by `n * repeatFrequencyPenalty`. Set to a value between `0` and `1` to enable.",
+            group: "Optional:"
+        })
+            .option("repeatPresencePenalty", {
+            alias: "rpp",
+            type: "number",
+            description: "Lower the probability of all the tokens in the `punishTokens` array by `repeatPresencePenalty`. Set to a value between `0` and `1` to enable.",
+            group: "Optional:"
+        })
+            .option("maxTokens", {
+            alias: "mt",
+            type: "number",
+            default: 0,
+            description: "Maximum number of tokens to generate in responses. Set to `0` to disable. Set to `-1` to set to the context size",
+            group: "Optional:"
+        })
+            .option("noInfoLog", {
+            alias: "nl",
+            type: "boolean",
+            default: false,
+            description: "Disable llama.cpp info logs",
+            group: "Optional:"
+        })
+            .option("printTimings", {
+            alias: "pt",
+            type: "boolean",
+            default: false,
+            description: "Print llama.cpp timings after each response",
+            group: "Optional:"
+        });
+    },
+    async handler({ model, systemInfo, prefix, prefixFile, suffix, suffixFile, contextSize, batchSize, threads, temperature, minP, topK, topP, gpuLayers, repeatPenalty, lastTokensRepeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, maxTokens, noInfoLog, printTimings }) {
+        try {
+            await RunInfill({
+                model, systemInfo, prefix, prefixFile, suffix, suffixFile, contextSize, batchSize,
+                threads, temperature, minP, topK, topP, gpuLayers, lastTokensRepeatPenalty,
+                repeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, maxTokens,
+                noInfoLog, printTimings
+            });
+        }
+        catch (err) {
+            console.error(err);
+            process.exit(1);
+        }
+    }
+};
+async function RunInfill({ model: modelArg, systemInfo, prefix, prefixFile, suffix, suffixFile, contextSize, batchSize, threads, temperature, minP, topK, topP, gpuLayers, lastTokensRepeatPenalty, repeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, maxTokens, noInfoLog, printTimings }) {
+    if (noInfoLog)
+        console.info(`${chalk.yellow("Log level:")} warn`);
+    const llama = await getLlama("lastBuild", {
+        logLevel: noInfoLog
+            ? LlamaLogLevel.warn
+            : LlamaLogLevel.debug
+    });
+    const logBatchSize = batchSize != null;
+    if (systemInfo)
+        console.log(llama.systemInfo);
+    if (prefixFile != null && prefixFile !== "") {
+        if (prefix != null && prefix !== "")
+            console.warn(chalk.yellow("Both `prefix` and `prefixFile` were specified. `prefixFile` will be used."));
+        prefix = await fs.readFile(path.resolve(process.cwd(), prefixFile), "utf8");
+    }
+    if (suffixFile != null && suffixFile !== "") {
+        if (suffix != null && suffix !== "")
+            console.warn(chalk.yellow("Both `suffix` and `suffixFile` were specified. `suffixFile` will be used."));
+        suffix = await fs.readFile(path.resolve(process.cwd(), suffixFile), "utf8");
+    }
+    if (suffix != null && prefix == null) {
+        console.warn(chalk.yellow("Suffix was specified but no prefix was specified. Suffix will be ignored."));
+        suffix = undefined;
+    }
+    if (batchSize == null)
+        batchSize = contextSize;
+    else if (batchSize > contextSize) {
+        console.warn(chalk.yellow("Batch size is greater than the context size. Batch size will be set to the context size."));
+        batchSize = contextSize;
+    }
+    let initialPrefix = prefix ?? null;
+    let initialSuffix = suffix ?? null;
+    const model = await withStatusLogs({
+        loading: chalk.blue("Loading model"),
+        success: chalk.blue("Model loaded"),
+        fail: chalk.blue("Failed to load model")
+    }, async () => new LlamaModel({
+        llama,
+        modelPath: path.resolve(process.cwd(), modelArg),
+        gpuLayers: gpuLayers != null ? gpuLayers : undefined
+    }));
+    const context = await withStatusLogs({
+        loading: chalk.blue("Creating context"),
+        success: chalk.blue("Context created"),
+        fail: chalk.blue("Failed to create context")
+    }, async () => new LlamaContext({
+        model,
+        contextSize,
+        batchSize,
+        threads
+    }));
+    const completion = new LlamaCompletion({
+        contextSequence: context.getSequence()
+    });
+    await new Promise((accept) => setTimeout(accept, 0)); // wait for logs to finish printing
+    console.info(`${chalk.yellow("Context size:")} ${context.contextSize}`);
+    if (logBatchSize)
+        console.info(`${chalk.yellow("Batch size:")} ${context.batchSize}`);
+    console.info(`${chalk.yellow("Train context size:")} ${model.trainContextSize}`);
+    console.info(`${chalk.yellow("Model type:")} ${model.typeDescription}`);
+    console.info(`${chalk.yellow("Repeat penalty:")} ${repeatPenalty} (apply to last ${lastTokensRepeatPenalty} tokens)`);
+    if (repeatFrequencyPenalty != null)
+        console.info(`${chalk.yellow("Repeat frequency penalty:")} ${repeatFrequencyPenalty}`);
+    if (repeatPresencePenalty != null)
+        console.info(`${chalk.yellow("Repeat presence penalty:")} ${repeatPresencePenalty}`);
+    if (!penalizeRepeatingNewLine)
+        console.info(`${chalk.yellow("Penalize repeating new line:")} disabled`);
+    // this is for ora to not interfere with readline
+    await new Promise(resolve => setTimeout(resolve, 1));
+    if (!completion.infillSupported) {
+        console.log(chalk.red("Infill is not supported for this model"));
+        process.exit(1);
+    }
+    const replPrefixHistory = [];
+    const replSuffixHistory = [];
+    async function getInput(name) {
+        const rl = readline.createInterface({
+            input: process.stdin,
+            output: process.stdout,
+            history: name === "Prefix"
+                ? replPrefixHistory.slice()
+                : replSuffixHistory.slice()
+        });
+        const res = await new Promise((accept) => rl.question(chalk.yellow(name + "> "), accept));
+        rl.close();
+        return res;
+    }
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        const prefixInput = initialPrefix != null
+            ? initialPrefix
+            : await getInput("Prefix");
+        if (initialPrefix != null) {
+            console.log(chalk.green("Prefix> ") + initialPrefix);
+            initialPrefix = null;
+        }
+        else
+            await replPrefixHistory.push(prefixInput);
+        if (prefixInput === ".exit")
+            break;
+        const suffixInput = initialSuffix != null
+            ? initialSuffix
+            : await getInput("Suffix");
+        if (initialSuffix != null) {
+            console.log(chalk.green("Suffix> ") + initialSuffix);
+            initialSuffix = null;
+        }
+        else
+            await replSuffixHistory.push(suffixInput);
+        if (suffixInput === ".exit")
+            break;
+        process.stdout.write(chalk.yellow("Infill: "));
+        const [startColor, endColor] = chalk.blue("MIDDLE").split("MIDDLE");
+        process.stdout.write(startColor);
+        await completion.generateInfillCompletion(prefixInput, suffixInput, {
+            temperature,
+            minP,
+            topK,
+            topP,
+            repeatPenalty: {
+                penalty: repeatPenalty,
+                frequencyPenalty: repeatFrequencyPenalty != null ? repeatFrequencyPenalty : undefined,
+                presencePenalty: repeatPresencePenalty != null ? repeatPresencePenalty : undefined,
+                penalizeNewLine: penalizeRepeatingNewLine,
+                lastTokens: lastTokensRepeatPenalty
+            },
+            maxTokens: maxTokens === -1
+                ? context.contextSize
+                : maxTokens <= 0
+                    ? undefined
+                    : maxTokens,
+            onToken(chunk) {
+                process.stdout.write(model.detokenize(chunk));
+            }
+        });
+        process.stdout.write(endColor);
+        console.log();
+        if (printTimings)
+            await context.printTimings();
+    }
+}
+//# sourceMappingURL=InfillCommand.js.map

package/dist/cli/commands/InfillCommand.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"InfillCommand.js","sourceRoot":"","sources":["../../../src/cli/commands/InfillCommand.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AACrC,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,cAAc,MAAM,+BAA+B,CAAC;AAC3D,OAAO,EAAC,QAAQ,EAAC,MAAM,4BAA4B,CAAC;AACpD,OAAO,EAAC,UAAU,EAAC,MAAM,+BAA+B,CAAC;AACzD,OAAO,EAAC,YAAY,EAAC,MAAM,8CAA8C,CAAC;AAC1E,OAAO,EAAC,aAAa,EAAC,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAC,eAAe,EAAC,MAAM,oCAAoC,CAAC;AA2BnE,MAAM,CAAC,MAAM,aAAa,GAAyC;IAC/D,OAAO,EAAE,QAAQ;IACjB,QAAQ,EAAE,mEAAmE;IAC7E,OAAO,CAAC,KAAK;QACT,OAAO,KAAK;aACP,MAAM,CAAC,OAAO,EAAE;YACb,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,YAAY,EAAE,IAAI;YAClB,WAAW,EAAE,sCAAsC;YACnD,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,YAAY,EAAE;YAClB,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,KAAK;YACd,WAAW,EAAE,6BAA6B;YAC1C,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,QAAQ,EAAE;YACd,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,yCAAyC;YACtD,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,YAAY,EAAE;YAClB,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,uDAAuD;YACpE,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,QAAQ,EAAE;YACd,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,sFAAsF;YACnG,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,YAAY,EAAE;YAClB,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,oGAAoG;YACjH,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,aAAa,EAAE;YACnB,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,IAAI,GAAG,CAAC;YACjB,WAAW,EAAE,2CAA2C;YACxD,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,WAAW,EAAE;YACjB,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,gFAAgF;YAC7F,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,SAAS,EAAE;YACf,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,CAAC;YACV,WAAW,EAAE,uDAAuD;YACpE,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,aAAa,EAAE;YACnB,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,CAAC;YACV,WAAW,EAAE,6jBAA6jB;YAC1kB,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,MAAM,EAAE;YACZ,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,CAAC;YACV,WAAW,EAAE,+XAA+X;YAC5Y,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,MAAM,EAAE;YACZ,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,EAAE;YACX,WAAW,EAAE,4SAA4S;YACzT,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,MAAM,EAAE;YACZ,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,IAAI;YACb,WAAW,EAAE,qRAAqR;YAClS,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,WAAW,EAAE;YACjB,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,mCAAmC;YAChD,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,eAAe,EAAE;YACrB,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,GAAG;YACZ,WAAW,EAAE,kFAAkF;YAC/F,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,yBAAyB,EAAE;YAC/B,KAAK,EAAE,KAAK;YACZ,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,EAAE;YACX,WAAW,EAAE,oFAAoF;YACjG,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,0BAA0B,EAAE;YAChC,KAAK,EAAE,MAAM;YACb,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,IAAI;YACb,WAAW,EAAE,6FAA6F;YAC1G,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,wBAAwB,EAAE;YAC9B,KAAK,EAAE,KAAK;YACZ,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,yJAAyJ;YACtK,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,uBAAuB,EAAE;YAC7B,KAAK,EAAE,KAAK;YACZ,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,+IAA+I;YAC5J,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,WAAW,EAAE;YACjB,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,CAAC;YACV,WAAW,EAAE,kHAAkH;YAC/H,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,WAAW,EAAE;YACjB,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,KAAK;YACd,WAAW,EAAE,6BAA6B;YAC1C,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,cAAc,EAAE;YACpB,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,KAAK;YACd,WAAW,EAAE,6CAA6C;YAC1D,KAAK,EAAE,WAAW;SACrB,CAAC,CAAC;IACX,CAAC;IACD,KAAK,CAAC,OAAO,CAAC,EACV,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EACjF,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAChC,IAAI,EAAE,SAAS,EAAE,aAAa,EAAE,uBAAuB,EAAE,wBAAwB,EACjF,sBAAsB,EAAE,qBAAqB,EAAE,SAAS,EACxD,SAAS,EAAE,YAAY,EAC1B;QACG,IAAI;YACA,MAAM,SAAS,CAAC;gBACZ,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS;gBACjF,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,uBAAuB;gBAC1E,aAAa,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,qBAAqB,EAAE,SAAS;gBACjG,SAAS,EAAE,YAAY;aAC1B,CAAC,CAAC;SACN;QAAC,OAAO,GAAG,EAAE;YACV,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACnB;IACL,CAAC;CACJ,CAAC;AAGF,KAAK,UAAU,SAAS,CAAC,EACrB,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAC3F,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EACjD,uBAAuB,EAAE,aAAa,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,qBAAqB,EAC/G,SAAS,EAAE,SAAS,EAAE,YAAY,EACtB;IACZ,IAAI,SAAS;QACT,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;IAEvD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE;QACtC,QAAQ,EAAE,SAAS;YACf,CAAC,CAAC,aAAa,CAAC,IAAI;YACpB,CAAC,CAAC,aAAa,CAAC,KAAK;KAC5B,CAAC,CAAC;IACH,MAAM,YAAY,GAAG,SAAS,IAAI,IAAI,CAAC;IAEvC,IAAI,UAAU;QACV,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAElC,IAAI,UAAU,IAAI,IAAI,IAAI,UAAU,KAAK,EAAE,EAAE;QACzC,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,KAAK,EAAE;YAC/B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,2EAA2E,CAAC,CAAC,CAAC;QAE5G,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,UAAU,CAAC,EAAE,MAAM,CAAC,CAAC;KAC/E;IAED,IAAI,UAAU,IAAI,IAAI,IAAI,UAAU,KAAK,EAAE,EAAE;QACzC,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,KAAK,EAAE;YAC/B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,2EAA2E,CAAC,CAAC,CAAC;QAE5G,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,UAAU,CAAC,EAAE,MAAM,CAAC,CAAC;KAC/E;IAED,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,IAAI,EAAE;QAClC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,2EAA2E,CAAC,CAAC,CAAC;QACxG,MAAM,GAAG,SAAS,CAAC;KACtB;IAED,IAAI,SAAS,IAAI,IAAI;QACjB,SAAS,GAAG,WAAW,CAAC;SACvB,IAAI,SAAS,GAAG,WAAW,EAAE;QAC9B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,0FAA0F,CAAC,CAAC,CAAC;QACvH,SAAS,GAAG,WAAW,CAAC;KAC3B;IAED,IAAI,aAAa,GAAG,MAAM,IAAI,IAAI,CAAC;IACnC,IAAI,aAAa,GAAG,MAAM,IAAI,IAAI,CAAC;IAEnC,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC;QAC/B,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC;QACpC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC;QACnC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC;KAC3C,EAAE,KAAK,IAAI,EAAE,CAAC,IAAI,UAAU,CAAC;QAC1B,KAAK;QACL,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,QAAQ,CAAC;QAChD,SAAS,EAAE,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;KACvD,CAAC,CAAC,CAAC;IACJ,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC;QACjC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC;QACvC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC;QACtC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC;KAC/C,EAAE,KAAK,IAAI,EAAE,CAAC,IAAI,YAAY,CAAC;QAC5B,KAAK;QACL,WAAW;QACX,SAAS;QACT,OAAO;KACV,CAAC,CAAC,CAAC;IAEJ,MAAM,UAAU,GAAG,IAAI,eAAe,CAAC;QACnC,eAAe,EAAE,OAAO,CAAC,WAAW,EAAE;KACzC,CAAC,CAAC;IAEH,MAAM,IAAI,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,mCAAmC;IAEzF,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IAExE,IAAI,YAAY;QACZ,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IAExE,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,qBAAqB,CAAC,IAAI,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC;IACjF,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC;IACxE,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI,aAAa,mBAAmB,uBAAuB,UAAU,CAAC,CAAC;IAEtH,IAAI,sBAAsB,IAAI,IAAI;QAC9B,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,2BAA2B,CAAC,IAAI,sBAAsB,EAAE,CAAC,CAAC;IAE3F,IAAI,qBAAqB,IAAI,IAAI;QAC7B,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,0BAA0B,CAAC,IAAI,qBAAqB,EAAE,CAAC,CAAC;IAEzF,IAAI,CAAC,wBAAwB;QACzB,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,8BAA8B,CAAC,WAAW,CAAC,CAAC;IAE7E,iDAAiD;IACjD,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;IAErD,IAAI,CAAC,UAAU,CAAC,eAAe,EAAE;QAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC,CAAC;QACjE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACnB;IAED,MAAM,iBAAiB,GAAa,EAAE,CAAC;IACvC,MAAM,iBAAiB,GAAa,EAAE,CAAC;IAEvC,KAAK,UAAU,QAAQ,CAAC,IAAyB;QAC7C,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;YAChC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,OAAO,EAAE,IAAI,KAAK,QAAQ;gBACtB,CAAC,CAAC,iBAAiB,CAAC,KAAK,EAAE;gBAC3B,CAAC,CAAC,iBAAiB,CAAC,KAAK,EAAE;SAClC,CAAC,CAAC;QAEH,MAAM,GAAG,GAAW,MAAM,IAAI,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;QAClG,EAAE,CAAC,KAAK,EAAE,CAAC;QAEX,OAAO,GAAG,CAAC;IACf,CAAC;IAED,iDAAiD;IACjD,OAAO,IAAI,EAAE;QACT,MAAM,WAAW,GAAG,aAAa,IAAI,IAAI;YACrC,CAAC,CAAC,aAAa;YACf,CAAC,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAE/B,IAAI,aAAa,IAAI,IAAI,EAAE;YACvB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,CAAC;YACrD,aAAa,GAAG,IAAI,CAAC;SACxB;;YACG,MAAM,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAE9C,IAAI,WAAW,KAAK,OAAO;YACvB,MAAM;QAEV,MAAM,WAAW,GAAG,aAAa,IAAI,IAAI;YACrC,CAAC,CAAC,aAAa;YACf,CAAC,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAE/B,IAAI,aAAa,IAAI,IAAI,EAAE;YACvB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,CAAC;YACrD,aAAa,GAAG,IAAI,CAAC;SACxB;;YACG,MAAM,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAE9C,IAAI,WAAW,KAAK,OAAO;YACvB,MAAM;QAEV,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;QAE/C,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAEpE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACjC,MAAM,UAAU,CAAC,wBAAwB,CAAC,WAAW,EAAE,WAAW,EAAE;YAChE,WAAW;YACX,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,aAAa,EAAE;gBACX,OAAO,EAAE,aAAa;gBACtB,gBAAgB,EAAE,sBAAsB,IAAI,IAAI,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,SAAS;gBACrF,eAAe,EAAE,qBAAqB,IAAI,IAAI,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,SAAS;gBAClF,eAAe,EAAE,wBAAwB;gBACzC,UAAU,EAAE,uBAAuB;aACtC;YACD,SAAS,EAAE,SAAS,KAAK,CAAC,CAAC;gBACvB,CAAC,CAAC,OAAO,CAAC,WAAW;gBACrB,CAAC,CAAC,SAAS,IAAI,CAAC;oBACZ,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,SAAS;YACnB,OAAO,CAAC,KAAK;gBACT,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;YAClD,CAAC;SACJ,CAAC,CAAC;QACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC/B,OAAO,CAAC,GAAG,EAAE,CAAC;QAEd,IAAI,YAAY;YACZ,MAAM,OAAO,CAAC,YAAY,EAAE,CAAC;KACpC;AACL,CAAC"}

package/dist/cli/commands/OnPostInstallCommand.js

@@ -1,21 +1,18 @@
-import { defaultLlamaCppCudaSupport, defaultLlamaCppGitHubRepo, defaultLlamaCppMetalSupport, defaultLlamaCppRelease, defaultSkipDownload } from "../../config.js";
-import { getPrebuildBinPath } from "../../utils/getBin.js";
-import { DownloadLlamaCppCommand } from "./DownloadCommand.js";
+import { defaultSkipDownload } from "../../config.js";
+import { getLlamaForOptions } from "../../bindings/getLlama.js";
 export const OnPostInstallCommand = {
     command: "postinstall",
     describe: false,
     async handler() {
         if (defaultSkipDownload)
             return;
-        if (await getPrebuildBinPath() != null)
-            return;
         try {
-            await DownloadLlamaCppCommand({
-                repo: defaultLlamaCppGitHubRepo,
-                release: defaultLlamaCppRelease,
-                metal: defaultLlamaCppMetalSupport,
-                cuda: defaultLlamaCppCudaSupport
+            await getLlamaForOptions({
+                progressLogs: true
+            }, {
+                updateLastBuildInfoOnCompile: true
             });
+            process.exit(0);
         }
         catch (err) {
             console.error(err);

package/dist/cli/commands/OnPostInstallCommand.js.map

@@ -1 +1 @@
-{"version":3,"file":"OnPostInstallCommand.js","sourceRoot":"","sources":["../../../src/cli/commands/OnPostInstallCommand.ts"],"names":[],"mappings":"AACA,OAAO,EACH,0BAA0B,EAAE,yBAAyB,EAAE,2BAA2B,EAAE,sBAAsB,EAAE,mBAAmB,EAClI,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAC,kBAAkB,EAAC,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAC,uBAAuB,EAAC,MAAM,sBAAsB,CAAC;AAI7D,MAAM,CAAC,MAAM,oBAAoB,GAAgD;IAC7E,OAAO,EAAE,aAAa;IACtB,QAAQ,EAAE,KAAK;IACf,KAAK,CAAC,OAAO;QACT,IAAI,mBAAmB;YACnB,OAAO;QAEX,IAAI,MAAM,kBAAkB,EAAE,IAAI,IAAI;YAClC,OAAO;QAEX,IAAI;YACA,MAAM,uBAAuB,CAAC;gBAC1B,IAAI,EAAE,yBAAyB;gBAC/B,OAAO,EAAE,sBAAsB;gBAC/B,KAAK,EAAE,2BAA2B;gBAClC,IAAI,EAAE,0BAA0B;aACnC,CAAC,CAAC;SACN;QAAC,OAAO,GAAG,EAAE;YACV,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACnB;IACL,CAAC;CACJ,CAAC"}
+{"version":3,"file":"OnPostInstallCommand.js","sourceRoot":"","sources":["../../../src/cli/commands/OnPostInstallCommand.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,mBAAmB,EAAC,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAC,kBAAkB,EAAC,MAAM,4BAA4B,CAAC;AAI9D,MAAM,CAAC,MAAM,oBAAoB,GAAgD;IAC7E,OAAO,EAAE,aAAa;IACtB,QAAQ,EAAE,KAAK;IACf,KAAK,CAAC,OAAO;QACT,IAAI,mBAAmB;YACnB,OAAO;QAEX,IAAI;YACA,MAAM,kBAAkB,CAAC;gBACrB,YAAY,EAAE,IAAI;aACrB,EAAE;gBACC,4BAA4B,EAAE,IAAI;aACrC,CAAC,CAAC;YAEH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACnB;QAAC,OAAO,GAAG,EAAE;YACV,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACnB;IACL,CAAC;CACJ,CAAC"}

package/dist/config.d.ts

@@ -1,24 +1,31 @@
+import { LlamaLogLevel } from "./bindings/types.js";
 export declare const llamaDirectory: string;
 export declare const llamaToolchainsDirectory: string;
-export declare const llamaBinsDirectory: string;
+export declare const llamaPrebuiltBinsDirectory: string;
+export declare const llamaLocalBuildBinsDirectory: string;
 export declare const llamaBinsGrammarsDirectory: string;
 export declare const llamaCppDirectory: string;
 export declare const llamaCppGrammarsDirectory: string;
 export declare const tempDownloadDirectory: string;
 export declare const chatCommandHistoryFilePath: string;
-export declare const usedBinFlagJsonPath: string;
+export declare const lastBuildInfoJsonPath: string;
 export declare const binariesGithubReleasePath: string;
-export declare const llamaCppDirectoryTagFilePath: string;
+export declare const llamaCppDirectoryInfoFilePath: string;
 export declare const currentReleaseGitBundlePath: string;
 export declare const xpackDirectory: string;
 export declare const localXpacksStoreDirectory: string;
 export declare const localXpacksCacheDirectory: string;
+export declare const buildMetadataFileName = ".buildMetadata.json";
 export declare const xpmVersion = "^0.16.3";
+export declare const builtinLlamaCppGitHubRepo = "ggerganov/llama.cpp";
+export declare const builtinLlamaCppRelease: string;
 export declare const isCI: boolean;
+export declare const isRunningInsideGoogleColab: boolean;
 export declare const defaultLlamaCppGitHubRepo: string;
 export declare const defaultLlamaCppRelease: string;
 export declare const defaultLlamaCppMetalSupport: boolean;
 export declare const defaultLlamaCppCudaSupport: boolean;
+export declare const defaultLlamaCppDebugLogs: LlamaLogLevel;
 export declare const defaultSkipDownload: boolean;
 export declare const defaultXpacksStoreDirectory: string;
 export declare const defaultXpacksCacheDirectory: string;

package/dist/config.js

@@ -4,40 +4,51 @@ import * as os from "os";
 import process from "process";
 import envVar from "env-var";
 import * as uuid from "uuid";
-import { getBinariesGithubRelease } from "./utils/binariesGithubRelease.js";
+import { getBinariesGithubRelease } from "./bindings/utils/binariesGithubRelease.js";
+import { LlamaLogLevel, LlamaLogLevelValues } from "./bindings/types.js";
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const env = envVar.from(process.env);
 export const llamaDirectory = path.join(__dirname, "..", "llama");
 export const llamaToolchainsDirectory = path.join(llamaDirectory, "toolchains");
-export const llamaBinsDirectory = path.join(__dirname, "..", "llamaBins");
+export const llamaPrebuiltBinsDirectory = path.join(__dirname, "..", "llamaBins");
+export const llamaLocalBuildBinsDirectory = path.join(llamaDirectory, "localBuilds");
 export const llamaBinsGrammarsDirectory = path.join(__dirname, "..", "llama", "grammars");
 export const llamaCppDirectory = path.join(llamaDirectory, "llama.cpp");
 export const llamaCppGrammarsDirectory = path.join(llamaDirectory, "llama.cpp", "grammars");
 export const tempDownloadDirectory = path.join(os.tmpdir(), "node-llama-cpp", uuid.v4());
 export const chatCommandHistoryFilePath = path.join(os.homedir(), ".node-llama-cpp.chat_repl_history");
-export const usedBinFlagJsonPath = path.join(llamaDirectory, "usedBin.json");
+export const lastBuildInfoJsonPath = path.join(llamaDirectory, "lastBuild.json");
 export const binariesGithubReleasePath = path.join(llamaDirectory, "binariesGithubRelease.json");
-export const llamaCppDirectoryTagFilePath = path.join(llamaDirectory, "llama.cpp.tag.json");
+export const llamaCppDirectoryInfoFilePath = path.join(llamaDirectory, "llama.cpp.info.json");
 export const currentReleaseGitBundlePath = path.join(llamaDirectory, "gitRelease.bundle");
 export const xpackDirectory = path.join(llamaDirectory, "xpack");
 export const localXpacksStoreDirectory = path.join(xpackDirectory, "store");
 export const localXpacksCacheDirectory = path.join(xpackDirectory, "cache");
+export const buildMetadataFileName = ".buildMetadata.json";
 export const xpmVersion = "^0.16.3";
+export const builtinLlamaCppGitHubRepo = "ggerganov/llama.cpp";
+export const builtinLlamaCppRelease = await getBinariesGithubRelease();
 export const isCI = env.get("CI")
     .default("false")
     .asBool();
+export const isRunningInsideGoogleColab = env.get("COLAB_RELEASE_TAG")
+    .default("")
+    .asString() !== "";
 export const defaultLlamaCppGitHubRepo = env.get("NODE_LLAMA_CPP_REPO")
-    .default("ggerganov/llama.cpp")
+    .default(builtinLlamaCppGitHubRepo)
     .asString();
 export const defaultLlamaCppRelease = env.get("NODE_LLAMA_CPP_REPO_RELEASE")
-    .default(await getBinariesGithubRelease())
+    .default(builtinLlamaCppRelease)
     .asString();
 export const defaultLlamaCppMetalSupport = env.get("NODE_LLAMA_CPP_METAL")
-    .default(process.platform === "darwin" ? "true" : "false")
+    .default((process.platform === "darwin" && process.arch !== "x64") ? "true" : "false")
     .asBool();
 export const defaultLlamaCppCudaSupport = env.get("NODE_LLAMA_CPP_CUDA")
     .default("false")
     .asBool();
+export const defaultLlamaCppDebugLogs = env.get("NODE_LLAMA_CPP_LOG_LEVEL")
+    .default(LlamaLogLevel.debug)
+    .asEnum(LlamaLogLevelValues);
 export const defaultSkipDownload = env.get("NODE_LLAMA_CPP_SKIP_DOWNLOAD")
     .default("false")
     .asBool();

package/dist/config.js.map

@@ -1 +1 @@
-{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,KAAK,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,MAAM,MAAM,SAAS,CAAC;AAC7B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAC,wBAAwB,EAAC,MAAM,kCAAkC,CAAC;AAE1E,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE/D,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;AAGrC,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;AAClE,MAAM,CAAC,MAAM,wBAAwB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;AAChF,MAAM,CAAC,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;AAC1E,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;AAC1F,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;AACxE,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;AAC5F,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,gBAAgB,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;AACzF,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,mCAAmC,CAAC,CAAC;AACvG,MAAM,CAAC,MAAM,mBAAmB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,cAAc,CAAC,CAAC;AAC7E,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,4BAA4B,CAAC,CAAC;AACjG,MAAM,CAAC,MAAM,4BAA4B,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,oBAAoB,CAAC,CAAC;AAC5F,MAAM,CAAC,MAAM,2BAA2B,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,mBAAmB,CAAC,CAAC;AAC1F,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;AACjE,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;AAC5E,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;AAC5E,MAAM,CAAC,MAAM,UAAU,GAAG,SAAS,CAAC;AAEpC,MAAM,CAAC,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC;KAC5B,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,yBAAyB,GAAG,GAAG,CAAC,GAAG,CAAC,qBAAqB,CAAC;KAClE,OAAO,CAAC,qBAAqB,CAAC;KAC9B,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC,GAAG,CAAC,6BAA6B,CAAC;KACvE,OAAO,CAAC,MAAM,wBAAwB,EAAE,CAAC;KACzC,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC,GAAG,CAAC,sBAAsB,CAAC;KACrE,OAAO,CAAC,OAAO,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;KACzD,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,0BAA0B,GAAG,GAAG,CAAC,GAAG,CAAC,qBAAqB,CAAC;KACnE,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,mBAAmB,GAAG,GAAG,CAAC,GAAG,CAAC,8BAA8B,CAAC;KACrE,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC,GAAG,CAAC,oCAAoC,CAAC;KACnF,OAAO,CAAC,yBAAyB,CAAC;KAClC,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC,GAAG,CAAC,oCAAoC,CAAC;KACnF,OAAO,CAAC,yBAAyB,CAAC;KAClC,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,8BAA8B,GAAG,8BAA8B,CAAC;AAC7E,MAAM,CAAC,MAAM,uBAAuB,GAAG,+FAA+F;IAClI,+HAA+H;IAC/H,mFAAmF,CAAC;AACxF,MAAM,CAAC,MAAM,UAAU,GAAG,gBAAgB,CAAC;AAC3C,MAAM,CAAC,MAAM,YAAY,GAAG,WAAW,CAAC;AAExC,MAAM,gBAAgB,GAAG,4CAA4C,CAAC;AACtE,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACjC,IAAI,EAAE,gBAAgB,GAAG,aAAa;CAChC,CAAC"}
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,KAAK,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,MAAM,MAAM,SAAS,CAAC;AAC7B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAC,wBAAwB,EAAC,MAAM,2CAA2C,CAAC;AACnF,OAAO,EAAC,aAAa,EAAE,mBAAmB,EAAC,MAAM,qBAAqB,CAAC;AAEvE,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE/D,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;AAGrC,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;AAClE,MAAM,CAAC,MAAM,wBAAwB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;AAChF,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;AAClF,MAAM,CAAC,MAAM,4BAA4B,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,aAAa,CAAC,CAAC;AACrF,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;AAC1F,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;AACxE,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;AAC5F,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,gBAAgB,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;AACzF,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,mCAAmC,CAAC,CAAC;AACvG,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;AACjF,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,4BAA4B,CAAC,CAAC;AACjG,MAAM,CAAC,MAAM,6BAA6B,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,qBAAqB,CAAC,CAAC;AAC9F,MAAM,CAAC,MAAM,2BAA2B,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,mBAAmB,CAAC,CAAC;AAC1F,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;AACjE,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;AAC5E,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;AAC5E,MAAM,CAAC,MAAM,qBAAqB,GAAG,qBAAqB,CAAC;AAC3D,MAAM,CAAC,MAAM,UAAU,GAAG,SAAS,CAAC;AACpC,MAAM,CAAC,MAAM,yBAAyB,GAAG,qBAAqB,CAAC;AAC/D,MAAM,CAAC,MAAM,sBAAsB,GAAG,MAAM,wBAAwB,EAAE,CAAC;AAEvE,MAAM,CAAC,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC;KAC5B,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,0BAA0B,GAAG,GAAG,CAAC,GAAG,CAAC,mBAAmB,CAAC;KACjE,OAAO,CAAC,EAAE,CAAC;KACX,QAAQ,EAAE,KAAK,EAAE,CAAC;AACvB,MAAM,CAAC,MAAM,yBAAyB,GAAG,GAAG,CAAC,GAAG,CAAC,qBAAqB,CAAC;KAClE,OAAO,CAAC,yBAAyB,CAAC;KAClC,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC,GAAG,CAAC,6BAA6B,CAAC;KACvE,OAAO,CAAC,sBAAsB,CAAC;KAC/B,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC,GAAG,CAAC,sBAAsB,CAAC;KACrE,OAAO,CAAC,CAAC,OAAO,CAAC,QAAQ,KAAK,QAAQ,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;KACrF,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,0BAA0B,GAAG,GAAG,CAAC,GAAG,CAAC,qBAAqB,CAAC;KACnE,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,wBAAwB,GAAG,GAAG,CAAC,GAAG,CAAC,0BAA0B,CAAC;KACtE,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC;KAC5B,MAAM,CAAC,mBAAmB,CAAC,CAAC;AACjC,MAAM,CAAC,MAAM,mBAAmB,GAAG,GAAG,CAAC,GAAG,CAAC,8BAA8B,CAAC;KACrE,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC,GAAG,CAAC,oCAAoC,CAAC;KACnF,OAAO,CAAC,yBAAyB,CAAC;KAClC,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC,GAAG,CAAC,oCAAoC,CAAC;KACnF,OAAO,CAAC,yBAAyB,CAAC;KAClC,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,8BAA8B,GAAG,8BAA8B,CAAC;AAC7E,MAAM,CAAC,MAAM,uBAAuB,GAAG,+FAA+F;IAClI,+HAA+H;IAC/H,mFAAmF,CAAC;AACxF,MAAM,CAAC,MAAM,UAAU,GAAG,gBAAgB,CAAC;AAC3C,MAAM,CAAC,MAAM,YAAY,GAAG,WAAW,CAAC;AAExC,MAAM,gBAAgB,GAAG,4CAA4C,CAAC;AACtE,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACjC,IAAI,EAAE,gBAAgB,GAAG,aAAa;CAChC,CAAC"}

package/dist/consts.d.ts

@@ -0,0 +1 @@
+export declare const UNKNOWN_UNICODE_CHAR = "\uFFFD";

package/dist/consts.js

@@ -0,0 +1,2 @@
+export const UNKNOWN_UNICODE_CHAR = "\ufffd";
+//# sourceMappingURL=consts.js.map

package/dist/consts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"consts.js","sourceRoot":"","sources":["../src/consts.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,oBAAoB,GAAG,QAAQ,CAAC"}

package/dist/evaluator/LlamaChat/LlamaChat.d.ts

@@ -0,0 +1,154 @@
+/// <reference types="node" />
+import { EventRelay } from "lifecycle-utils";
+import { ChatWrapper } from "../../ChatWrapper.js";
+import { LlamaContextSequence } from "../LlamaContext/LlamaContext.js";
+import { ChatHistoryItem, ChatModelFunctions, LLamaContextualRepeatPenalty, Token } from "../../types.js";
+import { GbnfJsonSchemaToType } from "../../utils/gbnfJson/types.js";
+import { LlamaGrammar } from "../LlamaGrammar.js";
+import { EvaluationPriority } from "../LlamaContext/types.js";
+export type LlamaChatOptions = {
+    contextSequence: LlamaContextSequence;
+    /** `"auto"` is used by default */
+    chatWrapper?: "auto" | ChatWrapper;
+    /** Automatically dispose the sequence when the session is disposed */
+    autoDisposeSequence?: boolean;
+};
+export type LLamaChatGenerateResponseOptions<Functions extends ChatModelFunctions | undefined = undefined> = {
+    onToken?: (tokens: Token[]) => void;
+    signal?: AbortSignal;
+    maxTokens?: number;
+    /**
+     * Temperature is a hyperparameter that controls the randomness of the generated text.
+     * It affects the probability distribution of the model's output tokens.
+     * A higher temperature (e.g., 1.5) makes the output more random and creative,
+     * while a lower temperature (e.g., 0.5) makes the output more focused, deterministic, and conservative.
+     * The suggested temperature is 0.8, which provides a balance between randomness and determinism.
+     * At the extreme, a temperature of 0 will always pick the most likely next token, leading to identical outputs in each run.
+     *
+     * Set to `0` to disable.
+     * Disabled by default (set to `0`).
+     */
+    temperature?: number;
+    /**
+     * From the next token candidates, discard the percentage of tokens with the lowest probability.
+     * For example, if set to `0.05`, 5% of the lowest probability tokens will be discarded.
+     * This is useful for generating more high-quality results when using a high temperature.
+     * Set to a value between `0` and `1` to enable.
+     *
+     * Only relevant when `temperature` is set to a value greater than `0`.
+     * Disabled by default.
+     */
+    minP?: number;
+    /**
+     * Limits the model to consider only the K most likely next tokens for sampling at each step of sequence generation.
+     * An integer number between `1` and the size of the vocabulary.
+     * Set to `0` to disable (which uses the full vocabulary).
+     *
+     * Only relevant when `temperature` is set to a value greater than 0.
+     */
+    topK?: number;
+    /**
+     * Dynamically selects the smallest set of tokens whose cumulative probability exceeds the threshold P,
+     * and samples the next token only from this set.
+     * A float number between `0` and `1`.
+     * Set to `1` to disable.
+     *
+     * Only relevant when `temperature` is set to a value greater than `0`.
+     */
+    topP?: number;
+    /**
+     * Trim whitespace from the end of the generated text
+     * Disabled by default.
+     */
+    trimWhitespaceSuffix?: boolean;
+    repeatPenalty?: false | LLamaContextualRepeatPenalty;
+    /**
+     * See the parameter `evaluationPriority` on the `LlamaContextSequence.evaluate()` function for more information.
+     */
+    evaluationPriority?: EvaluationPriority;
+    contextShift?: LLamaChatContextShiftOptions;
+    /**
+     * The evaluation context window returned from the last evaluation.
+     * This is an optimization to utilize existing context sequence state better when possible.
+     */
+    lastEvaluationContextWindow?: {
+        /** The history of the last evaluation. */
+        history?: ChatHistoryItem[];
+        /**
+         * Minimum overlap percentage with existing context sequence state to use the last evaluation context window.
+         * If the last evaluation context window is not used, a new context will be generated based on the full history,
+         * which will decrease the likelihood of another context shift happening so soon.
+         */
+        minimumOverlapPercentageToPreventContextShift?: number;
+    };
+} & ({
+    grammar?: LlamaGrammar;
+    functions?: never;
+    documentFunctionParams?: never;
+} | {
+    grammar?: never;
+    functions?: Functions | ChatModelFunctions;
+    documentFunctionParams?: boolean;
+});
+export type LLamaChatContextShiftOptions = {
+    /**
+     * The number of tokens to delete from the context window to make space for new ones.
+     * Defaults to 10% of the context size.
+     */
+    size?: number | ((sequence: LlamaContextSequence) => number | Promise<number>);
+    /**
+     * The strategy to use when deleting tokens from the context window.
+     * Defaults to `"eraseFirstResponseAndKeepFirstSystem"`.
+     */
+    strategy?: "eraseFirstResponseAndKeepFirstSystem" | ((options: {
+        chatHistory: ChatHistoryItem[];
+        maxTokensCount: number;
+        tokenizer(text: string, specialTokens?: boolean): Token[];
+        chatWrapper: ChatWrapper;
+        lastShiftMetadata?: object | null;
+    }) => {
+        chatHistory: ChatHistoryItem[];
+        metadata?: object | null;
+    } | Promise<{
+        chatHistory: ChatHistoryItem[];
+        metadata?: object | null;
+    }>);
+    /**
+     * The `contextShiftMetadata` returned from the last evaluation.
+     * This is an optimization to utilize the existing context state better when possible.
+     */
+    lastEvaluationMetadata?: object | undefined | null;
+};
+export declare class LlamaChat {
+    readonly onDispose: EventRelay<void>;
+    constructor({ contextSequence, chatWrapper, autoDisposeSequence }: LlamaChatOptions);
+    dispose({ disposeSequence }?: {
+        disposeSequence?: boolean;
+    }): void;
+    /** @hidden */
+    [Symbol.dispose](): void;
+    get disposed(): boolean;
+    get chatWrapper(): ChatWrapper;
+    get sequence(): LlamaContextSequence;
+    get context(): import("../LlamaContext/LlamaContext.js").LlamaContext;
+    get model(): import("../LlamaModel.js").LlamaModel;
+    generateResponse<const Functions extends ChatModelFunctions | undefined = undefined>(history: ChatHistoryItem[], { onToken, signal, maxTokens, temperature, minP, topK, topP, grammar, trimWhitespaceSuffix, repeatPenalty, evaluationPriority, functions, documentFunctionParams, contextShift, lastEvaluationContextWindow: { history: lastEvaluationContextWindowHistory, minimumOverlapPercentageToPreventContextShift } }?: LLamaChatGenerateResponseOptions<Functions>): Promise<LlamaChatResponse<Functions>>;
+}
+export type LlamaChatResponse<Functions extends ChatModelFunctions | undefined = undefined> = {
+    response: string;
+    functionCall?: Functions extends ChatModelFunctions ? LlamaChatResponseFunctionCall<Functions> : never;
+    lastEvaluation: {
+        cleanHistory: ChatHistoryItem[];
+        contextWindow: ChatHistoryItem[];
+        contextShiftMetadata: any;
+    };
+    metadata: {
+        remainingGenerationAfterStop?: string | Token[];
+        stopReason: "eosToken" | "stopGenerationTrigger" | "functionCall" | "maxTokens";
+    };
+};
+export type LlamaChatResponseFunctionCall<Functions extends ChatModelFunctions, FunctionCallName extends keyof Functions & string = string & keyof Functions, Params = Functions[FunctionCallName]["params"] extends undefined | null | void ? undefined : GbnfJsonSchemaToType<Functions[FunctionCallName]["params"]>> = {
+    functionName: FunctionCallName;
+    params: Params;
+    raw: string;
+};