node-llama-cpp 3.0.0-beta.11 → 3.0.0-beta.13

package/dist/cli/commands/InspectCommand.js

@@ -0,0 +1,113 @@
+import os from "os";
+import bytes from "bytes";
+import chalk from "chalk";
+import { getLlamaForOptions } from "../../bindings/getLlama.js";
+import { detectAvailableComputeLayers } from "../../bindings/utils/detectAvailableComputeLayers.js";
+import { getPlatform } from "../../bindings/utils/getPlatform.js";
+import { LlamaLogLevel } from "../../bindings/types.js";
+import { getPrettyBuildGpuName } from "../../bindings/consts.js";
+const inspectFunctions = ["gpu"];
+export const InspectCommand = {
+    command: "inspect [function]",
+    describe: "Inspect the inner workings of node-llama-cpp",
+    builder(yargs) {
+        return yargs
+            .option("function", {
+            type: "string",
+            choices: inspectFunctions,
+            demandOption: true,
+            description: "inspect function to run"
+        });
+    },
+    async handler({ function: func }) {
+        if (func === "gpu")
+            await InspectGpuFunction();
+        else
+            void (func);
+    }
+};
+async function InspectGpuFunction() {
+    const platform = getPlatform();
+    const arch = process.arch;
+    const availableComputeLayers = await detectAvailableComputeLayers({ platform });
+    const gpusToLogVramUsageOf = [];
+    if (platform === "mac" && arch === "arm64") {
+        console.info(`${chalk.yellow("Metal:")} ${chalk.green("available")}`);
+        gpusToLogVramUsageOf.push("metal");
+    }
+    else {
+        console.info(`${chalk.yellow("Metal:")} ${chalk.red("not supported by llama.cpp on Intel Macs")}`);
+    }
+    if (availableComputeLayers.cuda.hasNvidiaDriver && !availableComputeLayers.cuda.hasCudaRuntime) {
+        console.info(`${chalk.yellow("CUDA:")} ${chalk.red("NVIDIA driver is installed, but CUDA runtime is not")}`);
+    }
+    else if (availableComputeLayers.cuda.hasCudaRuntime && !availableComputeLayers.cuda.hasNvidiaDriver) {
+        console.info(`${chalk.yellow("CUDA:")} ${chalk.red("CUDA runtime is installed, but NVIDIA driver is not")}`);
+    }
+    else if (availableComputeLayers.cuda.hasCudaRuntime && availableComputeLayers.cuda.hasNvidiaDriver) {
+        console.info(`${chalk.yellow("CUDA:")} ${chalk.green("available")}`);
+        gpusToLogVramUsageOf.push("cuda");
+    }
+    if (availableComputeLayers.vulkan) {
+        console.info(`${chalk.yellow("Vulkan:")} ${chalk.green("available")}`);
+        gpusToLogVramUsageOf.push("vulkan");
+    }
+    for (const gpu of gpusToLogVramUsageOf) {
+        console.info();
+        await logGpuVramUsage(gpu);
+    }
+    console.info();
+    await logRamUsage();
+}
+async function logGpuVramUsage(gpu) {
+    try {
+        const llama = await getLlamaForOptions({
+            gpu: gpu,
+            build: "never",
+            progressLogs: false,
+            logLevel: LlamaLogLevel.warn
+        }, {
+            skipLlamaInit: true
+        });
+        const gpuName = getPrettyBuildGpuName(gpu);
+        const vramStatus = llama.getVramState();
+        console.info(`${chalk.yellow(`${gpuName} used VRAM:`)} ${getPercentageString(vramStatus.used, vramStatus.total)}% ${chalk.grey("(" + bytes(vramStatus.used) + "/" + bytes(vramStatus.total) + ")")}`);
+        console.info(`${chalk.yellow(`${gpuName} free VRAM:`)} ${getPercentageString(vramStatus.free, vramStatus.total)}% ${chalk.grey("(" + bytes(vramStatus.free) + "/" + bytes(vramStatus.total) + ")")}`);
+    }
+    catch (err) { }
+}
+async function logRamUsage() {
+    const totalMemory = os.totalmem();
+    const freeMemory = os.freemem();
+    const usedMemory = totalMemory - freeMemory;
+    console.info(`${chalk.yellow("Used RAM:")} ${getPercentageString(usedMemory, totalMemory)}% ${chalk.grey("(" + bytes(usedMemory) + "/" + bytes(totalMemory) + ")")}`);
+    console.info(`${chalk.yellow("Free RAM:")} ${getPercentageString(freeMemory, totalMemory)}% ${chalk.grey("(" + bytes(freeMemory) + "/" + bytes(totalMemory) + ")")}`);
+}
+function getPercentageString(amount, total) {
+    if (total === 0)
+        return "0";
+    return String(Math.floor((amount / total) * 100 * 100) / 100);
+}
+// // simple script to copy console logs as ansi to clipboard. Used to update the documentation
+// import {spawn} from "child_process";
+// const pendingLog: string[] = [];
+// const originalConsoleInfo = console.info;
+// console.info = function info(...args: any[]) {
+//     originalConsoleInfo.call(console, ...args);
+//     pendingLog.push(args.join(" "));
+// };
+//
+// function copyLogs() {
+//     const res = pendingLog.join("\n");
+//
+//     pbcopy(res);
+//     originalConsoleInfo.call(console, "Copied logs to clipboard");
+// }
+// function pbcopy(text: string) {
+//     const pbcopyProcess = spawn("pbcopy");
+//     pbcopyProcess.stdin.write(text);
+//     pbcopyProcess.stdin.end();
+// }
+//
+// process.on("exit", copyLogs);
+//# sourceMappingURL=InspectCommand.js.map

package/dist/cli/commands/InspectCommand.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"InspectCommand.js","sourceRoot":"","sources":["../../../src/cli/commands/InspectCommand.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAC,kBAAkB,EAAC,MAAM,4BAA4B,CAAC;AAC9D,OAAO,EAAC,4BAA4B,EAAC,MAAM,sDAAsD,CAAC;AAClG,OAAO,EAAC,WAAW,EAAC,MAAM,qCAAqC,CAAC;AAChE,OAAO,EAAW,aAAa,EAAC,MAAM,yBAAyB,CAAC;AAChE,OAAO,EAAC,qBAAqB,EAAC,MAAM,0BAA0B,CAAC;AAE/D,MAAM,gBAAgB,GAAG,CAAC,KAAK,CAAU,CAAC;AAK1C,MAAM,CAAC,MAAM,cAAc,GAA0C;IACjE,OAAO,EAAE,oBAAoB;IAC7B,QAAQ,EAAE,8CAA8C;IACxD,OAAO,CAAC,KAAK;QACT,OAAO,KAAK;aACP,MAAM,CAAC,UAAU,EAAE;YAChB,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,gBAAgB;YACzB,YAAY,EAAE,IAAI;YAClB,WAAW,EAAE,yBAAyB;SACzC,CAAC,CAAC;IACX,CAAC;IACD,KAAK,CAAC,OAAO,CAAC,EAAC,QAAQ,EAAE,IAAI,EAAiB;QAC1C,IAAI,IAAI,KAAK,KAAK;YACd,MAAM,kBAAkB,EAAE,CAAC;;YAE3B,KAAK,CAAC,IAAoB,CAAC,CAAC;IACpC,CAAC;CACJ,CAAC;AAEF,KAAK,UAAU,kBAAkB;IAC7B,MAAM,QAAQ,GAAG,WAAW,EAAE,CAAC;IAC/B,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAC1B,MAAM,sBAAsB,GAAG,MAAM,4BAA4B,CAAC,EAAC,QAAQ,EAAC,CAAC,CAAC;IAC9E,MAAM,oBAAoB,GAAe,EAAE,CAAC;IAE5C,IAAI,QAAQ,KAAK,KAAK,IAAI,IAAI,KAAK,OAAO,EAAE;QACxC,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QACtE,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;KACtC;SAAM;QACH,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,0CAA0C,CAAC,EAAE,CAAC,CAAC;KACtG;IAED,IAAI,sBAAsB,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,cAAc,EAAE;QAC5F,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,qDAAqD,CAAC,EAAE,CAAC,CAAC;KAChH;SAAM,IAAI,sBAAsB,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,eAAe,EAAE;QACnG,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,qDAAqD,CAAC,EAAE,CAAC,CAAC;KAChH;SAAM,IAAI,sBAAsB,CAAC,IAAI,CAAC,cAAc,IAAI,sBAAsB,CAAC,IAAI,CAAC,eAAe,EAAE;QAClG,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QACrE,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;KACrC;IAED,IAAI,sBAAsB,CAAC,MAAM,EAAE;QAC/B,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QACvE,oBAAoB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;KACvC;IAED,KAAK,MAAM,GAAG,IAAI,oBAAoB,EAAE;QACpC,OAAO,CAAC,IAAI,EAAE,CAAC;QACf,MAAM,eAAe,CAAC,GAAG,CAAC,CAAC;KAC9B;IAED,OAAO,CAAC,IAAI,EAAE,CAAC;IACf,MAAM,WAAW,EAAE,CAAC;AACxB,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,GAAa;IACxC,IAAI;QACA,MAAM,KAAK,GAAG,MAAM,kBAAkB,CAAC;YACnC,GAAG,EAAE,GAAG;YACR,KAAK,EAAE,OAAO;YACd,YAAY,EAAE,KAAK;YACnB,QAAQ,EAAE,aAAa,CAAC,IAAI;SAC/B,EAAE;YACC,aAAa,EAAE,IAAI;SACtB,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC;QAC3C,MAAM,UAAU,GAAG,KAAK,CAAC,YAAY,EAAE,CAAC;QAExC,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,OAAO,aAAa,CAAC,IAAI,mBAAmB,CAAC,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC;QACtM,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,OAAO,aAAa,CAAC,IAAI,mBAAmB,CAAC,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC;KACzM;IAAC,OAAO,GAAG,EAAE,GAAE;AACpB,CAAC;AAED,KAAK,UAAU,WAAW;IACtB,MAAM,WAAW,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC;IAClC,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;IAChC,MAAM,UAAU,GAAG,WAAW,GAAG,UAAU,CAAC;IAE5C,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,mBAAmB,CAAC,UAAU,EAAE,WAAW,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,GAAG,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC;IACtK,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,mBAAmB,CAAC,UAAU,EAAE,WAAW,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,GAAG,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC;AAC1K,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAc,EAAE,KAAa;IACtD,IAAI,KAAK,KAAK,CAAC;QACX,OAAO,GAAG,CAAC;IAEf,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,GAAG,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC;AAClE,CAAC;AAED,+FAA+F;AAC/F,uCAAuC;AACvC,mCAAmC;AACnC,4CAA4C;AAC5C,iDAAiD;AACjD,kDAAkD;AAClD,uCAAuC;AACvC,KAAK;AACL,EAAE;AACF,wBAAwB;AACxB,yCAAyC;AACzC,EAAE;AACF,mBAAmB;AACnB,qEAAqE;AACrE,IAAI;AACJ,kCAAkC;AAClC,6CAA6C;AAC7C,uCAAuC;AACvC,iCAAiC;AACjC,IAAI;AACJ,EAAE;AACF,gCAAgC"}

package/dist/cli/utils/logUsedGpuTypeOption.d.ts

@@ -0,0 +1,2 @@
+import { BuildGpu } from "../../bindings/types.js";
+export declare function logUsedGpuTypeOption(gpu: BuildGpu): void;

package/dist/cli/utils/logUsedGpuTypeOption.js

@@ -0,0 +1,9 @@
+import chalk from "chalk";
+import { getPrettyBuildGpuName } from "../../bindings/consts.js";
+export function logUsedGpuTypeOption(gpu) {
+    if (gpu == false)
+        console.log(`${chalk.yellow("GPU:")} disabled`);
+    else
+        console.log(`${chalk.yellow("GPU:")} ${getPrettyBuildGpuName(gpu)}`);
+}
+//# sourceMappingURL=logUsedGpuTypeOption.js.map

package/dist/cli/utils/logUsedGpuTypeOption.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"logUsedGpuTypeOption.js","sourceRoot":"","sources":["../../../src/cli/utils/logUsedGpuTypeOption.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,EAAC,qBAAqB,EAAC,MAAM,0BAA0B,CAAC;AAE/D,MAAM,UAAU,oBAAoB,CAAC,GAAa;IAC9C,IAAI,GAAG,IAAI,KAAK;QACZ,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;;QAEhD,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,qBAAqB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;AAC7E,CAAC"}

package/dist/config.d.ts

@@ -23,8 +23,7 @@ export declare const isCI: boolean;
 export declare const isRunningInsideGoogleColab: boolean;
 export declare const defaultLlamaCppGitHubRepo: string;
 export declare const defaultLlamaCppRelease: string;
-export declare const defaultLlamaCppMetalSupport: boolean;
-export declare const defaultLlamaCppCudaSupport: boolean;
+export declare const defaultLlamaCppGpuSupport: false | "cuda" | "vulkan" | "metal" | "auto";
 export declare const defaultLlamaCppDebugLogs: LlamaLogLevel;
 export declare const defaultSkipDownload: boolean;
 export declare const defaultXpacksStoreDirectory: string;
@@ -35,4 +34,6 @@ export declare const cliBinName = "node-llama-cpp";
 export declare const npxRunPrefix = "npx --no ";
 export declare const documentationPageUrls: {
     readonly CUDA: string;
+    readonly Vulkan: string;
 };
+export declare const recommendedBaseDockerImage = "node:20";

package/dist/config.js

@@ -5,7 +5,7 @@ import process from "process";
 import envVar from "env-var";
 import * as uuid from "uuid";
 import { getBinariesGithubRelease } from "./bindings/utils/binariesGithubRelease.js";
-import { LlamaLogLevel, LlamaLogLevelValues } from "./bindings/types.js";
+import { nodeLlamaCppGpuOptions, LlamaLogLevel, LlamaLogLevelValues, parseNodeLlamaCppGpuOption, nodeLlamaCppGpuOffStringOptions } from "./bindings/types.js";
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const env = envVar.from(process.env);
 export const llamaDirectory = path.join(__dirname, "..", "llama");
@@ -40,12 +40,12 @@ export const defaultLlamaCppGitHubRepo = env.get("NODE_LLAMA_CPP_REPO")
 export const defaultLlamaCppRelease = env.get("NODE_LLAMA_CPP_REPO_RELEASE")
     .default(builtinLlamaCppRelease)
     .asString();
-export const defaultLlamaCppMetalSupport = env.get("NODE_LLAMA_CPP_METAL")
-    .default((process.platform === "darwin" && process.arch !== "x64") ? "true" : "false")
-    .asBool();
-export const defaultLlamaCppCudaSupport = env.get("NODE_LLAMA_CPP_CUDA")
-    .default("false")
-    .asBool();
+export const defaultLlamaCppGpuSupport = parseNodeLlamaCppGpuOption(env.get("NODE_LLAMA_CPP_GPU")
+    .default("auto")
+    .asEnum(nodeLlamaCppGpuOptions
+    .flatMap((option) => (option === false
+    ? nodeLlamaCppGpuOffStringOptions
+    : [option]))));
 export const defaultLlamaCppDebugLogs = env.get("NODE_LLAMA_CPP_LOG_LEVEL")
     .default(LlamaLogLevel.debug)
     .asEnum(LlamaLogLevelValues);
@@ -60,12 +60,14 @@ export const defaultXpacksCacheDirectory = env.get("NODE_LLAMA_CPP_XPACKS_CACHE_
     .asString();
 export const customCmakeOptionsEnvVarPrefix = "NODE_LLAMA_CPP_CMAKE_OPTION_";
 export const defaultChatSystemPrompt = "You are a helpful, respectful and honest assistant. Always answer as helpfully as possible.\n" +
-    "If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. " +
-    "If you don't know the answer to a question, please don't share false information.";
+    "If a question does not make any sense, or is not factually coherent, explain why instead of answering something incorrectly. " +
+    "If you don't know the answer to a question, don't share false information.";
 export const cliBinName = "node-llama-cpp";
 export const npxRunPrefix = "npx --no ";
 const documentationUrl = "https://withcatai.github.io/node-llama-cpp";
 export const documentationPageUrls = {
-    CUDA: documentationUrl + "/guide/CUDA"
+    CUDA: documentationUrl + "/guide/CUDA",
+    Vulkan: documentationUrl + "/guide/vulkan"
 };
+export const recommendedBaseDockerImage = "node:20";
 //# sourceMappingURL=config.js.map

package/dist/config.js.map

@@ -1 +1 @@
-{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,KAAK,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,MAAM,MAAM,SAAS,CAAC;AAC7B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAC,wBAAwB,EAAC,MAAM,2CAA2C,CAAC;AACnF,OAAO,EAAC,aAAa,EAAE,mBAAmB,EAAC,MAAM,qBAAqB,CAAC;AAEvE,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE/D,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;AAGrC,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;AAClE,MAAM,CAAC,MAAM,wBAAwB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;AAChF,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;AAClF,MAAM,CAAC,MAAM,4BAA4B,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,aAAa,CAAC,CAAC;AACrF,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;AAC1F,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;AACxE,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;AAC5F,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,gBAAgB,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;AACzF,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,mCAAmC,CAAC,CAAC;AACvG,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;AACjF,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,4BAA4B,CAAC,CAAC;AACjG,MAAM,CAAC,MAAM,6BAA6B,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,qBAAqB,CAAC,CAAC;AAC9F,MAAM,CAAC,MAAM,2BAA2B,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,mBAAmB,CAAC,CAAC;AAC1F,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;AACjE,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;AAC5E,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;AAC5E,MAAM,CAAC,MAAM,qBAAqB,GAAG,qBAAqB,CAAC;AAC3D,MAAM,CAAC,MAAM,UAAU,GAAG,SAAS,CAAC;AACpC,MAAM,CAAC,MAAM,yBAAyB,GAAG,qBAAqB,CAAC;AAC/D,MAAM,CAAC,MAAM,sBAAsB,GAAG,MAAM,wBAAwB,EAAE,CAAC;AAEvE,MAAM,CAAC,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC;KAC5B,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,0BAA0B,GAAG,GAAG,CAAC,GAAG,CAAC,mBAAmB,CAAC;KACjE,OAAO,CAAC,EAAE,CAAC;KACX,QAAQ,EAAE,KAAK,EAAE,CAAC;AACvB,MAAM,CAAC,MAAM,yBAAyB,GAAG,GAAG,CAAC,GAAG,CAAC,qBAAqB,CAAC;KAClE,OAAO,CAAC,yBAAyB,CAAC;KAClC,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC,GAAG,CAAC,6BAA6B,CAAC;KACvE,OAAO,CAAC,sBAAsB,CAAC;KAC/B,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC,GAAG,CAAC,sBAAsB,CAAC;KACrE,OAAO,CAAC,CAAC,OAAO,CAAC,QAAQ,KAAK,QAAQ,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;KACrF,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,0BAA0B,GAAG,GAAG,CAAC,GAAG,CAAC,qBAAqB,CAAC;KACnE,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,wBAAwB,GAAG,GAAG,CAAC,GAAG,CAAC,0BAA0B,CAAC;KACtE,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC;KAC5B,MAAM,CAAC,mBAAmB,CAAC,CAAC;AACjC,MAAM,CAAC,MAAM,mBAAmB,GAAG,GAAG,CAAC,GAAG,CAAC,8BAA8B,CAAC;KACrE,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC,GAAG,CAAC,oCAAoC,CAAC;KACnF,OAAO,CAAC,yBAAyB,CAAC;KAClC,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC,GAAG,CAAC,oCAAoC,CAAC;KACnF,OAAO,CAAC,yBAAyB,CAAC;KAClC,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,8BAA8B,GAAG,8BAA8B,CAAC;AAC7E,MAAM,CAAC,MAAM,uBAAuB,GAAG,+FAA+F;IAClI,+HAA+H;IAC/H,mFAAmF,CAAC;AACxF,MAAM,CAAC,MAAM,UAAU,GAAG,gBAAgB,CAAC;AAC3C,MAAM,CAAC,MAAM,YAAY,GAAG,WAAW,CAAC;AAExC,MAAM,gBAAgB,GAAG,4CAA4C,CAAC;AACtE,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACjC,IAAI,EAAE,gBAAgB,GAAG,aAAa;CAChC,CAAC"}
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,KAAK,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,MAAM,MAAM,SAAS,CAAC;AAC7B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAC,wBAAwB,EAAC,MAAM,2CAA2C,CAAC;AACnF,OAAO,EACH,sBAAsB,EAAE,aAAa,EAAE,mBAAmB,EAAE,0BAA0B,EAAE,+BAA+B,EAC1H,MAAM,qBAAqB,CAAC;AAE7B,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE/D,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;AAGrC,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;AAClE,MAAM,CAAC,MAAM,wBAAwB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;AAChF,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;AAClF,MAAM,CAAC,MAAM,4BAA4B,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,aAAa,CAAC,CAAC;AACrF,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;AAC1F,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;AACxE,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;AAC5F,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,gBAAgB,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;AACzF,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,mCAAmC,CAAC,CAAC;AACvG,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;AACjF,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,4BAA4B,CAAC,CAAC;AACjG,MAAM,CAAC,MAAM,6BAA6B,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,qBAAqB,CAAC,CAAC;AAC9F,MAAM,CAAC,MAAM,2BAA2B,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,mBAAmB,CAAC,CAAC;AAC1F,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;AACjE,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;AAC5E,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;AAC5E,MAAM,CAAC,MAAM,qBAAqB,GAAG,qBAAqB,CAAC;AAC3D,MAAM,CAAC,MAAM,UAAU,GAAG,SAAS,CAAC;AACpC,MAAM,CAAC,MAAM,yBAAyB,GAAG,qBAAqB,CAAC;AAC/D,MAAM,CAAC,MAAM,sBAAsB,GAAG,MAAM,wBAAwB,EAAE,CAAC;AAEvE,MAAM,CAAC,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC;KAC5B,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,0BAA0B,GAAG,GAAG,CAAC,GAAG,CAAC,mBAAmB,CAAC;KACjE,OAAO,CAAC,EAAE,CAAC;KACX,QAAQ,EAAE,KAAK,EAAE,CAAC;AACvB,MAAM,CAAC,MAAM,yBAAyB,GAAG,GAAG,CAAC,GAAG,CAAC,qBAAqB,CAAC;KAClE,OAAO,CAAC,yBAAyB,CAAC;KAClC,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC,GAAG,CAAC,6BAA6B,CAAC;KACvE,OAAO,CAAC,sBAAsB,CAAC;KAC/B,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,yBAAyB,GAAG,0BAA0B,CAC/D,GAAG,CAAC,GAAG,CAAC,oBAAoB,CAAC;KACxB,OAAO,CAAC,MAAM,CAAC;KACf,MAAM,CACH,sBAAsB;KACjB,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CACjB,MAAM,KAAK,KAAK;IACZ,CAAC,CAAC,+BAA+B;IACjC,CAAC,CAAC,CAAC,MAAM,CAAC,CACjB,CAAC,CACT,CACR,CAAC;AACF,MAAM,CAAC,MAAM,wBAAwB,GAAG,GAAG,CAAC,GAAG,CAAC,0BAA0B,CAAC;KACtE,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC;KAC5B,MAAM,CAAC,mBAAmB,CAAC,CAAC;AACjC,MAAM,CAAC,MAAM,mBAAmB,GAAG,GAAG,CAAC,GAAG,CAAC,8BAA8B,CAAC;KACrE,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC,GAAG,CAAC,oCAAoC,CAAC;KACnF,OAAO,CAAC,yBAAyB,CAAC;KAClC,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC,GAAG,CAAC,oCAAoC,CAAC;KACnF,OAAO,CAAC,yBAAyB,CAAC;KAClC,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,8BAA8B,GAAG,8BAA8B,CAAC;AAC7E,MAAM,CAAC,MAAM,uBAAuB,GAAG,+FAA+F;IAClI,+HAA+H;IAC/H,4EAA4E,CAAC;AACjF,MAAM,CAAC,MAAM,UAAU,GAAG,gBAAgB,CAAC;AAC3C,MAAM,CAAC,MAAM,YAAY,GAAG,WAAW,CAAC;AAExC,MAAM,gBAAgB,GAAG,4CAA4C,CAAC;AACtE,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACjC,IAAI,EAAE,gBAAgB,GAAG,aAAa;IACtC,MAAM,EAAE,gBAAgB,GAAG,eAAe;CACpC,CAAC;AACX,MAAM,CAAC,MAAM,0BAA0B,GAAG,SAAS,CAAC"}

package/dist/evaluator/LlamaChatSession/utils/defineChatSessionFunction.d.ts

@@ -1,5 +1,8 @@
 import { GbnfJsonSchema, GbnfJsonSchemaToType } from "../../../utils/gbnfJson/types.js";
 import { ChatSessionModelFunction } from "../../../types.js";
+/**
+ * @param functionDefinition
+ */
 export declare function defineChatSessionFunction<const Params extends GbnfJsonSchema | undefined>({ description, params, handler }: {
     description?: string;
     params?: Params;

package/dist/evaluator/LlamaChatSession/utils/defineChatSessionFunction.js

@@ -1,3 +1,6 @@
+/**
+ * @param functionDefinition
+ */
 export function defineChatSessionFunction({ description, params, handler }) {
     return {
         description,

package/dist/evaluator/LlamaChatSession/utils/defineChatSessionFunction.js.map

@@ -1 +1 @@
-{"version":3,"file":"defineChatSessionFunction.js","sourceRoot":"","sources":["../../../../src/evaluator/LlamaChatSession/utils/defineChatSessionFunction.ts"],"names":[],"mappings":"AAGA,MAAM,UAAU,yBAAyB,CAAkD,EACvF,WAAW,EACX,MAAM,EACN,OAAO,EAKV;IACG,OAAO;QACH,WAAW;QACX,MAAM;QACN,OAAO;KACV,CAAC;AACN,CAAC"}
+{"version":3,"file":"defineChatSessionFunction.js","sourceRoot":"","sources":["../../../../src/evaluator/LlamaChatSession/utils/defineChatSessionFunction.ts"],"names":[],"mappings":"AAGA;;GAEG;AACH,MAAM,UAAU,yBAAyB,CAAkD,EACvF,WAAW,EACX,MAAM,EACN,OAAO,EAKV;IACG,OAAO;QACH,WAAW;QACX,MAAM;QACN,OAAO;KACV,CAAC;AACN,CAAC"}

package/dist/gguf/GGUFInsights.d.ts

@@ -0,0 +1,28 @@
+import { Llama } from "../bindings/Llama.js";
+import { GGUFMetadataResponse } from "./ggufParser/GGUFParser.js";
+export type GGUFInsightsOptions = {
+    contextCount?: number;
+    nodeLlama?: Llama;
+    modelSize?: number;
+};
+export default class GGUFInsights {
+    readonly metadataResponse: GGUFMetadataResponse;
+    readonly options: GGUFInsightsOptions;
+    get metadata(): import("./ggufParser/GGUFTypes.js").GGUFMetadataAny;
+    get architectureMetadata(): import("./ggufParser/GGUFTypes.js").GGUFMetadataArchitectureProperties;
+    /**
+     * fp16 k,v matrices
+     */
+    get kvMatrices(): number;
+    /**
+     * This amount is the overhead + tensors in memory
+     */
+    get graphSize(): number;
+    get VRAMUsage(): number;
+    protected get _availableVRam(): number;
+    constructor(metadataResponse: GGUFMetadataResponse, options?: GGUFInsightsOptions);
+    /**
+     * The score of the model by how much it's compatible to the current system
+     */
+    modelScore(): number;
+}

package/dist/gguf/GGUFInsights.js

@@ -0,0 +1,58 @@
+import MissingNodeLlamaError from "./errors/MissingNodeLlamaError.js";
+import NotEnoughVRamError from "./errors/ModelScore/NotEnoughVRamError.js";
+const PAD_AVAILABLE_VRAM = 1024 ** 2 * 500; // 500MB
+export default class GGUFInsights {
+    metadataResponse;
+    options = {};
+    get metadata() {
+        return this.metadataResponse.metadata;
+    }
+    get architectureMetadata() {
+        return this.metadata[this.metadata.general.architecture];
+    }
+    /**
+     * fp16 k,v matrices
+     */
+    get kvMatrices() {
+        // 2 bytes each * 2 key and value
+        return (2 * 2 *
+            this.architectureMetadata.context_length *
+            this.architectureMetadata.block_count *
+            this.architectureMetadata.embedding_length *
+            this.architectureMetadata.attention.head_count_kv /
+            this.architectureMetadata.attention.head_count);
+    }
+    /**
+     * This amount is the overhead + tensors in memory
+     */
+    get graphSize() {
+        // TODO: get this from the llama.cpp's graph calculations instead of
+        // estimating it's 1/6 * kv_cache_size * num_gqa
+        return ((this.architectureMetadata.attention.head_count_kv /
+            this.architectureMetadata.attention.head_count) * this.kvMatrices / 6);
+    }
+    get VRAMUsage() {
+        return this.graphSize + this.kvMatrices + this.metadataResponse.metadataSize;
+    }
+    get _availableVRam() {
+        if (!this.options?.nodeLlama) {
+            throw new MissingNodeLlamaError("GGUFInsights Calculations");
+        }
+        return this.options.nodeLlama.getVramState().total - PAD_AVAILABLE_VRAM;
+    }
+    constructor(metadataResponse, options = {}) {
+        this.options = options;
+        this.metadataResponse = metadataResponse;
+    }
+    /**
+     * The score of the model by how much it's compatible to the current system
+     */
+    modelScore() {
+        const vramScore = this.VRAMUsage / this._availableVRam;
+        if (vramScore >= 1) {
+            throw new NotEnoughVRamError(this.VRAMUsage, this._availableVRam);
+        }
+        return vramScore;
+    }
+}
+//# sourceMappingURL=GGUFInsights.js.map

package/dist/gguf/GGUFInsights.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GGUFInsights.js","sourceRoot":"","sources":["../../src/gguf/GGUFInsights.ts"],"names":[],"mappings":"AACA,OAAO,qBAAqB,MAAM,mCAAmC,CAAC;AAEtE,OAAO,kBAAkB,MAAM,2CAA2C,CAAC;AAE3E,MAAM,kBAAkB,GAAG,IAAI,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ;AAQpD,MAAM,CAAC,OAAO,OAAO,YAAY;IACb,gBAAgB,CAAuB;IACvC,OAAO,GAAwB,EAAE,CAAC;IAElD,IAAW,QAAQ;QACf,OAAO,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC;IAC1C,CAAC;IAED,IAAW,oBAAoB;QAC3B,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IAC7D,CAAC;IAED;;OAEG;IACH,IAAW,UAAU;QACjB,iCAAiC;QACjC,OAAO,CACH,CAAC,GAAG,CAAC;YACL,IAAI,CAAC,oBAAoB,CAAC,cAAc;YACxC,IAAI,CAAC,oBAAoB,CAAC,WAAW;YACrC,IAAI,CAAC,oBAAoB,CAAC,gBAAgB;YAC1C,IAAI,CAAC,oBAAoB,CAAC,SAAS,CAAC,aAAa;YACjD,IAAI,CAAC,oBAAoB,CAAC,SAAS,CAAC,UAAU,CACjD,CAAC;IACN,CAAC;IAED;;OAEG;IACH,IAAW,SAAS;QAChB,oEAAoE;QACpE,gDAAgD;QAChD,OAAO,CACH,CAAC,IAAI,CAAC,oBAAoB,CAAC,SAAS,CAAC,aAAa;YAClD,IAAI,CAAC,oBAAoB,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,UAAU,GAAG,CAAC,CACxE,CAAC;IACN,CAAC;IAED,IAAW,SAAS;QAChB,OAAO,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC;IACjF,CAAC;IAED,IAAc,cAAc;QACxB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,EAAC;YACzB,MAAM,IAAI,qBAAqB,CAAC,2BAA2B,CAAC,CAAC;SAChE;QACD,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,YAAY,EAAE,CAAC,KAAK,GAAG,kBAAkB,CAAC;IAC5E,CAAC;IAED,YAAmB,gBAAsC,EAAE,UAA+B,EAAE;QACxF,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;IAE7C,CAAC;IAGD;;OAEG;IACI,UAAU;QACb,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC;QACvD,IAAI,SAAS,IAAI,CAAC,EAAC;YACf,MAAM,IAAI,kBAAkB,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;SACrE;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;CAEJ"}

package/dist/gguf/GGUFMetadata.d.ts

@@ -0,0 +1,19 @@
+import retry from "async-retry";
+import GGUFInsights, { GGUFInsightsOptions } from "./GGUFInsights.js";
+import { GGUFMetadataResponse } from "./ggufParser/GGUFParser.js";
+export type GGUFMetadataOptions = {
+    source?: "network" | "local";
+    retry?: retry.Options;
+    ignoreKeys?: string[];
+    insights?: GGUFInsightsOptions;
+};
+export default class GGUFMetadata {
+    protected _metadata?: GGUFMetadataResponse;
+    readonly path: string;
+    readonly options: Partial<GGUFMetadataOptions>;
+    get metadata(): GGUFMetadataResponse;
+    get insights(): GGUFInsights;
+    constructor(path: string, options?: Partial<GGUFMetadataOptions>);
+    parse(): Promise<GGUFMetadataResponse>;
+    private _createStream;
+}

package/dist/gguf/GGUFMetadata.js

@@ -0,0 +1,38 @@
+import MetadataNotParsedYetError from "./errors/MetadataNotParsedYetError.js";
+import GGUFInsights from "./GGUFInsights.js";
+import GGUFParser from "./ggufParser/GGUFParser.js";
+import GGUFFetchStream from "./ggufParser/stream/GGUFFetchStream.js";
+import GGUFReadStream from "./ggufParser/stream/GGUFReadStream.js";
+export default class GGUFMetadata {
+    _metadata;
+    path;
+    options = {};
+    get metadata() {
+        if (!this._metadata) {
+            throw new MetadataNotParsedYetError(this.path);
+        }
+        return this._metadata;
+    }
+    get insights() {
+        return new GGUFInsights(this.metadata, this.options.insights);
+    }
+    constructor(path, options = {}) {
+        this.options = options;
+        this.path = path;
+    }
+    async parse() {
+        const stream = this._createStream();
+        const parser = new GGUFParser(stream, this.options.ignoreKeys);
+        return this._metadata = await parser.parseMetadata();
+    }
+    _createStream() {
+        switch (this.options.source) {
+            case "network":
+                return new GGUFFetchStream(this.path, { retry: this.options.retry });
+            case "local":
+            default:
+                return new GGUFReadStream(this.path, { retry: this.options.retry });
+        }
+    }
+}
+//# sourceMappingURL=GGUFMetadata.js.map

package/dist/gguf/GGUFMetadata.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GGUFMetadata.js","sourceRoot":"","sources":["../../src/gguf/GGUFMetadata.ts"],"names":[],"mappings":"AACA,OAAO,yBAAyB,MAAM,uCAAuC,CAAC;AAC9E,OAAO,YAAmC,MAAM,mBAAmB,CAAC;AACpE,OAAO,UAAkC,MAAM,4BAA4B,CAAC;AAC5E,OAAO,eAAe,MAAM,wCAAwC,CAAC;AACrE,OAAO,cAAc,MAAM,uCAAuC,CAAC;AASnE,MAAM,CAAC,OAAO,OAAO,YAAY;IACnB,SAAS,CAAwB;IAC3B,IAAI,CAAS;IACb,OAAO,GAAiC,EAAE,CAAC;IAE3D,IAAW,QAAQ;QACf,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE;YACjB,MAAM,IAAI,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SAClD;QACD,OAAO,IAAI,CAAC,SAAS,CAAC;IAC1B,CAAC;IAED,IAAW,QAAQ;QACf,OAAO,IAAI,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAClE,CAAC;IAED,YAAmB,IAAY,EAAE,UAAwC,EAAE;QACvE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACrB,CAAC;IAEM,KAAK,CAAC,KAAK;QACd,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAC/D,OAAO,IAAI,CAAC,SAAS,GAAG,MAAM,MAAM,CAAC,aAAa,EAAE,CAAC;IACzD,CAAC;IAEO,aAAa;QACjB,QAAQ,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YACzB,KAAK,SAAS;gBACV,OAAO,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,EAAE,EAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAC,CAAC,CAAC;YACvE,KAAK,OAAO,CAAC;YACb;gBACI,OAAO,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,EAAE,EAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAC,CAAC,CAAC;SACzE;IACL,CAAC;CACJ"}

package/dist/gguf/errors/InvalidGGUFMagicError.d.ts

@@ -0,0 +1,3 @@
+export default class InvalidGGUFMagicError extends Error {
+    constructor(message?: string);
+}

package/dist/gguf/errors/InvalidGGUFMagicError.js

@@ -0,0 +1,6 @@
+export default class InvalidGGUFMagicError extends Error {
+    constructor(message = "Invalid GGUF magic") {
+        super(message);
+    }
+}
+//# sourceMappingURL=InvalidGGUFMagicError.js.map

package/dist/gguf/errors/InvalidGGUFMagicError.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"InvalidGGUFMagicError.js","sourceRoot":"","sources":["../../../src/gguf/errors/InvalidGGUFMagicError.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,OAAO,qBAAsB,SAAQ,KAAK;IACpD,YAAmB,OAAO,GAAG,oBAAoB;QAC7C,KAAK,CAAC,OAAO,CAAC,CAAC;IACnB,CAAC;CACJ"}

package/dist/gguf/errors/MetadataNotParsedYetError.d.ts

@@ -0,0 +1,3 @@
+export default class MetadataNotParsedYetError extends Error {
+    constructor(path: string);
+}

package/dist/gguf/errors/MetadataNotParsedYetError.js

@@ -0,0 +1,6 @@
+export default class MetadataNotParsedYetError extends Error {
+    constructor(path) {
+        super(`Metadata not parsed yet: "${path}"`);
+    }
+}
+//# sourceMappingURL=MetadataNotParsedYetError.js.map

package/dist/gguf/errors/MetadataNotParsedYetError.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MetadataNotParsedYetError.js","sourceRoot":"","sources":["../../../src/gguf/errors/MetadataNotParsedYetError.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,OAAO,yBAA0B,SAAQ,KAAK;IACxD,YAAmB,IAAY;QAC3B,KAAK,CAAC,6BAA6B,IAAI,GAAG,CAAC,CAAC;IAChD,CAAC;CACJ"}

package/dist/gguf/errors/MissingNodeLlamaError.d.ts

@@ -0,0 +1,3 @@
+export default class MissingNodeLlamaError extends Error {
+    constructor(purpose: string);
+}

package/dist/gguf/errors/MissingNodeLlamaError.js

@@ -0,0 +1,6 @@
+export default class MissingNodeLlamaError extends Error {
+    constructor(purpose) {
+        super(`Missing nodeLlama options, this in required for ${purpose}`);
+    }
+}
+//# sourceMappingURL=MissingNodeLlamaError.js.map

package/dist/gguf/errors/MissingNodeLlamaError.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MissingNodeLlamaError.js","sourceRoot":"","sources":["../../../src/gguf/errors/MissingNodeLlamaError.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,OAAO,qBAAsB,SAAQ,KAAK;IACpD,YAAmB,OAAe;QAC9B,KAAK,CAAC,mDAAmD,OAAO,EAAE,CAAC,CAAC;IACxE,CAAC;CACJ"}

package/dist/gguf/errors/ModelScore/NotEnoughVRamError.d.ts

@@ -0,0 +1,5 @@
+export default class NotEnoughVRamError extends Error {
+    readonly requiredVRAM: number;
+    readonly availableVRAM: number;
+    constructor(requiredVRAM: number, availableVRAM: number);
+}

package/dist/gguf/errors/ModelScore/NotEnoughVRamError.js

@@ -0,0 +1,12 @@
+import prettyBytes from "pretty-bytes";
+export default class NotEnoughVRamError extends Error {
+    requiredVRAM;
+    availableVRAM;
+    constructor(requiredVRAM, availableVRAM) {
+        super();
+        this.availableVRAM = availableVRAM;
+        this.requiredVRAM = requiredVRAM;
+        this.message = `Not enough VRAM, require ${prettyBytes(requiredVRAM)}, but only ${prettyBytes(availableVRAM)} available`;
+    }
+}
+//# sourceMappingURL=NotEnoughVRamError.js.map

package/dist/gguf/errors/ModelScore/NotEnoughVRamError.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"NotEnoughVRamError.js","sourceRoot":"","sources":["../../../../src/gguf/errors/ModelScore/NotEnoughVRamError.ts"],"names":[],"mappings":"AAAA,OAAO,WAAW,MAAM,cAAc,CAAC;AAEvC,MAAM,CAAC,OAAO,OAAO,kBAAmB,SAAQ,KAAK;IACjC,YAAY,CAAS;IACrB,aAAa,CAAS;IAEtC,YAAmB,YAAoB,EAAE,aAAqB;QAC1D,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,OAAO,GAAG,4BAA4B,WAAW,CAAC,YAAY,CAAC,cAAc,WAAW,CAAC,aAAa,CAAC,YAAY,CAAC;IAC7H,CAAC;CACJ"}

package/dist/gguf/errors/UnsupportedMetadataTypeError.d.ts

@@ -0,0 +1,4 @@
+export default class UnsupportedMetadataTypeError extends Error {
+    readonly type: number;
+    constructor(type: number);
+}

package/dist/gguf/errors/UnsupportedMetadataTypeError.js

@@ -0,0 +1,8 @@
+export default class UnsupportedMetadataTypeError extends Error {
+    type;
+    constructor(type) {
+        super(`Unsupported metadata type: "${type}"`);
+        this.type = type;
+    }
+}
+//# sourceMappingURL=UnsupportedMetadataTypeError.js.map

package/dist/gguf/errors/UnsupportedMetadataTypeError.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"UnsupportedMetadataTypeError.js","sourceRoot":"","sources":["../../../src/gguf/errors/UnsupportedMetadataTypeError.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,OAAO,4BAA6B,SAAQ,KAAK;IAC3C,IAAI,CAAS;IAE7B,YAAmB,IAAY;QAC3B,KAAK,CAAC,+BAA+B,IAAI,GAAG,CAAC,CAAC;QAC9C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACrB,CAAC;CACJ"}

package/dist/gguf/ggufParser/GGUFParser.d.ts

@@ -0,0 +1,18 @@
+import { GGUFMetadataAny } from "./GGUFTypes.js";
+import GGUFBaseStream from "./stream/GGUFBaseStream.js";
+export type GGUFMetadataResponse = {
+    metadataSize: number;
+    metadata: GGUFMetadataAny;
+};
+export default class GGUFParser {
+    protected readonly _stream: GGUFBaseStream;
+    ignoreKeys: string[];
+    constructor(_stream: GGUFBaseStream, ignoreKeys?: string[]);
+    private _readMetadataValue;
+    private _parseMetadataRaw;
+    parseMetadata(): Promise<GGUFMetadataResponse>;
+    protected static _getNestedObject(key: string, currentNestedObject: any): {
+        lastObject: any;
+        lastKey: string;
+    };
+}