node-llama-cpp 3.0.0-beta.2 → 3.0.0-beta.20

package/dist/gguf/insights/utils/resolveContextContextSizeOption.js

@@ -0,0 +1,76 @@
+import { minAllowedContextSizeInCalculations } from "../../../config.js";
+import { getDefaultContextBatchSize, getDefaultModelContextSize } from "../../../evaluator/LlamaContext/LlamaContext.js";
+export function resolveContextContextSizeOption({ contextSize, batchSize, sequences, modelFileInsights, modelGpuLayers, modelTrainContextSize, getVramState, llamaGpu, ignoreMemorySafetyChecks = false, isEmbeddingContext = false }) {
+    if (contextSize == null)
+        contextSize = "auto";
+    if (typeof contextSize === "number") {
+        const resolvedContextSize = Math.max(1, Math.floor(contextSize));
+        if (ignoreMemorySafetyChecks)
+            return resolvedContextSize;
+        const vramState = getVramState();
+        const contextVram = modelFileInsights.estimateContextResourceRequirements({
+            contextSize: resolvedContextSize,
+            batchSize: batchSize ?? getDefaultContextBatchSize({ contextSize: resolvedContextSize, sequences }),
+            modelGpuLayers: modelGpuLayers,
+            sequences,
+            isEmbeddingContext
+        }).gpuVram;
+        if (contextVram > vramState.free)
+            throw new Error(`The context size of ${resolvedContextSize}${sequences > 1 ? ` with ${sequences} sequences` : ""} is too large for the available VRAM`);
+        return resolvedContextSize;
+    }
+    else if (contextSize === "auto" || typeof contextSize === "object") {
+        if (llamaGpu === false)
+            return modelTrainContextSize;
+        const vramState = getVramState();
+        if (vramState.total === 0)
+            return modelTrainContextSize;
+        const freeVram = vramState.free;
+        const maxContextSize = contextSize === "auto"
+            ? getDefaultModelContextSize({ trainContextSize: modelTrainContextSize })
+            : Math.min(contextSize.max ?? getDefaultModelContextSize({ trainContextSize: modelTrainContextSize }), getDefaultModelContextSize({ trainContextSize: modelTrainContextSize }));
+        const minContextSize = contextSize === "auto"
+            ? minAllowedContextSizeInCalculations
+            : Math.max(contextSize.min ?? minAllowedContextSizeInCalculations, minAllowedContextSizeInCalculations);
+        let highestCompatibleContextSize = null;
+        let step = -Math.max(1, Math.floor((maxContextSize - minContextSize) / 4));
+        for (let testContextSize = maxContextSize; testContextSize >= minContextSize && testContextSize <= maxContextSize;) {
+            const contextVram = modelFileInsights.estimateContextResourceRequirements({
+                contextSize: testContextSize,
+                batchSize: batchSize ?? getDefaultContextBatchSize({ contextSize: testContextSize, sequences }),
+                modelGpuLayers: modelGpuLayers,
+                sequences,
+                isEmbeddingContext
+            }).gpuVram;
+            if (contextVram <= freeVram) {
+                if (highestCompatibleContextSize == null || testContextSize > highestCompatibleContextSize) {
+                    highestCompatibleContextSize = testContextSize;
+                    if (step === -1)
+                        break;
+                    else if (step < 0)
+                        step = Math.max(1, Math.floor(-step / 2));
+                }
+            }
+            else if (step > 0)
+                step = -Math.max(1, Math.floor(step / 2));
+            if (testContextSize == minContextSize && step === -1)
+                break;
+            testContextSize += step;
+            if (testContextSize < minContextSize) {
+                testContextSize = minContextSize;
+                step = Math.max(1, Math.floor(Math.abs(step) / 2));
+            }
+            else if (testContextSize > maxContextSize) {
+                testContextSize = maxContextSize;
+                step = -Math.max(1, Math.floor(Math.abs(step) / 2));
+            }
+        }
+        if (highestCompatibleContextSize != null)
+            return highestCompatibleContextSize;
+        if (ignoreMemorySafetyChecks)
+            return minContextSize;
+        throw new Error(`The available VRAM is too small to fit the context size of ${maxContextSize}${sequences > 1 ? ` with ${sequences} sequences` : ""}`);
+    }
+    throw new Error(`Invalid context size: "${contextSize}"`);
+}
+//# sourceMappingURL=resolveContextContextSizeOption.js.map

package/dist/gguf/insights/utils/resolveContextContextSizeOption.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolveContextContextSizeOption.js","sourceRoot":"","sources":["../../../../src/gguf/insights/utils/resolveContextContextSizeOption.ts"],"names":[],"mappings":"AAGA,OAAO,EAAC,mCAAmC,EAAC,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAC,0BAA0B,EAAE,0BAA0B,EAAC,MAAM,iDAAiD,CAAC;AAEvH,MAAM,UAAU,+BAA+B,CAAC,EAC5C,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,iBAAiB,EAAE,cAAc,EAAE,qBAAqB,EAAE,YAAY,EAAE,QAAQ,EACnH,wBAAwB,GAAG,KAAK,EAAE,kBAAkB,GAAG,KAAK,EAY/D;IACG,IAAI,WAAW,IAAI,IAAI;QACnB,WAAW,GAAG,MAAM,CAAC;IAEzB,IAAI,OAAO,WAAW,KAAK,QAAQ,EAAE;QACjC,MAAM,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;QAEjE,IAAI,wBAAwB;YACxB,OAAO,mBAAmB,CAAC;QAE/B,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;QACjC,MAAM,WAAW,GAAG,iBAAiB,CAAC,mCAAmC,CAAC;YACtE,WAAW,EAAE,mBAAmB;YAChC,SAAS,EAAE,SAAS,IAAI,0BAA0B,CAAC,EAAC,WAAW,EAAE,mBAAmB,EAAE,SAAS,EAAC,CAAC;YACjG,cAAc,EAAE,cAAc;YAC9B,SAAS;YACT,kBAAkB;SACrB,CAAC,CAAC,OAAO,CAAC;QAEX,IAAI,WAAW,GAAG,SAAS,CAAC,IAAI;YAC5B,MAAM,IAAI,KAAK,CAAC,uBAAuB,mBAAmB,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,SAAS,YAAY,CAAC,CAAC,CAAC,EAAE,sCAAsC,CAAC,CAAC;QAE5J,OAAO,mBAAmB,CAAC;KAC9B;SAAM,IAAI,WAAW,KAAK,MAAM,IAAI,OAAO,WAAW,KAAK,QAAQ,EAAE;QAClE,IAAI,QAAQ,KAAK,KAAK;YAClB,OAAO,qBAAqB,CAAC;QAEjC,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;QAEjC,IAAI,SAAS,CAAC,KAAK,KAAK,CAAC;YACrB,OAAO,qBAAqB,CAAC;QAEjC,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC;QAEhC,MAAM,cAAc,GAAG,WAAW,KAAK,MAAM;YACzC,CAAC,CAAC,0BAA0B,CAAC,EAAC,gBAAgB,EAAE,qBAAqB,EAAC,CAAC;YACvE,CAAC,CAAC,IAAI,CAAC,GAAG,CACN,WAAW,CAAC,GAAG,IAAI,0BAA0B,CAAC,EAAC,gBAAgB,EAAE,qBAAqB,EAAC,CAAC,EACxF,0BAA0B,CAAC,EAAC,gBAAgB,EAAE,qBAAqB,EAAC,CAAC,CACxE,CAAC;QAEN,MAAM,cAAc,GAAG,WAAW,KAAK,MAAM;YACzC,CAAC,CAAC,mCAAmC;YACrC,CAAC,CAAC,IAAI,CAAC,GAAG,CACN,WAAW,CAAC,GAAG,IAAI,mCAAmC,EACtD,mCAAmC,CACtC,CAAC;QAEN,IAAI,4BAA4B,GAAkB,IAAI,CAAC;QACvD,IAAI,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC3E,KAAK,IAAI,eAAe,GAAG,cAAc,EAAE,eAAe,IAAI,cAAc,IAAI,eAAe,IAAI,cAAc,GAAG;YAChH,MAAM,WAAW,GAAG,iBAAiB,CAAC,mCAAmC,CAAC;gBACtE,WAAW,EAAE,eAAe;gBAC5B,SAAS,EAAE,SAAS,IAAI,0BAA0B,CAAC,EAAC,WAAW,EAAE,eAAe,EAAE,SAAS,EAAC,CAAC;gBAC7F,cAAc,EAAE,cAAc;gBAC9B,SAAS;gBACT,kBAAkB;aACrB,CAAC,CAAC,OAAO,CAAC;YAEX,IAAI,WAAW,IAAI,QAAQ,EAAE;gBACzB,IAAI,4BAA4B,IAAI,IAAI,IAAI,eAAe,GAAG,4BAA4B,EAAE;oBACxF,4BAA4B,GAAG,eAAe,CAAC;oBAE/C,IAAI,IAAI,KAAK,CAAC,CAAC;wBACX,MAAM;yBACL,IAAI,IAAI,GAAG,CAAC;wBACb,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;iBACjD;aACJ;iBAAM,IAAI,IAAI,GAAG,CAAC;gBACf,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;YAE9C,IAAI,eAAe,IAAI,cAAc,IAAI,IAAI,KAAK,CAAC,CAAC;gBAChD,MAAM;YAEV,eAAe,IAAI,IAAI,CAAC;YACxB,IAAI,eAAe,GAAG,cAAc,EAAE;gBAClC,eAAe,GAAG,cAAc,CAAC;gBACjC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;aACtD;iBAAM,IAAI,eAAe,GAAG,cAAc,EAAE;gBACzC,eAAe,GAAG,cAAc,CAAC;gBACjC,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;aACvD;SACJ;QAED,IAAI,4BAA4B,IAAI,IAAI;YACpC,OAAO,4BAA4B,CAAC;QAExC,IAAI,wBAAwB;YACxB,OAAO,cAAc,CAAC;QAE1B,MAAM,IAAI,KAAK,CAAC,8DAA8D,cAAc,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,SAAS,YAAY,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;KACzJ;IAED,MAAM,IAAI,KAAK,CAAC,0BAA0B,WAAW,GAAG,CAAC,CAAC;AAC9D,CAAC"}

package/dist/gguf/insights/utils/resolveModelGpuLayersOption.d.ts

@@ -0,0 +1,14 @@
+import { LlamaModelOptions } from "../../../evaluator/LlamaModel.js";
+import { BuildGpu } from "../../../bindings/types.js";
+import type { GgufInsights } from "../GgufInsights.js";
+export declare function resolveModelGpuLayersOption(gpuLayers: LlamaModelOptions["gpuLayers"], { ggufInsights, ignoreMemorySafetyChecks, getVramState, llamaVramPaddingSize, llamaGpu, llamaSupportsGpuOffloading }: {
+    ggufInsights: GgufInsights;
+    ignoreMemorySafetyChecks?: boolean;
+    getVramState(): {
+        total: number;
+        free: number;
+    };
+    llamaVramPaddingSize: number;
+    llamaGpu: BuildGpu;
+    llamaSupportsGpuOffloading: boolean;
+}): number;

package/dist/gguf/insights/utils/resolveModelGpuLayersOption.js

@@ -0,0 +1,177 @@
+import { InsufficientMemoryError } from "../../../utils/InsufficientMemoryError.js";
+import { findBestOption } from "../../../utils/findBestOption.js";
+import { getDefaultContextBatchSize, getDefaultModelContextSize } from "../../../evaluator/LlamaContext/LlamaContext.js";
+import { minAllowedContextSizeInCalculations } from "../../../config.js";
+import { scoreLevels } from "./scoreLevels.js";
+const fitContextExtraMemoryPaddingPercentage = 0.5;
+export function resolveModelGpuLayersOption(gpuLayers, { ggufInsights, ignoreMemorySafetyChecks = false, getVramState, llamaVramPaddingSize, llamaGpu, llamaSupportsGpuOffloading }) {
+    if (gpuLayers == null)
+        gpuLayers = "auto";
+    if (!llamaSupportsGpuOffloading)
+        return 0;
+    if (gpuLayers === "max" || typeof gpuLayers === "number") {
+        const resolvedGpuLayers = typeof gpuLayers === "number"
+            ? Math.max(0, Math.min(ggufInsights.totalLayers, gpuLayers))
+            : ggufInsights.totalLayers;
+        if (ignoreMemorySafetyChecks)
+            return resolvedGpuLayers;
+        const vramState = getVramState();
+        const maxLayersRequirements = getVramRequiredForGpuLayers({
+            gpuLayers: resolvedGpuLayers,
+            ggufInsights,
+            currentVram: vramState.free
+        });
+        if (maxLayersRequirements == null)
+            throw new InsufficientMemoryError("Not enough VRAM to fit the model with the specified settings");
+        return resolvedGpuLayers;
+    }
+    else if (gpuLayers === "auto" || typeof gpuLayers === "object") {
+        if (llamaGpu === false)
+            return 0;
+        const vramState = getVramState();
+        if (vramState.total === 0)
+            return 0;
+        let freeVram = vramState.free;
+        if (typeof gpuLayers === "object" && gpuLayers.fitContext?.contextSize != null) {
+            freeVram -= llamaVramPaddingSize * fitContextExtraMemoryPaddingPercentage;
+            if (freeVram < 0)
+                freeVram = 0;
+        }
+        const bestGpuLayersOption = getBestGpuLayersForFreeVram({
+            ggufInsights,
+            freeVram,
+            fitContext: typeof gpuLayers === "object"
+                ? gpuLayers.fitContext
+                : undefined,
+            minGpuLayers: typeof gpuLayers === "object"
+                ? gpuLayers.min
+                : undefined,
+            maxGpuLayers: typeof gpuLayers === "object"
+                ? gpuLayers.max
+                : undefined
+        });
+        const hasGpuLayersRequirements = typeof gpuLayers === "object" &&
+            (gpuLayers.min != null || gpuLayers.max != null || gpuLayers.fitContext?.contextSize != null);
+        if (!ignoreMemorySafetyChecks && bestGpuLayersOption == null && hasGpuLayersRequirements)
+            throw new InsufficientMemoryError("Not enough VRAM to fit the model with the specified settings");
+        return bestGpuLayersOption ?? 0;
+    }
+    throw new Error(`Invalid gpuLayers value: ${gpuLayers}`);
+}
+function getBestGpuLayersForFreeVram({ ggufInsights, freeVram, fitContext, minGpuLayers, maxGpuLayers }) {
+    return findBestOption({
+        *generator() {
+            const minLayers = Math.floor(Math.max(0, minGpuLayers ?? 0));
+            const maxLayers = Math.floor(Math.min(ggufInsights.totalLayers, maxGpuLayers ?? ggufInsights.totalLayers));
+            for (let layers = maxLayers; layers >= minLayers; layers--) {
+                yield {
+                    gpuLayers: layers
+                };
+            }
+        },
+        score(option) {
+            const layersRequirements = getVramRequiredForGpuLayers({
+                gpuLayers: option.gpuLayers,
+                ggufInsights,
+                currentVram: freeVram,
+                fitContext
+            });
+            if (layersRequirements == null)
+                return null;
+            return scoreGpuLayersAndContextCombination({ gpuLayers: option.gpuLayers, contextSize: layersRequirements.contextSize }, {
+                totalGpuLayers: ggufInsights.totalLayers,
+                trainContextSize: getDefaultModelContextSize({ trainContextSize: ggufInsights.trainContextSize })
+            });
+        }
+    })?.gpuLayers ?? null;
+}
+function scoreGpuLayersAndContextCombination({ gpuLayers, contextSize }, { totalGpuLayers, trainContextSize }) {
+    function scoreGpuLayers() {
+        return scoreLevels(gpuLayers, [{
+                start: 0,
+                points: 4
+            }, {
+                start: 1,
+                points: 26
+            }, {
+                start: totalGpuLayers,
+                points: 14,
+                end: totalGpuLayers
+            }]);
+    }
+    function scoreContextSize() {
+        const gpuLayersPercentage = gpuLayers / totalGpuLayers;
+        return scoreLevels(contextSize, [{
+                start: 0,
+                points: 2
+            }, {
+                start: 1024,
+                points: 4
+            }, {
+                start: 2048,
+                points: gpuLayersPercentage < 0.1 ? 1 : 8
+            }, {
+                start: 4096,
+                points: gpuLayersPercentage < 0.3 ? 4 : 16
+            }, {
+                start: 8192,
+                points: gpuLayersPercentage < 0.6 ? 1 : 8,
+                end: Math.max(trainContextSize, 16384)
+            }]);
+    }
+    return scoreGpuLayers() + scoreContextSize();
+}
+function getVramRequiredForGpuLayers({ gpuLayers, ggufInsights, currentVram, fitContext }) {
+    const modelVram = ggufInsights.estimateModelResourceRequirements({ gpuLayers }).gpuVram;
+    if (modelVram > currentVram)
+        return null;
+    if (fitContext != null && fitContext.contextSize != null) {
+        const contextVram = ggufInsights.estimateContextResourceRequirements({
+            contextSize: fitContext.contextSize,
+            batchSize: getDefaultContextBatchSize({ contextSize: fitContext.contextSize, sequences: 1 }),
+            modelGpuLayers: gpuLayers,
+            sequences: 1,
+            isEmbeddingContext: fitContext.embeddingContext ?? false
+        }).gpuVram;
+        const totalVram = modelVram + contextVram;
+        if (totalVram > currentVram)
+            return null;
+        return {
+            contextSize: fitContext.contextSize,
+            contextVram,
+            totalVram
+        };
+    }
+    const maxContext = findMaxPossibleContextSizeForVram({
+        gpuLayers,
+        ggufInsights,
+        vram: currentVram - modelVram,
+        isEmbeddingContext: fitContext?.embeddingContext ?? false
+    });
+    if (maxContext == null || modelVram + maxContext.vram > currentVram)
+        return null;
+    return {
+        contextSize: maxContext.contextSize,
+        contextVram: maxContext.vram,
+        totalVram: modelVram + maxContext.vram
+    };
+}
+function findMaxPossibleContextSizeForVram({ gpuLayers, ggufInsights, vram, isEmbeddingContext }) {
+    const maxContextSize = getDefaultModelContextSize({ trainContextSize: ggufInsights.trainContextSize });
+    for (let contextSize = maxContextSize; contextSize >= minAllowedContextSizeInCalculations; contextSize--) {
+        const contextVram = ggufInsights.estimateContextResourceRequirements({
+            contextSize,
+            batchSize: getDefaultContextBatchSize({ contextSize, sequences: 1 }),
+            modelGpuLayers: gpuLayers,
+            sequences: 1,
+            isEmbeddingContext
+        }).gpuVram;
+        if (contextVram <= vram)
+            return {
+                contextSize,
+                vram: contextVram
+            };
+    }
+    return null;
+}
+//# sourceMappingURL=resolveModelGpuLayersOption.js.map

package/dist/gguf/insights/utils/resolveModelGpuLayersOption.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolveModelGpuLayersOption.js","sourceRoot":"","sources":["../../../../src/gguf/insights/utils/resolveModelGpuLayersOption.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,uBAAuB,EAAC,MAAM,2CAA2C,CAAC;AAClF,OAAO,EAAC,cAAc,EAAC,MAAM,kCAAkC,CAAC;AAChE,OAAO,EAAC,0BAA0B,EAAE,0BAA0B,EAAC,MAAM,iDAAiD,CAAC;AACvH,OAAO,EAAC,mCAAmC,EAAC,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAC,WAAW,EAAC,MAAM,kBAAkB,CAAC;AAG7C,MAAM,sCAAsC,GAAG,GAAG,CAAC;AAEnD,MAAM,UAAU,2BAA2B,CAAC,SAAyC,EAAE,EACnF,YAAY,EAAE,wBAAwB,GAAG,KAAK,EAAE,YAAY,EAAE,oBAAoB,EAClF,QAAQ,EAAE,0BAA0B,EAIvC;IACG,IAAI,SAAS,IAAI,IAAI;QACjB,SAAS,GAAG,MAAM,CAAC;IAEvB,IAAI,CAAC,0BAA0B;QAC3B,OAAO,CAAC,CAAC;IAEb,IAAI,SAAS,KAAK,KAAK,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE;QACtD,MAAM,iBAAiB,GAAG,OAAO,SAAS,KAAK,QAAQ;YACnD,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;YAC5D,CAAC,CAAC,YAAY,CAAC,WAAW,CAAC;QAE/B,IAAI,wBAAwB;YACxB,OAAO,iBAAiB,CAAC;QAE7B,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;QACjC,MAAM,qBAAqB,GAAG,2BAA2B,CAAC;YACtD,SAAS,EAAE,iBAAiB;YAC5B,YAAY;YACZ,WAAW,EAAE,SAAS,CAAC,IAAI;SAC9B,CAAC,CAAC;QAEH,IAAI,qBAAqB,IAAI,IAAI;YAC7B,MAAM,IAAI,uBAAuB,CAAC,8DAA8D,CAAC,CAAC;QAEtG,OAAO,iBAAiB,CAAC;KAC5B;SAAM,IAAI,SAAS,KAAK,MAAM,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE;QAC9D,IAAI,QAAQ,KAAK,KAAK;YAClB,OAAO,CAAC,CAAC;QAEb,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;QACjC,IAAI,SAAS,CAAC,KAAK,KAAK,CAAC;YACrB,OAAO,CAAC,CAAC;QAEb,IAAI,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC;QAC9B,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,CAAC,UAAU,EAAE,WAAW,IAAI,IAAI,EAAE;YAC5E,QAAQ,IAAI,oBAAoB,GAAG,sCAAsC,CAAC;YAE1E,IAAI,QAAQ,GAAG,CAAC;gBACZ,QAAQ,GAAG,CAAC,CAAC;SACpB;QAED,MAAM,mBAAmB,GAAG,2BAA2B,CAAC;YACpD,YAAY;YACZ,QAAQ;YACR,UAAU,EAAE,OAAO,SAAS,KAAK,QAAQ;gBACrC,CAAC,CAAC,SAAS,CAAC,UAAU;gBACtB,CAAC,CAAC,SAAS;YACf,YAAY,EAAE,OAAO,SAAS,KAAK,QAAQ;gBACvC,CAAC,CAAC,SAAS,CAAC,GAAG;gBACf,CAAC,CAAC,SAAS;YACf,YAAY,EAAE,OAAO,SAAS,KAAK,QAAQ;gBACvC,CAAC,CAAC,SAAS,CAAC,GAAG;gBACf,CAAC,CAAC,SAAS;SAClB,CAAC,CAAC;QAEH,MAAM,wBAAwB,GAAG,OAAO,SAAS,KAAK,QAAQ;YAC1D,CAAC,SAAS,CAAC,GAAG,IAAI,IAAI,IAAI,SAAS,CAAC,GAAG,IAAI,IAAI,IAAI,SAAS,CAAC,UAAU,EAAE,WAAW,IAAI,IAAI,CAAC,CAAC;QAElG,IAAI,CAAC,wBAAwB,IAAI,mBAAmB,IAAI,IAAI,IAAI,wBAAwB;YACpF,MAAM,IAAI,uBAAuB,CAAC,8DAA8D,CAAC,CAAC;QAEtG,OAAO,mBAAmB,IAAI,CAAC,CAAC;KACnC;IAED,MAAM,IAAI,KAAK,CAAC,4BAA4B,SAAS,EAAE,CAAC,CAAC;AAC7D,CAAC;AAED,SAAS,2BAA2B,CAAC,EACjC,YAAY,EACZ,QAAQ,EACR,UAAU,EACV,YAAY,EACZ,YAAY,EAOf;IACG,OAAO,cAAc,CAAC;QAClB,CAAC,SAAS;YACN,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,IAAI,CAAC,CAAC,CAAC,CAAC;YAC7D,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,WAAW,EAAE,YAAY,IAAI,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC;YAE3G,KAAK,IAAI,MAAM,GAAG,SAAS,EAAE,MAAM,IAAI,SAAS,EAAE,MAAM,EAAE,EAAE;gBACxD,MAAM;oBACF,SAAS,EAAE,MAAM;iBACpB,CAAC;aACL;QACL,CAAC;QACD,KAAK,CAAC,MAAM;YACR,MAAM,kBAAkB,GAAG,2BAA2B,CAAC;gBACnD,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,YAAY;gBACZ,WAAW,EAAE,QAAQ;gBACrB,UAAU;aACb,CAAC,CAAC;YAEH,IAAI,kBAAkB,IAAI,IAAI;gBAC1B,OAAO,IAAI,CAAC;YAEhB,OAAO,mCAAmC,CAAC,EAAC,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,WAAW,EAAE,kBAAkB,CAAC,WAAW,EAAC,EAAE;gBACnH,cAAc,EAAE,YAAY,CAAC,WAAW;gBACxC,gBAAgB,EAAE,0BAA0B,CAAC,EAAC,gBAAgB,EAAE,YAAY,CAAC,gBAAgB,EAAC,CAAC;aAClG,CAAC,CAAC;QACP,CAAC;KACJ,CAAC,EAAE,SAAS,IAAI,IAAI,CAAC;AAC1B,CAAC;AAED,SAAS,mCAAmC,CAAC,EAAC,SAAS,EAAE,WAAW,EAA2C,EAAE,EAC7G,cAAc,EAAE,gBAAgB,EAGnC;IACG,SAAS,cAAc;QACnB,OAAO,WAAW,CAAC,SAAS,EAAE,CAAC;gBAC3B,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,CAAC;aACZ,EAAE;gBACC,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,EAAE;aACb,EAAE;gBACC,KAAK,EAAE,cAAc;gBACrB,MAAM,EAAE,EAAE;gBACV,GAAG,EAAE,cAAc;aACtB,CAAC,CAAC,CAAC;IACR,CAAC;IAED,SAAS,gBAAgB;QACrB,MAAM,mBAAmB,GAAG,SAAS,GAAG,cAAc,CAAC;QAEvD,OAAO,WAAW,CAAC,WAAW,EAAE,CAAC;gBAC7B,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,CAAC;aACZ,EAAE;gBACC,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,CAAC;aACZ,EAAE;gBACC,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,mBAAmB,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;aAC5C,EAAE;gBACC,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,mBAAmB,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;aAC7C,EAAE;gBACC,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,mBAAmB,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACzC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,KAAK,CAAC;aACzC,CAAC,CAAC,CAAC;IACR,CAAC;IAED,OAAO,cAAc,EAAE,GAAG,gBAAgB,EAAE,CAAC;AACjD,CAAC;AAED,SAAS,2BAA2B,CAAC,EACjC,SAAS,EAAE,YAAY,EAAE,WAAW,EAAE,UAAU,EAGnD;IACG,MAAM,SAAS,GAAG,YAAY,CAAC,iCAAiC,CAAC,EAAC,SAAS,EAAC,CAAC,CAAC,OAAO,CAAC;IAEtF,IAAI,SAAS,GAAG,WAAW;QACvB,OAAO,IAAI,CAAC;IAEhB,IAAI,UAAU,IAAI,IAAI,IAAI,UAAU,CAAC,WAAW,IAAI,IAAI,EAAE;QACtD,MAAM,WAAW,GAAG,YAAY,CAAC,mCAAmC,CAAC;YACjE,WAAW,EAAE,UAAU,CAAC,WAAW;YACnC,SAAS,EAAE,0BAA0B,CAAC,EAAC,WAAW,EAAE,UAAU,CAAC,WAAW,EAAE,SAAS,EAAE,CAAC,EAAC,CAAC;YAC1F,cAAc,EAAE,SAAS;YACzB,SAAS,EAAE,CAAC;YACZ,kBAAkB,EAAE,UAAU,CAAC,gBAAgB,IAAI,KAAK;SAC3D,CAAC,CAAC,OAAO,CAAC;QAEX,MAAM,SAAS,GAAG,SAAS,GAAG,WAAW,CAAC;QAC1C,IAAI,SAAS,GAAG,WAAW;YACvB,OAAO,IAAI,CAAC;QAEhB,OAAO;YACH,WAAW,EAAE,UAAU,CAAC,WAAW;YACnC,WAAW;YACX,SAAS;SACZ,CAAC;KACL;IAED,MAAM,UAAU,GAAG,iCAAiC,CAAC;QACjD,SAAS;QACT,YAAY;QACZ,IAAI,EAAE,WAAW,GAAG,SAAS;QAC7B,kBAAkB,EAAE,UAAU,EAAE,gBAAgB,IAAI,KAAK;KAC5D,CAAC,CAAC;IAEH,IAAI,UAAU,IAAI,IAAI,IAAI,SAAS,GAAG,UAAU,CAAC,IAAI,GAAG,WAAW;QAC/D,OAAO,IAAI,CAAC;IAEhB,OAAO;QACH,WAAW,EAAE,UAAU,CAAC,WAAW;QACnC,WAAW,EAAE,UAAU,CAAC,IAAI;QAC5B,SAAS,EAAE,SAAS,GAAG,UAAU,CAAC,IAAI;KACzC,CAAC;AACN,CAAC;AAED,SAAS,iCAAiC,CAAC,EAAC,SAAS,EAAE,YAAY,EAAE,IAAI,EAAE,kBAAkB,EAE5F;IACG,MAAM,cAAc,GAAG,0BAA0B,CAAC,EAAC,gBAAgB,EAAE,YAAY,CAAC,gBAAgB,EAAC,CAAC,CAAC;IAErG,KAAK,IAAI,WAAW,GAAG,cAAc,EAAE,WAAW,IAAI,mCAAmC,EAAE,WAAW,EAAE,EAAE;QACtG,MAAM,WAAW,GAAG,YAAY,CAAC,mCAAmC,CAAC;YACjE,WAAW;YACX,SAAS,EAAE,0BAA0B,CAAC,EAAC,WAAW,EAAE,SAAS,EAAE,CAAC,EAAC,CAAC;YAClE,cAAc,EAAE,SAAS;YACzB,SAAS,EAAE,CAAC;YACZ,kBAAkB;SACrB,CAAC,CAAC,OAAO,CAAC;QAEX,IAAI,WAAW,IAAI,IAAI;YACnB,OAAO;gBACH,WAAW;gBACX,IAAI,EAAE,WAAW;aACpB,CAAC;KACT;IAED,OAAO,IAAI,CAAC;AAChB,CAAC"}

package/dist/gguf/insights/utils/scoreLevels.d.ts

@@ -0,0 +1,5 @@
+export declare function scoreLevels(num: number, levels: {
+    start: number;
+    end?: number;
+    points: number;
+}[]): number;

package/dist/gguf/insights/utils/scoreLevels.js

@@ -0,0 +1,16 @@
+export function scoreLevels(num, levels) {
+    let res = 0;
+    for (let i = 0; i < levels.length; i++) {
+        const level = levels[i];
+        const start = level.start;
+        const end = level.end ?? levels[i + 1]?.start ?? Math.max(start, num);
+        if (num < start)
+            break;
+        else if (num >= end)
+            res += level.points;
+        else
+            res += level.points * ((num - start) / (end - start));
+    }
+    return res;
+}
+//# sourceMappingURL=scoreLevels.js.map

package/dist/gguf/insights/utils/scoreLevels.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scoreLevels.js","sourceRoot":"","sources":["../../../../src/gguf/insights/utils/scoreLevels.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,WAAW,CAAC,GAAW,EAAE,MAAyD;IAC9F,IAAI,GAAG,GAAG,CAAC,CAAC;IAEZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QACpC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACxB,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;QAC1B,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,IAAI,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAEtE,IAAI,GAAG,GAAG,KAAK;YACX,MAAM;aACL,IAAI,GAAG,IAAI,GAAG;YACf,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC;;YAEpB,GAAG,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC;KAC7D;IAED,OAAO,GAAG,CAAC;AACf,CAAC"}

package/dist/gguf/parser/GgufV2Parser.d.ts

@@ -0,0 +1,19 @@
+import { GgufReadOffset } from "../utils/GgufReadOffset.js";
+import { GgufValueType, GgufVersionParserOptions, GgufVersionParserResult, MetadataKeyValueRecord, MetadataValue } from "../types/GgufFileInfoTypes.js";
+export declare class GgufV2Parser {
+    private readonly _fileReader;
+    private readonly _shouldReadTensorInfo;
+    private readonly _ignoreKeys;
+    private readonly _readOffset;
+    private readonly _logWarnings;
+    constructor({ fileReader, readTensorInfo, ignoreKeys, readOffset, logWarnings }: GgufVersionParserOptions);
+    parse(): Promise<GgufVersionParserResult>;
+    protected _readGgufValue(type: GgufValueType, offset: number | GgufReadOffset): Promise<MetadataValue>;
+    protected _readStringValue(offset: number | GgufReadOffset): Promise<string>;
+    protected _readRawHeader(readOffset: GgufReadOffset): Promise<{
+        tensorCount: number | bigint;
+        metadata: MetadataKeyValueRecord;
+        headerSize: number;
+    }>;
+    private _readTensorInfo;
+}

package/dist/gguf/parser/GgufV2Parser.js

@@ -0,0 +1,115 @@
+import { GgufFileReader, valueTypeToBytesToRead } from "../fileReaders/GgufFileReader.js";
+import { GgufReadOffset } from "../utils/GgufReadOffset.js";
+import { UnsupportedGgufValueTypeError } from "../errors/UnsupportedGgufValueTypeError.js";
+import { GgufValueType } from "../types/GgufFileInfoTypes.js";
+import { convertMetadataKeyValueRecordToNestedObject } from "../utils/convertMetadataKeyValueRecordToNestedObject.js";
+export class GgufV2Parser {
+    _fileReader;
+    _shouldReadTensorInfo;
+    _ignoreKeys;
+    _readOffset;
+    _logWarnings;
+    constructor({ fileReader, readTensorInfo = true, ignoreKeys = [], readOffset, logWarnings }) {
+        this._fileReader = fileReader;
+        this._shouldReadTensorInfo = readTensorInfo;
+        this._ignoreKeys = ignoreKeys;
+        this._readOffset = readOffset;
+        this._logWarnings = logWarnings;
+    }
+    async parse() {
+        const readOffset = this._readOffset;
+        const initialOffset = readOffset.offset;
+        const headerReadResult = await this._readRawHeader(readOffset);
+        const tensorReadResult = this._shouldReadTensorInfo
+            ? await this._readTensorInfo(headerReadResult.tensorCount, readOffset)
+            : null;
+        const metadata = convertMetadataKeyValueRecordToNestedObject(headerReadResult.metadata, {
+            logOverrideWarnings: this._logWarnings,
+            ignoreKeys: this._ignoreKeys
+        });
+        return {
+            tensorCount: headerReadResult.tensorCount,
+            metadata: metadata,
+            tensorInfo: tensorReadResult?.tensorInfo,
+            metadataSize: headerReadResult.headerSize + initialOffset,
+            tensorInfoSize: tensorReadResult?.tensorInfoSize
+        };
+    }
+    async _readGgufValue(type, offset) {
+        const readOffset = GgufReadOffset.resolveReadOffset(offset);
+        switch (type) {
+            case GgufValueType.Uint8: return await this._fileReader.readUint8(readOffset);
+            case GgufValueType.Int8: return await this._fileReader.readInt8(readOffset);
+            case GgufValueType.Uint16: return await this._fileReader.readUint16(readOffset);
+            case GgufValueType.Int16: return await this._fileReader.readInt16(readOffset);
+            case GgufValueType.Uint32: return await this._fileReader.readUint32(readOffset);
+            case GgufValueType.Int32: return await this._fileReader.readInt32(readOffset);
+            case GgufValueType.Float32: return await this._fileReader.readFloat32(readOffset);
+            case GgufValueType.Bool: return await this._fileReader.readBool(readOffset);
+            case GgufValueType.String: return await this._readStringValue(readOffset);
+            case GgufValueType.Uint64: return await this._fileReader.readUint64(readOffset);
+            case GgufValueType.Int64: return await this._fileReader.readInt64(readOffset);
+            case GgufValueType.Float64: return await this._fileReader.readFloat64(readOffset);
+        }
+        if (type === GgufValueType.Array) {
+            const arrayType = await this._fileReader.readUint32(readOffset);
+            const arrayLength = await this._fileReader.readUint64(readOffset);
+            const arrayValues = [];
+            for (let i = 0; i < arrayLength; i++) {
+                const value = await this._readGgufValue(arrayType, readOffset);
+                arrayValues.push(value);
+            }
+            return arrayValues;
+        }
+        throw new UnsupportedGgufValueTypeError(type);
+    }
+    async _readStringValue(offset) {
+        const readOffset = GgufReadOffset.resolveReadOffset(offset);
+        const length = Number(await this._fileReader.readUint64(readOffset));
+        const readLength = valueTypeToBytesToRead.uint8 * length;
+        const stringBytes = await this._fileReader.readByteRange(readOffset, readLength);
+        return String.fromCharCode(...stringBytes);
+    }
+    async _readRawHeader(readOffset) {
+        const initialOffset = readOffset.offset;
+        const tensorCount = await this._fileReader.readUint64(readOffset);
+        const metadataKVCount = Number(await this._fileReader.readUint64(readOffset));
+        const metadata = {};
+        for (let i = 0; i < metadataKVCount; i++) {
+            const keyResult = await this._readStringValue(readOffset);
+            const valueType = await this._fileReader.readUint32(readOffset);
+            metadata[keyResult] = await this._readGgufValue(valueType, readOffset);
+        }
+        return {
+            tensorCount: GgufFileReader.castNumberIfSafe(tensorCount),
+            metadata: metadata,
+            headerSize: readOffset.offset - initialOffset
+        };
+    }
+    async _readTensorInfo(tensorCount, readOffset) {
+        const initialOffset = readOffset.offset;
+        const tensorInfo = [];
+        for (let i = 0n; i < BigInt(tensorCount); i++) {
+            const name = await this._readStringValue(readOffset);
+            const dimensionsNumber = await this._fileReader.readUint32(readOffset);
+            const dimensions = [];
+            for (let i = 0; i < dimensionsNumber; i++) {
+                const dimension = await this._fileReader.readUint64(readOffset);
+                dimensions.push(GgufFileReader.castNumberIfSafe(dimension));
+            }
+            const ggmlType = await this._fileReader.readUint32(readOffset);
+            const offset = await this._fileReader.readUint64(readOffset);
+            tensorInfo.push({
+                name,
+                dimensions,
+                ggmlType: ggmlType,
+                offset: GgufFileReader.castNumberIfSafe(offset)
+            });
+        }
+        return {
+            tensorInfo,
+            tensorInfoSize: readOffset.offset - initialOffset
+        };
+    }
+}
+//# sourceMappingURL=GgufV2Parser.js.map

package/dist/gguf/parser/GgufV2Parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GgufV2Parser.js","sourceRoot":"","sources":["../../../src/gguf/parser/GgufV2Parser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,cAAc,EAAE,sBAAsB,EAAC,MAAM,kCAAkC,CAAC;AACxF,OAAO,EAAC,cAAc,EAAC,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAC,6BAA6B,EAAC,MAAM,4CAA4C,CAAC;AACzF,OAAO,EACH,aAAa,EAChB,MAAM,+BAA+B,CAAC;AAGvC,OAAO,EAAC,2CAA2C,EAAC,MAAM,yDAAyD,CAAC;AAEpH,MAAM,OAAO,YAAY;IACJ,WAAW,CAAiB;IAC5B,qBAAqB,CAAU;IAC/B,WAAW,CAAW;IACtB,WAAW,CAAiB;IAC5B,YAAY,CAAU;IAEvC,YAAmB,EAAC,UAAU,EAAE,cAAc,GAAG,IAAI,EAAE,UAAU,GAAG,EAAE,EAAE,UAAU,EAAE,WAAW,EAA2B;QACtH,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,IAAI,CAAC,qBAAqB,GAAG,cAAc,CAAC;QAC5C,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;IACpC,CAAC;IAEM,KAAK,CAAC,KAAK;QACd,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC;QACpC,MAAM,aAAa,GAAG,UAAU,CAAC,MAAM,CAAC;QAExC,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;QAC/D,MAAM,gBAAgB,GAAG,IAAI,CAAC,qBAAqB;YAC/C,CAAC,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,gBAAgB,CAAC,WAAW,EAAE,UAAU,CAAC;YACtE,CAAC,CAAC,IAAI,CAAC;QACX,MAAM,QAAQ,GAAG,2CAA2C,CAAC,gBAAgB,CAAC,QAAQ,EAAE;YACpF,mBAAmB,EAAE,IAAI,CAAC,YAAY;YACtC,UAAU,EAAE,IAAI,CAAC,WAAW;SAC/B,CAAC,CAAC;QAEH,OAAO;YACH,WAAW,EAAE,gBAAgB,CAAC,WAAW;YACzC,QAAQ,EAAE,QAA+B;YACzC,UAAU,EAAE,gBAAgB,EAAE,UAAU;YACxC,YAAY,EAAE,gBAAgB,CAAC,UAAU,GAAG,aAAa;YACzD,cAAc,EAAE,gBAAgB,EAAE,cAAc;SACnD,CAAC;IACN,CAAC;IAES,KAAK,CAAC,cAAc,CAAC,IAAmB,EAAE,MAA+B;QAC/E,MAAM,UAAU,GAAG,cAAc,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAE5D,QAAQ,IAAI,EAAE;YACV,KAAK,aAAa,CAAC,KAAK,CAAC,CAAC,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YAC9E,KAAK,aAAa,CAAC,IAAI,CAAC,CAAC,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YAC5E,KAAK,aAAa,CAAC,MAAM,CAAC,CAAC,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;YAChF,KAAK,aAAa,CAAC,KAAK,CAAC,CAAC,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YAC9E,KAAK,aAAa,CAAC,MAAM,CAAC,CAAC,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;YAChF,KAAK,aAAa,CAAC,KAAK,CAAC,CAAC,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YAC9E,KAAK,aAAa,CAAC,OAAO,CAAC,CAAC,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;YAClF,KAAK,aAAa,CAAC,IAAI,CAAC,CAAC,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YAC5E,KAAK,aAAa,CAAC,MAAM,CAAC,CAAC,OAAO,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;YAC1E,KAAK,aAAa,CAAC,MAAM,CAAC,CAAC,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;YAChF,KAAK,aAAa,CAAC,KAAK,CAAC,CAAC,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YAC9E,KAAK,aAAa,CAAC,OAAO,CAAC,CAAC,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;SACrF;QAED,IAAI,IAAI,KAAK,aAAa,CAAC,KAAK,EAAE;YAC9B,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;YAChE,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;YAElE,MAAM,WAAW,GAAoB,EAAE,CAAC;YACxC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE;gBAClC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;gBAC/D,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;aAC3B;YACD,OAAO,WAAW,CAAC;SACtB;QAED,MAAM,IAAI,6BAA6B,CAAC,IAAI,CAAC,CAAC;IAClD,CAAC;IAES,KAAK,CAAC,gBAAgB,CAAC,MAA+B;QAC5D,MAAM,UAAU,GAAG,cAAc,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC;QAErE,MAAM,UAAU,GAAG,sBAAsB,CAAC,KAAK,GAAG,MAAM,CAAC;QACzD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QAEjF,OAAO,MAAM,CAAC,YAAY,CAAC,GAAG,WAAW,CAAC,CAAC;IAC/C,CAAC;IAES,KAAK,CAAC,cAAc,CAAC,UAA0B;QACrD,MAAM,aAAa,GAAG,UAAU,CAAC,MAAM,CAAC;QAExC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QAClE,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC;QAE9E,MAAM,QAAQ,GAA2B,EAAE,CAAC;QAE5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,eAAe,EAAE,CAAC,EAAE,EAAE;YACtC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;YAC1D,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;YAChE,QAAQ,CAAC,SAAS,CAAC,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;SAC1E;QAED,OAAO;YACH,WAAW,EAAE,cAAc,CAAC,gBAAgB,CAAC,WAAW,CAAC;YACzD,QAAQ,EAAE,QAAQ;YAClB,UAAU,EAAE,UAAU,CAAC,MAAM,GAAG,aAAa;SAChD,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,WAA4B,EAAE,UAA0B;QAClF,MAAM,aAAa,GAAG,UAAU,CAAC,MAAM,CAAC;QACxC,MAAM,UAAU,GAAqB,EAAE,CAAC;QAExC,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,EAAE;YAC3C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;YACrD,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;YACvE,MAAM,UAAU,GAAwB,EAAE,CAAC;YAE3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,EAAE,CAAC,EAAE,EAAE;gBACvC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;gBAChE,UAAU,CAAC,IAAI,CAAC,cAAc,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,CAAC;aAC/D;YAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;YAC/D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;YAE7D,UAAU,CAAC,IAAI,CAAC;gBACZ,IAAI;gBACJ,UAAU;gBACV,QAAQ,EAAE,QAAoB;gBAC9B,MAAM,EAAE,cAAc,CAAC,gBAAgB,CAAC,MAAM,CAAC;aAClD,CAAC,CAAC;SACN;QAED,OAAO;YACH,UAAU;YACV,cAAc,EAAE,UAAU,CAAC,MAAM,GAAG,aAAa;SACpD,CAAC;IACN,CAAC;CACJ"}

package/dist/gguf/parser/GgufV3Parser.d.ts

@@ -0,0 +1,3 @@
+import { GgufV2Parser } from "./GgufV2Parser.js";
+export declare class GgufV3Parser extends GgufV2Parser {
+}

package/dist/gguf/parser/GgufV3Parser.js

@@ -0,0 +1,4 @@
+import { GgufV2Parser } from "./GgufV2Parser.js";
+export class GgufV3Parser extends GgufV2Parser {
+}
+//# sourceMappingURL=GgufV3Parser.js.map

package/dist/gguf/parser/GgufV3Parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GgufV3Parser.js","sourceRoot":"","sources":["../../../src/gguf/parser/GgufV3Parser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,mBAAmB,CAAC;AAE/C,MAAM,OAAO,YAAa,SAAQ,YAAY;CAE7C"}

package/dist/gguf/parser/parseGguf.d.ts

@@ -0,0 +1,8 @@
+import { GgufFileReader } from "../fileReaders/GgufFileReader.js";
+import { GgufFileInfo } from "../types/GgufFileInfoTypes.js";
+export declare function parseGguf({ fileReader, readTensorInfo, ignoreKeys, logWarnings }: {
+    fileReader: GgufFileReader;
+    readTensorInfo?: boolean;
+    ignoreKeys?: string[];
+    logWarnings?: boolean;
+}): Promise<GgufFileInfo>;

package/dist/gguf/parser/parseGguf.js

@@ -0,0 +1,63 @@
+import { InvalidGgufMagicError } from "../errors/InvalidGgufMagicError.js";
+import { getConsoleLogPrefix } from "../../utils/getConsoleLogPrefix.js";
+import { UnsupportedError } from "../../utils/UnsupportedError.js";
+import { GgufReadOffset } from "../utils/GgufReadOffset.js";
+import { valueTypeToBytesToRead } from "../fileReaders/GgufFileReader.js";
+import { getGgufMetadataArchitectureData } from "../utils/getGgufMetadataArchitectureData.js";
+import { GgufV2Parser } from "./GgufV2Parser.js";
+import { GgufV3Parser } from "./GgufV3Parser.js";
+const ggufMagic = "GGUF";
+export async function parseGguf({ fileReader, readTensorInfo = true, ignoreKeys = [], logWarnings = true }) {
+    const readOffset = new GgufReadOffset(0);
+    const magicAndVersion = await parseMagicAndVersion(fileReader, readOffset);
+    const gguifInfo = await parseGgufUsingASpecificVersionParser({
+        fileReader,
+        readTensorInfo,
+        ignoreKeys,
+        version: magicAndVersion.version,
+        readOffset,
+        logWarnings
+    });
+    const architectureMetadata = getGgufMetadataArchitectureData(gguifInfo.metadata);
+    return {
+        version: magicAndVersion.version,
+        tensorCount: gguifInfo.tensorCount,
+        metadata: gguifInfo.metadata,
+        architectureMetadata: architectureMetadata,
+        tensorInfo: gguifInfo.tensorInfo,
+        metadataSize: gguifInfo.metadataSize,
+        splicedParts: 1,
+        totalTensorInfoSize: gguifInfo.tensorInfoSize,
+        totalTensorCount: gguifInfo.tensorCount,
+        totalMetadataSize: gguifInfo.metadataSize,
+        fullTensorInfo: gguifInfo.tensorInfo,
+        tensorInfoSize: gguifInfo.tensorInfoSize
+    };
+}
+async function parseMagicAndVersion(fileReader, readOffset) {
+    const fileMagicBytes = await fileReader.readByteRange(readOffset, valueTypeToBytesToRead.uint8 * ggufMagic.length);
+    const fileMagicText = String.fromCharCode(...fileMagicBytes);
+    if (fileMagicText !== ggufMagic)
+        throw new InvalidGgufMagicError(ggufMagic, fileMagicText);
+    const version = await fileReader.readUint32(readOffset);
+    return {
+        magic: ggufMagic,
+        version
+    };
+}
+async function parseGgufUsingASpecificVersionParser(specificVersionParserOptions) {
+    switch (specificVersionParserOptions.version) {
+        case 1:
+            throw new UnsupportedError("GGUF version 1 is not supported by llama.cpp anymore");
+        case 2:
+            return await (new GgufV2Parser(specificVersionParserOptions)).parse();
+        case 3:
+            return await (new GgufV3Parser(specificVersionParserOptions)).parse();
+        default:
+            if (specificVersionParserOptions.logWarnings)
+                console.warn(getConsoleLogPrefix() +
+                    `Unsupported GGUF version "${specificVersionParserOptions.version}". Reading the file as GGUF version 3`);
+            return await (new GgufV3Parser(specificVersionParserOptions)).parse();
+    }
+}
+//# sourceMappingURL=parseGguf.js.map

package/dist/gguf/parser/parseGguf.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parseGguf.js","sourceRoot":"","sources":["../../../src/gguf/parser/parseGguf.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,qBAAqB,EAAC,MAAM,oCAAoC,CAAC;AACzE,OAAO,EAAC,mBAAmB,EAAC,MAAM,oCAAoC,CAAC;AACvE,OAAO,EAAC,gBAAgB,EAAC,MAAM,iCAAiC,CAAC;AACjE,OAAO,EAAC,cAAc,EAAC,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAiB,sBAAsB,EAAC,MAAM,kCAAkC,CAAC;AAExF,OAAO,EAAC,+BAA+B,EAAC,MAAM,6CAA6C,CAAC;AAC5F,OAAO,EAAC,YAAY,EAAC,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAC,YAAY,EAAC,MAAM,mBAAmB,CAAC;AAE/C,MAAM,SAAS,GAAG,MAAM,CAAC;AAEzB,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,EAC5B,UAAU,EACV,cAAc,GAAG,IAAI,EACrB,UAAU,GAAG,EAAE,EACf,WAAW,GAAG,IAAI,EAMrB;IACG,MAAM,UAAU,GAAG,IAAI,cAAc,CAAC,CAAC,CAAC,CAAC;IACzC,MAAM,eAAe,GAAG,MAAM,oBAAoB,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IAC3E,MAAM,SAAS,GAAG,MAAM,oCAAoC,CAAC;QACzD,UAAU;QACV,cAAc;QACd,UAAU;QAEV,OAAO,EAAE,eAAe,CAAC,OAAO;QAChC,UAAU;QACV,WAAW;KACd,CAAC,CAAC;IACH,MAAM,oBAAoB,GAAG,+BAA+B,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IAEjF,OAAO;QACH,OAAO,EAAE,eAAe,CAAC,OAAO;QAChC,WAAW,EAAE,SAAS,CAAC,WAAW;QAClC,QAAQ,EAAE,SAAS,CAAC,QAAQ;QAC5B,oBAAoB,EAAE,oBAAoB;QAC1C,UAAU,EAAE,SAAS,CAAC,UAAU;QAChC,YAAY,EAAE,SAAS,CAAC,YAAY;QACpC,YAAY,EAAE,CAAC;QACf,mBAAmB,EAAE,SAAS,CAAC,cAAc;QAC7C,gBAAgB,EAAE,SAAS,CAAC,WAAW;QACvC,iBAAiB,EAAE,SAAS,CAAC,YAAY;QACzC,cAAc,EAAE,SAAS,CAAC,UAAU;QACpC,cAAc,EAAE,SAAS,CAAC,cAAc;KAC3C,CAAC;AACN,CAAC;AAED,KAAK,UAAU,oBAAoB,CAAC,UAA0B,EAAE,UAA0B;IACtF,MAAM,cAAc,GAAG,MAAM,UAAU,CAAC,aAAa,CAAC,UAAU,EAAE,sBAAsB,CAAC,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;IACnH,MAAM,aAAa,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,cAAc,CAAC,CAAC;IAE7D,IAAI,aAAa,KAAK,SAAS;QAC3B,MAAM,IAAI,qBAAqB,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;IAE9D,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;IAExD,OAAO;QACH,KAAK,EAAE,SAAS;QAChB,OAAO;KACV,CAAC;AACN,CAAC;AAED,KAAK,UAAU,oCAAoC,CAC/C,4BAAsD;IAEtD,QAAQ,4BAA4B,CAAC,OAAO,EAAE;QAC1C,KAAK,CAAC;YACF,MAAM,IAAI,gBAAgB,CAAC,sDAAsD,CAAC,CAAC;QAEvF,KAAK,CAAC;YACF,OAAO,MAAM,CAAC,IAAI,YAAY,CAAC,4BAA4B,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;QAE1E,KAAK,CAAC;YACF,OAAO,MAAM,CAAC,IAAI,YAAY,CAAC,4BAA4B,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;QAE1E;YACI,IAAI,4BAA4B,CAAC,WAAW;gBACxC,OAAO,CAAC,IAAI,CACR,mBAAmB,EAAE;oBACrB,6BAA6B,4BAA4B,CAAC,OAAO,uCAAuC,CAC3G,CAAC;YAEN,OAAO,MAAM,CAAC,IAAI,YAAY,CAAC,4BAA4B,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;KAC7E;AACL,CAAC"}

package/dist/gguf/readGgufFileInfo.d.ts

@@ -0,0 +1,33 @@
+/// <reference types="node" />
+import retry from "async-retry";
+import { GgufFileInfo } from "./types/GgufFileInfoTypes.js";
+/**
+ * Read a GGUF file and return its metadata and tensor info (unless `readTensorInfo` is set to `false`).
+ * Only the parts of the file required for the metadata and tensor info are read.
+ */
+export declare function readGgufFileInfo(pathOrUrl: string, { readTensorInfo, sourceType, ignoreKeys, logWarnings, fetchRetryOptions, fetchHeaders, spliceSplitFiles, signal }?: {
+    /**
+     * Whether to read the tensor info from the file's header
+     * Enabled by default.
+     */
+    readTensorInfo?: boolean;
+    /**
+     * Set to a specific value to force it to only use that source type.
+     * By default, it detects whether the path is a network URL or a filesystem path and uses the appropriate reader accordingly.
+     */
+    sourceType?: "network" | "filesystem";
+    /**
+     * Metadata keys to ignore when parsing the metadata.
+     * For example, `["tokenizer.ggml.tokens"]`
+     */
+    ignoreKeys?: string[];
+    /** Whether to log warnings */
+    logWarnings?: boolean;
+    /** Relevant only when fetching from a network */
+    fetchRetryOptions?: retry.Options;
+    /** Relevant only when fetching from a network */
+    fetchHeaders?: Record<string, string>;
+    /** When split files are detected, read the metadata of the first file and splice the tensor info from all the parts */
+    spliceSplitFiles?: boolean;
+    signal?: AbortSignal;
+}): Promise<GgufFileInfo>;