node-llama-cpp 3.0.0-beta.14 → 3.0.0-beta.16

package/dist/gguf/fileReaders/GgufFsFileReader.js

@@ -0,0 +1,45 @@
+import fs from "node:fs/promises";
+import { withLock } from "lifecycle-utils";
+import { GgufReadOffset } from "../utils/GgufReadOffset.js";
+import { defaultExtraAllocationSize } from "../consts.js";
+import { GgufFileReader } from "./GgufFileReader.js";
+export class GgufFsFileReader extends GgufFileReader {
+    filePath;
+    _signal;
+    constructor({ filePath, signal }) {
+        super();
+        this.filePath = filePath;
+        this._signal = signal;
+    }
+    async readByteRange(offset, length) {
+        const readOffset = GgufReadOffset.resolveReadOffset(offset);
+        const endOffset = readOffset.offset + length;
+        if (endOffset >= this._buffer.length)
+            await this._readToExpandBufferUpToOffset(endOffset);
+        const res = this._buffer.subarray(readOffset.offset, endOffset);
+        readOffset.moveBy(length);
+        return res;
+    }
+    async _readToExpandBufferUpToOffset(endOffset, extraAllocationSize = defaultExtraAllocationSize) {
+        return await withLock(this, "modifyBuffer", this._signal, async () => {
+            if (endOffset < this._buffer.length)
+                return;
+            const missingBytesBuffer = await this._readByteRange(this._buffer.length, endOffset + extraAllocationSize - this._buffer.length);
+            this._addToBuffer(missingBytesBuffer);
+        });
+    }
+    async _readByteRange(start, length) {
+        const fd = await fs.open(this.filePath, "r");
+        try {
+            if (this._signal?.aborted)
+                throw this._signal.reason;
+            const buffer = Buffer.alloc(length);
+            await fd.read(buffer, 0, length, start);
+            return buffer;
+        }
+        finally {
+            await fd.close();
+        }
+    }
+}
+//# sourceMappingURL=GgufFsFileReader.js.map

package/dist/gguf/fileReaders/GgufFsFileReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GgufFsFileReader.js","sourceRoot":"","sources":["../../../src/gguf/fileReaders/GgufFsFileReader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,EAAC,QAAQ,EAAC,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAC,cAAc,EAAC,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAC,0BAA0B,EAAC,MAAM,cAAc,CAAC;AACxD,OAAO,EAAC,cAAc,EAAC,MAAM,qBAAqB,CAAC;AAOnD,MAAM,OAAO,gBAAiB,SAAQ,cAAc;IAChC,QAAQ,CAAS;IAChB,OAAO,CAAe;IAEvC,YAAmB,EAAC,QAAQ,EAAE,MAAM,EAA0B;QAC1D,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IAC1B,CAAC;IAEM,KAAK,CAAC,aAAa,CAAC,MAA+B,EAAE,MAAc;QACtE,MAAM,UAAU,GAAG,cAAc,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC;QAE7C,IAAI,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM;YAChC,MAAM,IAAI,CAAC,6BAA6B,CAAC,SAAS,CAAC,CAAC;QAExD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAChE,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC1B,OAAO,GAAG,CAAC;IACf,CAAC;IAEO,KAAK,CAAC,6BAA6B,CAAC,SAAiB,EAAE,sBAA8B,0BAA0B;QACnH,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;YACjE,IAAI,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM;gBAC/B,OAAO;YAEX,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,cAAc,CAChD,IAAI,CAAC,OAAO,CAAC,MAAM,EACnB,SAAS,GAAG,mBAAmB,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CACxD,CAAC;YAEF,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,KAAa,EAAE,MAAc;QACtD,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAC7C,IAAI;YACA,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO;gBACrB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAE9B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACpC,MAAM,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;YACxC,OAAO,MAAM,CAAC;SACjB;gBAAS;YACN,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;SACpB;IACL,CAAC;CACJ"}

package/dist/gguf/fileReaders/GgufNetworkFetchFileReader.d.ts

@@ -0,0 +1,22 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import retry from "async-retry";
+import { GgufReadOffset } from "../utils/GgufReadOffset.js";
+import { GgufFileReader } from "./GgufFileReader.js";
+type GgufFetchFileReaderOptions = {
+    url: string;
+    retryOptions?: retry.Options;
+    headers?: Record<string, string>;
+    signal?: AbortSignal;
+};
+export declare class GgufNetworkFetchFileReader extends GgufFileReader {
+    readonly url: string;
+    readonly retryOptions: retry.Options;
+    readonly headers: Record<string, string>;
+    private readonly _signal?;
+    constructor({ url, retryOptions, headers, signal }: GgufFetchFileReaderOptions);
+    readByteRange(offset: number | GgufReadOffset, length: number): Promise<Buffer>;
+    private _fetchToExpandBufferUpToOffset;
+    private _fetchByteRange;
+}
+export {};

package/dist/gguf/fileReaders/GgufNetworkFetchFileReader.js

@@ -0,0 +1,63 @@
+import retry from "async-retry";
+import { withLock } from "lifecycle-utils";
+import { GgufReadOffset } from "../utils/GgufReadOffset.js";
+import { defaultExtraAllocationSize, ggufDefaultFetchRetryOptions } from "../consts.js";
+import { GgufFileReader } from "./GgufFileReader.js";
+export class GgufNetworkFetchFileReader extends GgufFileReader {
+    url;
+    retryOptions;
+    headers;
+    _signal;
+    constructor({ url, retryOptions = ggufDefaultFetchRetryOptions, headers, signal }) {
+        super();
+        this.url = url;
+        this.retryOptions = retryOptions;
+        this.headers = headers ?? {};
+        this._signal = signal;
+    }
+    async readByteRange(offset, length) {
+        const readOffset = GgufReadOffset.resolveReadOffset(offset);
+        const endOffset = readOffset.offset + length;
+        if (endOffset >= this._buffer.length)
+            await this._fetchToExpandBufferUpToOffset(endOffset);
+        const res = this._buffer.subarray(readOffset.offset, endOffset);
+        readOffset.moveBy(length);
+        return res;
+    }
+    async _fetchToExpandBufferUpToOffset(endOffset, extraAllocationSize = defaultExtraAllocationSize) {
+        await withLock(this, "modifyBuffer", this._signal, async () => {
+            if (endOffset < this._buffer.length)
+                return;
+            const missingBytesBuffer = await retry(async (bail) => {
+                try {
+                    return await this._fetchByteRange(this._buffer.length, endOffset + extraAllocationSize - this._buffer.length);
+                }
+                catch (err) {
+                    if (this._signal?.aborted) {
+                        bail(this._signal.reason);
+                        throw this._signal.reason;
+                    }
+                    throw err;
+                }
+            }, this.retryOptions);
+            if (this._signal?.aborted)
+                throw this._signal.reason;
+            this._addToBuffer(missingBytesBuffer);
+        });
+    }
+    async _fetchByteRange(start, length) {
+        const response = await fetch(this.url, {
+            headers: {
+                ...this.headers,
+                Range: `bytes=${start}-${start + length}`,
+                accept: "*/*"
+            },
+            signal: this._signal
+        });
+        if (!response.ok)
+            throw new Error(`Failed to fetch byte range: ${response.status} ${response.statusText}`);
+        const arrayBuffer = await response.arrayBuffer();
+        return Buffer.from(arrayBuffer);
+    }
+}
+//# sourceMappingURL=GgufNetworkFetchFileReader.js.map

package/dist/gguf/fileReaders/GgufNetworkFetchFileReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GgufNetworkFetchFileReader.js","sourceRoot":"","sources":["../../../src/gguf/fileReaders/GgufNetworkFetchFileReader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,aAAa,CAAC;AAChC,OAAO,EAAC,QAAQ,EAAC,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAC,cAAc,EAAC,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAC,0BAA0B,EAAE,4BAA4B,EAAC,MAAM,cAAc,CAAC;AACtF,OAAO,EAAC,cAAc,EAAC,MAAM,qBAAqB,CAAC;AASnD,MAAM,OAAO,0BAA2B,SAAQ,cAAc;IAC1C,GAAG,CAAS;IACZ,YAAY,CAAgB;IAC5B,OAAO,CAAyB;IAC/B,OAAO,CAAe;IAEvC,YAAmB,EAAC,GAAG,EAAE,YAAY,GAAG,4BAA4B,EAAE,OAAO,EAAE,MAAM,EAA6B;QAC9G,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,OAAO,GAAG,OAAO,IAAI,EAAE,CAAC;QAC7B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IAC1B,CAAC;IAEM,KAAK,CAAC,aAAa,CAAC,MAA+B,EAAE,MAAc;QACtE,MAAM,UAAU,GAAG,cAAc,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC;QAE7C,IAAI,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM;YAChC,MAAM,IAAI,CAAC,8BAA8B,CAAC,SAAS,CAAC,CAAC;QAEzD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAChE,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC1B,OAAO,GAAG,CAAC;IACf,CAAC;IAEO,KAAK,CAAC,8BAA8B,CAAC,SAAiB,EAAE,sBAA8B,0BAA0B;QACpH,MAAM,QAAQ,CAAC,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;YAC1D,IAAI,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM;gBAC/B,OAAO;YAEX,MAAM,kBAAkB,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;gBAClD,IAAI;oBACA,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,SAAS,GAAG,mBAAmB,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;iBACjH;gBAAC,OAAO,GAAG,EAAE;oBACV,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE;wBACvB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;wBAC1B,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;qBAC7B;oBAED,MAAM,GAAG,CAAC;iBACb;YACL,CAAC,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;YAEtB,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO;gBACrB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAE9B,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,KAAa,EAAE,MAAc;QACvD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE;YACnC,OAAO,EAAE;gBACL,GAAG,IAAI,CAAC,OAAO;gBACf,KAAK,EAAE,SAAS,KAAK,IAAI,KAAK,GAAG,MAAM,EAAE;gBACzC,MAAM,EAAE,KAAK;aAChB;YACD,MAAM,EAAE,IAAI,CAAC,OAAO;SACvB,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE;YACZ,MAAM,IAAI,KAAK,CAAC,+BAA+B,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QAE7F,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;QACjD,OAAO,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACpC,CAAC;CACJ"}

package/dist/gguf/insights/GgufInsights.d.ts

@@ -0,0 +1,42 @@
+import { Llama } from "../../bindings/Llama.js";
+import { GgufFileInfo } from "../types/GgufFileInfoTypes.js";
+import { GgufInsightsConfigurationResolver } from "./GgufInsightsConfigurationResolver.js";
+export type GgufInsightsResourceRequirements = {
+    cpuRam: number;
+    gpuVram: number;
+};
+export declare class GgufInsights {
+    private constructor();
+    get ggufFileInfo(): GgufFileInfo;
+    get configurationResolver(): GgufInsightsConfigurationResolver;
+    /** The context size the model was trained on */
+    get trainContextSize(): number | undefined;
+    /** The size of an embedding vector the model can produce */
+    get embeddingVectorSize(): number | undefined;
+    get totalLayers(): number;
+    get modelSize(): number;
+    estimateModelResourceRequirements({ gpuLayers }: {
+        gpuLayers: number;
+    }): GgufInsightsResourceRequirements;
+    /**
+     * Estimates the memory required to create a context of the given parameters based on the implementation details of `llama.cpp`.
+     * The calculation doesn't include a precise estimation of the graph overhead memory, so it uses a rough estimate for that.
+     * The estimation for the graph overhead memory will be improved in the future to be more precise, but it's good enough for now.
+     */
+    estimateContextResourceRequirements({ contextSize, modelGpuLayers, batchSize, sequences, isEmbeddingContext, includeGraphOverhead }: {
+        contextSize: number;
+        modelGpuLayers: number;
+        batchSize?: number;
+        sequences?: number;
+        isEmbeddingContext?: boolean;
+        includeGraphOverhead?: boolean;
+    }): GgufInsightsResourceRequirements;
+    /**
+     * @param ggufFileInfo
+     * @param llama - If you already have a `Llama` instance, pass it to reuse it for the `GgufInsights` instance.
+     * If you don't pass a `Llama` instance, a basic `Llama` instance is created as a fallback - it's a slim instance that
+     * doesn't instantiate a `llama.cpp` backend, so it won't utilize the GPU at all, and be shared with other `GgufInsights` instances
+     * that need a fallback `Llama` instance.
+     */
+    static from(ggufFileInfo: GgufFileInfo, llama?: Llama): Promise<GgufInsights>;
+}

package/dist/gguf/insights/GgufInsights.js

@@ -0,0 +1,361 @@
+import { getLlamaWithoutBackend } from "../../bindings/utils/getLlamaWithoutBackend.js";
+import { getDefaultContextBatchSize, getDefaultContextSequences } from "../../evaluator/LlamaContext/LlamaContext.js";
+import { GgufArchitectureType } from "../types/GgufMetadataTypes.js";
+import { GgufInsightsConfigurationResolver } from "./GgufInsightsConfigurationResolver.js";
+export class GgufInsights {
+    /** @internal */ _llama;
+    /** @internal */ _modelSize;
+    /** @internal */ _totalLayers = null;
+    /** @internal */ _ggufFileInfo;
+    /** @internal */ _configurationResolver;
+    constructor(ggufFileInfo, llama) {
+        this._llama = llama;
+        this._ggufFileInfo = ggufFileInfo;
+        this._modelSize = calculateTensorsSize(ggufFileInfo.tensorInfo ?? [], llama);
+        this._configurationResolver = GgufInsightsConfigurationResolver._create(this);
+    }
+    get ggufFileInfo() {
+        return this._ggufFileInfo;
+    }
+    get configurationResolver() {
+        return this._configurationResolver;
+    }
+    /** The context size the model was trained on */
+    get trainContextSize() {
+        return this._ggufFileInfo.architectureMetadata.context_length;
+    }
+    /** The size of an embedding vector the model can produce */
+    get embeddingVectorSize() {
+        return this._ggufFileInfo.architectureMetadata.embedding_length;
+    }
+    get totalLayers() {
+        if (this._totalLayers != null)
+            return this._totalLayers;
+        const outputLayers = 1;
+        this._totalLayers = this._getFileLayers() + outputLayers;
+        return this._totalLayers;
+    }
+    get modelSize() {
+        return this._modelSize;
+    }
+    estimateModelResourceRequirements({ gpuLayers }) {
+        const { cpu, gpu } = this._getTensorResourceSplit(gpuLayers);
+        return {
+            cpuRam: calculateTensorsSize(cpu, this._llama),
+            gpuVram: calculateTensorsSize(gpu, this._llama)
+        };
+    }
+    /**
+     * Estimates the memory required to create a context of the given parameters based on the implementation details of `llama.cpp`.
+     * The calculation doesn't include a precise estimation of the graph overhead memory, so it uses a rough estimate for that.
+     * The estimation for the graph overhead memory will be improved in the future to be more precise, but it's good enough for now.
+     */
+    estimateContextResourceRequirements({ contextSize, modelGpuLayers, batchSize, sequences, isEmbeddingContext = false, includeGraphOverhead = true }) {
+        if (sequences == null)
+            sequences = getDefaultContextSequences();
+        if (batchSize == null)
+            batchSize = getDefaultContextBatchSize({ contextSize, sequences });
+        const actualContextSize = contextSize * sequences;
+        const totalLayers = this.totalLayers;
+        const finalGpuLayers = Math.max(0, Math.min(modelGpuLayers ?? totalLayers, totalLayers));
+        const finalCpuLayers = totalLayers - finalGpuLayers;
+        const llmData = this._ggufFileInfo.architectureMetadata;
+        const vocabularySize = llmData.vocab_size ?? this._ggufFileInfo.metadata.tokenizer.ggml.tokens.length;
+        const logitsSize = vocabularySize * batchSize;
+        const embedSize = isEmbeddingContext
+            ? (llmData.embedding_length ?? 0) * batchSize
+            : 0;
+        const sizeTBytes = 8; // sizeof(size_t)
+        const floatBytes = 4; // sizeof(float)
+        const uint32TBytes = 4; // sizeof(uint32_t)
+        const int32TBytes = 4; // sizeof(int32_t)
+        // source: `llama_get_state_size` in `llama.cpp`
+        const sRngSize = sizeTBytes;
+        const sRng = this._llama._consts.llamaMaxRngState;
+        const sNOutputs = sizeTBytes;
+        const sNOutputPos = batchSize * int32TBytes;
+        const sLogitsSize = sizeTBytes;
+        const sLogits = logitsSize * floatBytes;
+        const sEmbeddingSize = sizeTBytes;
+        const sEmbedding = embedSize * floatBytes;
+        const sKvBufSize = sizeTBytes;
+        const sKvHead = uint32TBytes;
+        const sKvSize = uint32TBytes;
+        const sKvUsed = uint32TBytes;
+        const sKv = 2 * int32TBytes * modelGpuLayers * this._llama._consts.ggmlTensorOverhead;
+        const sKvCell = this._llama._consts.llamaPosSize + sizeTBytes + this._llama._consts.llamaSeqIdSize;
+        const kvSelfLength = this._ggufFileInfo.metadata.general.architecture === GgufArchitectureType.mamba
+            ? Math.max(1, sequences)
+            : actualContextSize;
+        const sKvCells = kvSelfLength * sKvCell;
+        const overheadMemory = (sRngSize +
+            sRng +
+            sNOutputs +
+            sNOutputPos +
+            sLogitsSize +
+            sLogits +
+            sEmbeddingSize +
+            sEmbedding +
+            sKvBufSize +
+            sKvHead +
+            sKvSize +
+            sKvUsed +
+            sKv +
+            sKvCells);
+        // Estimates the memory allocated by `ggml_backend_sched_reserve` in `llama_new_context_with_model` in `llama.cpp`.
+        // If you read this line and have better insights on how to estimate this memory, please open a PR to improve it :)
+        const estimateGraphOverheadMemory = () => {
+            const s1MB = Math.pow(1024, 2);
+            const tensorInfo = this._ggufFileInfo.tensorInfo ?? [];
+            let defaultCalculationAdjustment = 0;
+            if (batchSize == null)
+                return 0;
+            if (this._ggufFileInfo.metadata.general.architecture === GgufArchitectureType.llama) {
+                const expertCount = this._ggufFileInfo.architectureMetadata.expert_count ?? 0;
+                const headCount = this._ggufFileInfo.architectureMetadata.attention?.head_count ?? 0;
+                const embeddingLength = llmData.embedding_length ?? 0;
+                if (expertCount > 0) {
+                    const expertsUsedCount = this._ggufFileInfo.architectureMetadata.expert_used_count ?? 2;
+                    return int32TBytes * batchSize * (((expertsUsedCount + 1) * embeddingLength) + (actualContextSize * headCount));
+                }
+                return int32TBytes * batchSize * (embeddingLength + (actualContextSize * headCount));
+            }
+            else if (this._ggufFileInfo.metadata.general.architecture === GgufArchitectureType.qwen2) {
+                if (modelGpuLayers === this.totalLayers) {
+                    defaultCalculationAdjustment -= (s1MB * 340) * (this.trainContextSize == null
+                        ? 1
+                        : actualContextSize / this.trainContextSize);
+                }
+                else {
+                    defaultCalculationAdjustment -= (s1MB * 250) + ((s1MB * 50) * (this.trainContextSize == null
+                        ? 1
+                        : actualContextSize / this.trainContextSize));
+                }
+            }
+            else if (this._ggufFileInfo.metadata.general.architecture === GgufArchitectureType.gemma) {
+                // only works properly when all layers are on the GPU, which is why it's commented out:
+                // return int32TBytes * batchSize * ((llmData.embedding_length ?? 0));
+                if (modelGpuLayers === this.totalLayers) {
+                    defaultCalculationAdjustment += (s1MB * 40) - ((s1MB * 270) * (this.trainContextSize == null
+                        ? 1
+                        : actualContextSize / this.trainContextSize));
+                }
+                else {
+                    defaultCalculationAdjustment += -(s1MB * 550) + ((s1MB * 150) * (this.trainContextSize == null
+                        ? 1
+                        : Math.max(0, (1 - (actualContextSize / this.trainContextSize)))));
+                }
+            }
+            else if (this._ggufFileInfo.metadata.general.architecture === GgufArchitectureType.stablelm) {
+                const headCount = this._ggufFileInfo.architectureMetadata.attention?.head_count ?? 0;
+                return (int32TBytes * batchSize * actualContextSize * headCount) - (50 * s1MB);
+                // if (modelGpuLayers === this.totalLayers) {
+                //     defaultCalculationAdjustment += -(s1MB * 20) + (
+                //         (s1MB * 250) * (
+                //             this.trainContextSize == null
+                //                 ? 1
+                //                 : actualContextSize / this.trainContextSize
+                //         )
+                //     );
+                // } else {
+                //     defaultCalculationAdjustment += -(s1MB * 40) + (
+                //         (s1MB * 300) * (
+                //             this.trainContextSize == null
+                //                 ? 1
+                //                 : actualContextSize / this.trainContextSize
+                //         )
+                //     );
+                // }
+            }
+            const totalElements = tensorInfo.length === 0
+                ? this.totalLayers * (((llmData.embedding_length ?? 0) +
+                    (llmData.feed_forward_length ?? 0)) / 2)
+                : tensorInfo.reduce((res, tensor) => {
+                    return res + tensor.dimensions.reduce((res, dim) => res + Number(dim), 0);
+                }, 0);
+            // magic numbers for estimation. will be improved in the future
+            return (totalElements * 77.655 * (actualContextSize / 4096)) + defaultCalculationAdjustment;
+        };
+        const graphOverheadMemory = !includeGraphOverhead
+            ? 0
+            : estimateGraphOverheadMemory();
+        const usingGpu = finalGpuLayers !== 0;
+        const cpuRam = (!usingGpu
+            ? (overheadMemory + graphOverheadMemory)
+            : 0) +
+            this._estimateKvMemorySizeInBytes(actualContextSize, finalCpuLayers);
+        const gpuVram = usingGpu
+            ? (overheadMemory +
+                graphOverheadMemory +
+                this._estimateKvMemorySizeInBytes(actualContextSize, finalGpuLayers < totalLayers
+                    ? (finalGpuLayers + 1)
+                    : finalGpuLayers))
+            : 0;
+        return {
+            cpuRam,
+            gpuVram
+        };
+    }
+    /**
+     * Get the split tensor resources for CPU and GPU based on the number of GPU layers
+     * @internal
+     */
+    _getTensorResourceSplit(gpuLayers) {
+        const tensorInfo = this._ggufFileInfo.tensorInfo ?? [];
+        if (gpuLayers === 0) {
+            return {
+                cpu: tensorInfo,
+                gpu: []
+            };
+        }
+        const fileLayers = this._getFileLayers();
+        const startGpuLayer = Math.max(0, fileLayers - gpuLayers);
+        const gpuTensors = [];
+        const cpuTensors = [];
+        for (const singleTensorInfo of tensorInfo) {
+            const { layerNumber } = parseTensorName(singleTensorInfo.name);
+            if (gpuLayers !== this.totalLayers) {
+                const architecture = this._ggufFileInfo.metadata?.general?.architecture;
+                if (architecture === GgufArchitectureType.qwen2 || architecture === GgufArchitectureType.gemma) {
+                    if (layerNumber != null && layerNumber >= startGpuLayer)
+                        gpuTensors.push(singleTensorInfo);
+                    else
+                        cpuTensors.push(singleTensorInfo);
+                    continue;
+                }
+            }
+            if (layerNumber == null || layerNumber >= startGpuLayer)
+                gpuTensors.push(singleTensorInfo);
+            else
+                cpuTensors.push(singleTensorInfo);
+        }
+        return {
+            cpu: cpuTensors,
+            gpu: gpuTensors
+        };
+    }
+    /** @internal */
+    _determineNumberOfLayersFromTensorInfo() {
+        const layerNumbers = new Set();
+        for (const singleTensorInfo of (this._ggufFileInfo.tensorInfo ?? [])) {
+            const { layerNumber } = parseTensorName(singleTensorInfo.name);
+            if (layerNumber != null)
+                layerNumbers.add(layerNumber);
+        }
+        return layerNumbers.size;
+    }
+    /** @internal */
+    _getFileLayers() {
+        return this._ggufFileInfo.architectureMetadata.block_count ?? this._determineNumberOfLayersFromTensorInfo();
+    }
+    /** @internal */
+    _estimateKvMemorySizeInBytes(contextSize, layers) {
+        // source: `llama_kv_cache_init` in `llama.cpp`
+        const nHead = this._ggufFileInfo.architectureMetadata.attention?.head_count ?? 0;
+        const nEmbd = this._ggufFileInfo.architectureMetadata.embedding_length ?? 0;
+        const nEmbdHeadK = this._ggufFileInfo.architectureMetadata.attention?.key_length ?? ((nHead == 0) ? 0 : (nEmbd / nHead));
+        const nHeadKv = this._ggufFileInfo.architectureMetadata.attention?.head_count_kv ?? nHead;
+        const modelNEmbdKGqa = nEmbdHeadK * nHeadKv;
+        const ssmDConv = this._ggufFileInfo.architectureMetadata.ssm?.conv_kernel ?? 0;
+        const ssmDInner = this._ggufFileInfo.architectureMetadata.ssm?.inner_size ?? 0;
+        const modelNEmbdKS = (ssmDConv > 0 ? (ssmDConv - 1) : 0) * ssmDInner;
+        const nEmbdHeadV = this._ggufFileInfo.architectureMetadata.attention?.value_length ?? ((nHead == 0) ? 0 : nEmbd / nHead);
+        const modelNEmbdVGqa = nEmbdHeadV * nHeadKv;
+        const ssmDState = this._ggufFileInfo.architectureMetadata.ssm?.state_size ?? 0;
+        const modelNEmbdVS = ssmDState * ssmDInner;
+        const totalNEmbdKGqa = modelNEmbdKGqa + modelNEmbdKS;
+        const totalNEmbdVGqa = modelNEmbdVGqa + modelNEmbdVS;
+        const keyTypeSize = this._ggufFileInfo.metadata.general.architecture === GgufArchitectureType.mamba
+            // if `type_k` of `llama_context_params` changes to be configurable in `LlamaContext`,
+            // this would have to depend on that value
+            ? this._llama._consts.ggmlTypeF32Size
+            : this._llama._consts.ggmlTypeF16Size;
+        const valueTypeSize = this._ggufFileInfo.metadata.general.architecture === GgufArchitectureType.mamba
+            // if `type_v` of `llama_context_params` changes to be configurable in `LlamaContext`,
+            // this would have to depend on that value
+            ? this._llama._consts.ggmlTypeF32Size
+            : this._llama._consts.ggmlTypeF16Size;
+        const keyTensorsSize = layers * totalNEmbdKGqa * contextSize * keyTypeSize;
+        const valueTensorsSize = layers * totalNEmbdVGqa * contextSize * valueTypeSize;
+        return keyTensorsSize + valueTensorsSize;
+    }
+    /**
+     * @param ggufFileInfo
+     * @param llama - If you already have a `Llama` instance, pass it to reuse it for the `GgufInsights` instance.
+     * If you don't pass a `Llama` instance, a basic `Llama` instance is created as a fallback - it's a slim instance that
+     * doesn't instantiate a `llama.cpp` backend, so it won't utilize the GPU at all, and be shared with other `GgufInsights` instances
+     * that need a fallback `Llama` instance.
+     */
+    static async from(ggufFileInfo, llama) {
+        let resolvedLlama = llama;
+        if (resolvedLlama == null)
+            resolvedLlama = await getLlamaWithoutBackend();
+        return new GgufInsights(ggufFileInfo, resolvedLlama);
+    }
+}
+function parseTensorName(tensorName) {
+    if (tensorName == null)
+        return { layerNumber: undefined };
+    const layerTensorPrefix = "blk.";
+    if (!tensorName.startsWith(layerTensorPrefix))
+        return { layerNumber: undefined };
+    const dotIndex = tensorName.indexOf(".", layerTensorPrefix.length);
+    const layerNumberString = tensorName.slice(layerTensorPrefix.length, dotIndex < 0
+        ? tensorName.length
+        : dotIndex);
+    const layerNumber = parseInt(layerNumberString);
+    if (Number.isFinite(layerNumber))
+        return { layerNumber };
+    return { layerNumber: undefined };
+}
+function calculateTensorsSize(tensorsInfo, llama) {
+    let size = 0;
+    for (const tensorInfo of tensorsInfo)
+        size += calculateTensorSize(tensorInfo, llama);
+    return size;
+}
+function calculateTensorSize(tensor, llama) {
+    const typeSize = llama._bindings.getTypeSizeForGgmlType(tensor.ggmlType);
+    const blockSize = llama._bindings.getBlockSizeForGgmlType(tensor.ggmlType);
+    const ggmlMaxDims = llama._consts.ggmlMaxDims;
+    if (typeSize == null || blockSize == null)
+        throw new Error("Invalid type or block size");
+    const { ne, nb } = getTensorNeAndNb(tensor, { typeSize, blockSize, ggmlMaxDims });
+    if (blockSize === 1) {
+        let totalBytes = typeSize;
+        for (let i = 0; i < ggmlMaxDims; i++) {
+            totalBytes += (ne[i] - 1) * nb[i];
+        }
+        return totalBytes;
+    }
+    else {
+        let totalBytes = Math.floor((ne[0] * nb[0]) / blockSize);
+        for (let i = 1; i < ggmlMaxDims; i++) {
+            totalBytes += (ne[i] - 1) * nb[i];
+        }
+        return totalBytes;
+    }
+}
+function getTensorNeAndNb(tensor, { typeSize, blockSize, ggmlMaxDims }) {
+    // number of elements
+    // source: `ggml_new_tensor_impl` in `ggml.c`
+    const ne = [
+        ...tensor.dimensions,
+        ...(Array(Math.max(0, ggmlMaxDims - tensor.dimensions.length)).fill(1))
+    ].slice(0, ggmlMaxDims);
+    // number of bytes
+    // source: `ggml_new_tensor_impl` in `ggml.c`
+    const nb = [
+        typeSize,
+        Math.floor(typeSize * (ne[0] / blockSize)),
+        ...Array(ggmlMaxDims - 2).fill(0)
+    ];
+    for (let i = 2; i < ggmlMaxDims; i++) {
+        nb[i] = nb[i - 1] * ne[i - 1];
+    }
+    return {
+        ne,
+        nb
+    };
+}
+//# sourceMappingURL=GgufInsights.js.map

package/dist/gguf/insights/GgufInsights.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GgufInsights.js","sourceRoot":"","sources":["../../../src/gguf/insights/GgufInsights.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,sBAAsB,EAAC,MAAM,gDAAgD,CAAC;AACtF,OAAO,EAAC,0BAA0B,EAAE,0BAA0B,EAAC,MAAM,8CAA8C,CAAC;AAGpH,OAAO,EAAC,oBAAoB,EAAC,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAC,iCAAiC,EAAC,MAAM,wCAAwC,CAAC;AAOzF,MAAM,OAAO,YAAY;IACrB,gBAAgB,CAAiB,MAAM,CAAQ;IAC/C,gBAAgB,CAAkB,UAAU,CAAS;IACrD,gBAAgB,CAAS,YAAY,GAAkB,IAAI,CAAC;IAC5D,gBAAgB,CAAkB,aAAa,CAAe;IAC9D,gBAAgB,CAAkB,sBAAsB,CAAoC;IAE5F,YAAoB,YAA0B,EAAE,KAAY;QACxD,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;QAElC,IAAI,CAAC,UAAU,GAAG,oBAAoB,CAAC,YAAY,CAAC,UAAU,IAAI,EAAE,EAAE,KAAK,CAAC,CAAC;QAC7E,IAAI,CAAC,sBAAsB,GAAG,iCAAiC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAClF,CAAC;IAED,IAAW,YAAY;QACnB,OAAO,IAAI,CAAC,aAAa,CAAC;IAC9B,CAAC;IAED,IAAW,qBAAqB;QAC5B,OAAO,IAAI,CAAC,sBAAsB,CAAC;IACvC,CAAC;IAED,gDAAgD;IAChD,IAAW,gBAAgB;QACvB,OAAO,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,cAAc,CAAC;IAClE,CAAC;IAED,4DAA4D;IAC5D,IAAW,mBAAmB;QAC1B,OAAO,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,gBAAgB,CAAC;IACpE,CAAC;IAED,IAAW,WAAW;QAClB,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI;YACzB,OAAO,IAAI,CAAC,YAAY,CAAC;QAE7B,MAAM,YAAY,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,cAAc,EAAE,GAAG,YAAY,CAAC;QAEzD,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED,IAAW,SAAS;QAChB,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAEM,iCAAiC,CAAC,EAAC,SAAS,EAAsB;QACrE,MAAM,EAAC,GAAG,EAAE,GAAG,EAAC,GAAG,IAAI,CAAC,uBAAuB,CAAC,SAAS,CAAC,CAAC;QAE3D,OAAO;YACH,MAAM,EAAE,oBAAoB,CAAC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC;YAC9C,OAAO,EAAE,oBAAoB,CAAC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC;SAClD,CAAC;IACN,CAAC;IAED;;;;OAIG;IACI,mCAAmC,CAAC,EACvC,WAAW,EAAE,cAAc,EAAE,SAAS,EAAE,SAAS,EAAE,kBAAkB,GAAG,KAAK,EAAE,oBAAoB,GAAG,IAAI,EAI7G;QACG,IAAI,SAAS,IAAI,IAAI;YAAE,SAAS,GAAG,0BAA0B,EAAE,CAAC;QAChE,IAAI,SAAS,IAAI,IAAI;YAAE,SAAS,GAAG,0BAA0B,CAAC,EAAC,WAAW,EAAE,SAAS,EAAC,CAAC,CAAC;QAExF,MAAM,iBAAiB,GAAG,WAAW,GAAG,SAAS,CAAC;QAElD,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;QACrC,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,cAAc,IAAI,WAAW,EAAE,WAAW,CAAC,CAAC,CAAC;QACzF,MAAM,cAAc,GAAG,WAAW,GAAG,cAAc,CAAC;QACpD,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC;QAExD,MAAM,cAAc,GAAG,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QACtG,MAAM,UAAU,GAAG,cAAc,GAAG,SAAS,CAAC;QAC9C,MAAM,SAAS,GAAG,kBAAkB;YAChC,CAAC,CAAC,CAAC,OAAO,CAAC,gBAAgB,IAAI,CAAC,CAAC,GAAG,SAAS;YAC7C,CAAC,CAAC,CAAC,CAAC;QAER,MAAM,UAAU,GAAG,CAAC,CAAC,CAAC,iBAAiB;QACvC,MAAM,UAAU,GAAG,CAAC,CAAC,CAAC,gBAAgB;QACtC,MAAM,YAAY,GAAG,CAAC,CAAC,CAAC,mBAAmB;QAC3C,MAAM,WAAW,GAAG,CAAC,CAAC,CAAC,kBAAkB;QAEzC,gDAAgD;QAChD,MAAM,QAAQ,GAAG,UAAU,CAAC;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC;QAClD,MAAM,SAAS,GAAG,UAAU,CAAC;QAC7B,MAAM,WAAW,GAAG,SAAS,GAAG,WAAW,CAAC;QAC5C,MAAM,WAAW,GAAG,UAAU,CAAC;QAC/B,MAAM,OAAO,GAAG,UAAU,GAAG,UAAU,CAAC;QACxC,MAAM,cAAc,GAAG,UAAU,CAAC;QAClC,MAAM,UAAU,GAAG,SAAS,GAAG,UAAU,CAAC;QAC1C,MAAM,UAAU,GAAG,UAAU,CAAC;QAC9B,MAAM,OAAO,GAAG,YAAY,CAAC;QAC7B,MAAM,OAAO,GAAG,YAAY,CAAC;QAC7B,MAAM,OAAO,GAAG,YAAY,CAAC;QAC7B,MAAM,GAAG,GAAG,CAAC,GAAG,WAAW,GAAG,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC;QACtF,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,GAAG,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC;QACnG,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,YAAY,KAAK,oBAAoB,CAAC,KAAK;YAChG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC;YACxB,CAAC,CAAC,iBAAiB,CAAC;QACxB,MAAM,QAAQ,GAAG,YAAY,GAAG,OAAO,CAAC;QAExC,MAAM,cAAc,GAAG,CACnB,QAAQ;YACR,IAAI;YACJ,SAAS;YACT,WAAW;YACX,WAAW;YACX,OAAO;YACP,cAAc;YACd,UAAU;YACV,UAAU;YACV,OAAO;YACP,OAAO;YACP,OAAO;YACP,GAAG;YACH,QAAQ,CACX,CAAC;QAEF,mHAAmH;QACnH,mHAAmH;QACnH,MAAM,2BAA2B,GAAG,GAAG,EAAE;YACrC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC/B,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,UAAU,IAAI,EAAE,CAAC;YAEvD,IAAI,4BAA4B,GAAG,CAAC,CAAC;YAErC,IAAI,SAAS,IAAI,IAAI;gBACjB,OAAO,CAAC,CAAC;YAEb,IAAI,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,YAAY,KAAK,oBAAoB,CAAC,KAAK,EAAE;gBACjF,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,YAAY,IAAI,CAAC,CAAC;gBAC9E,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,SAAS,EAAE,UAAU,IAAI,CAAC,CAAC;gBACrF,MAAM,eAAe,GAAG,OAAO,CAAC,gBAAgB,IAAI,CAAC,CAAC;gBAEtD,IAAI,WAAW,GAAG,CAAC,EAAE;oBACjB,MAAM,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,iBAAiB,IAAI,CAAC,CAAC;oBAExF,OAAO,WAAW,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,gBAAgB,GAAG,CAAC,CAAC,GAAG,eAAe,CAAC,GAAG,CAAC,iBAAiB,GAAG,SAAS,CAAC,CAAC,CAAC;iBACnH;gBAED,OAAO,WAAW,GAAG,SAAS,GAAG,CAAC,eAAe,GAAG,CAAC,iBAAiB,GAAG,SAAS,CAAC,CAAC,CAAC;aACxF;iBAAM,IAAI,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,YAAY,KAAK,oBAAoB,CAAC,KAAK,EAAE;gBACxF,IAAI,cAAc,KAAK,IAAI,CAAC,WAAW,EAAE;oBACrC,4BAA4B,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC,GAAG,CAC3C,IAAI,CAAC,gBAAgB,IAAI,IAAI;wBACzB,CAAC,CAAC,CAAC;wBACH,CAAC,CAAC,iBAAiB,GAAG,IAAI,CAAC,gBAAgB,CAClD,CAAC;iBACL;qBAAM;oBACH,4BAA4B,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC,GAAG,CAC3C,CAAC,IAAI,GAAG,EAAE,CAAC,GAAG,CACV,IAAI,CAAC,gBAAgB,IAAI,IAAI;wBACzB,CAAC,CAAC,CAAC;wBACH,CAAC,CAAC,iBAAiB,GAAG,IAAI,CAAC,gBAAgB,CAClD,CACJ,CAAC;iBACL;aACJ;iBAAM,IAAI,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,YAAY,KAAK,oBAAoB,CAAC,KAAK,EAAE;gBACxF,uFAAuF;gBACvF,sEAAsE;gBAEtE,IAAI,cAAc,KAAK,IAAI,CAAC,WAAW,EAAE;oBACrC,4BAA4B,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC,GAAG,CAC1C,CAAC,IAAI,GAAG,GAAG,CAAC,GAAG,CACX,IAAI,CAAC,gBAAgB,IAAI,IAAI;wBACzB,CAAC,CAAC,CAAC;wBACH,CAAC,CAAC,iBAAiB,GAAG,IAAI,CAAC,gBAAgB,CAClD,CACJ,CAAC;iBACL;qBAAM;oBACH,4BAA4B,IAAI,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC,GAAG,CAC5C,CAAC,IAAI,GAAG,GAAG,CAAC,GAAG,CACX,IAAI,CAAC,gBAAgB,IAAI,IAAI;wBACzB,CAAC,CAAC,CAAC;wBACH,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,iBAAiB,GAAG,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,CACvE,CACJ,CAAC;iBACL;aACJ;iBAAM,IAAI,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,YAAY,KAAK,oBAAoB,CAAC,QAAQ,EAAE;gBAC3F,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,SAAS,EAAE,UAAU,IAAI,CAAC,CAAC;gBAErF,OAAO,CAAC,WAAW,GAAG,SAAS,GAAG,iBAAiB,GAAG,SAAS,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC;gBAE/E,6CAA6C;gBAC7C,uDAAuD;gBACvD,2BAA2B;gBAC3B,4CAA4C;gBAC5C,sBAAsB;gBACtB,8DAA8D;gBAC9D,YAAY;gBACZ,SAAS;gBACT,WAAW;gBACX,uDAAuD;gBACvD,2BAA2B;gBAC3B,4CAA4C;gBAC5C,sBAAsB;gBACtB,8DAA8D;gBAC9D,YAAY;gBACZ,SAAS;gBACT,IAAI;aACP;YAED,MAAM,aAAa,GAAG,UAAU,CAAC,MAAM,KAAK,CAAC;gBACzC,CAAC,CAAC,IAAI,CAAC,WAAW,GAAG,CACjB,CACI,CAAC,OAAO,CAAC,gBAAgB,IAAI,CAAC,CAAC;oBAC/B,CAAC,OAAO,CAAC,mBAAmB,IAAI,CAAC,CAAC,CACrC,GAAG,CAAC,CACR;gBACD,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE;oBAChC,OAAO,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAW,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;gBACtF,CAAC,EAAE,CAAC,CAAC,CAAC;YAEV,+DAA+D;YAC/D,OAAO,CAAC,aAAa,GAAG,MAAM,GAAG,CAAC,iBAAiB,GAAG,IAAI,CAAC,CAAC,GAAG,4BAA4B,CAAC;QAChG,CAAC,CAAC;QAEF,MAAM,mBAAmB,GAAG,CAAC,oBAAoB;YAC7C,CAAC,CAAC,CAAC;YACH,CAAC,CAAC,2BAA2B,EAAE,CAAC;QAEpC,MAAM,QAAQ,GAAG,cAAc,KAAK,CAAC,CAAC;QAEtC,MAAM,MAAM,GAAG,CACX,CAAC,QAAQ;YACL,CAAC,CAAC,CAAC,cAAc,GAAG,mBAAmB,CAAC;YACxC,CAAC,CAAC,CAAC,CACV;YACG,IAAI,CAAC,4BAA4B,CAAC,iBAAiB,EAAE,cAAc,CAAC,CAAC;QACzE,MAAM,OAAO,GAAG,QAAQ;YACpB,CAAC,CAAC,CACE,cAAc;gBACd,mBAAmB;gBACnB,IAAI,CAAC,4BAA4B,CAC7B,iBAAiB,EACjB,cAAc,GAAG,WAAW;oBACxB,CAAC,CAAC,CAAC,cAAc,GAAG,CAAC,CAAC;oBACtB,CAAC,CAAC,cAAc,CACvB,CACJ;YACD,CAAC,CAAC,CAAC,CAAC;QAER,OAAO;YACH,MAAM;YACN,OAAO;SACV,CAAC;IACN,CAAC;IAED;;;OAGG;IACI,uBAAuB,CAAC,SAAiB;QAI5C,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,UAAU,IAAI,EAAE,CAAC;QAEvD,IAAI,SAAS,KAAK,CAAC,EAAE;YACjB,OAAO;gBACH,GAAG,EAAE,UAAU;gBACf,GAAG,EAAE,EAAE;aACV,CAAC;SACL;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QACzC,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,GAAG,SAAS,CAAC,CAAC;QAE1D,MAAM,UAAU,GAAqB,EAAE,CAAC;QACxC,MAAM,UAAU,GAAqB,EAAE,CAAC;QAExC,KAAK,MAAM,gBAAgB,IAAI,UAAU,EAAE;YACvC,MAAM,EAAC,WAAW,EAAC,GAAG,eAAe,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;YAE7D,IAAI,SAAS,KAAK,IAAI,CAAC,WAAW,EAAE;gBAChC,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,YAAY,CAAC;gBAExE,IAAI,YAAY,KAAK,oBAAoB,CAAC,KAAK,IAAI,YAAY,KAAK,oBAAoB,CAAC,KAAK,EAAE;oBAC5F,IAAI,WAAW,IAAI,IAAI,IAAI,WAAW,IAAI,aAAa;wBACnD,UAAU,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;;wBAElC,UAAU,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;oBAEtC,SAAS;iBACZ;aACJ;YAED,IAAI,WAAW,IAAI,IAAI,IAAI,WAAW,IAAI,aAAa;gBACnD,UAAU,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;;gBAElC,UAAU,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SACzC;QAED,OAAO;YACH,GAAG,EAAE,UAAU;YACf,GAAG,EAAE,UAAU;SAClB,CAAC;IACN,CAAC;IAED,gBAAgB;IACT,sCAAsC;QACzC,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;QAEvC,KAAK,MAAM,gBAAgB,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE;YAClE,MAAM,EAAC,WAAW,EAAC,GAAG,eAAe,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;YAE7D,IAAI,WAAW,IAAI,IAAI;gBACnB,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;SACrC;QAED,OAAO,YAAY,CAAC,IAAI,CAAC;IAC7B,CAAC;IAED,gBAAgB;IACT,cAAc;QACjB,OAAO,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,WAAW,IAAI,IAAI,CAAC,sCAAsC,EAAE,CAAC;IAChH,CAAC;IAED,gBAAgB;IACT,4BAA4B,CAAC,WAAmB,EAAE,MAAc;QACnE,+CAA+C;QAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,SAAS,EAAE,UAAU,IAAI,CAAC,CAAC;QACjF,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,gBAAgB,IAAI,CAAC,CAAC;QAC5E,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,SAAS,EAAE,UAAU,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC;QACzH,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,SAAS,EAAE,aAAa,IAAI,KAAK,CAAC;QAC1F,MAAM,cAAc,GAAG,UAAU,GAAG,OAAO,CAAC;QAE5C,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,GAAG,EAAE,WAAW,IAAI,CAAC,CAAC;QAC/E,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,GAAG,EAAE,UAAU,IAAI,CAAC,CAAC;QAC/E,MAAM,YAAY,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC;QAErE,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,SAAS,EAAE,YAAY,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC;QACzH,MAAM,cAAc,GAAG,UAAU,GAAG,OAAO,CAAC;QAE5C,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,GAAG,EAAE,UAAU,IAAI,CAAC,CAAC;QAC/E,MAAM,YAAY,GAAG,SAAS,GAAG,SAAS,CAAC;QAE3C,MAAM,cAAc,GAAG,cAAc,GAAG,YAAY,CAAC;QACrD,MAAM,cAAc,GAAG,cAAc,GAAG,YAAY,CAAC;QAErD,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,YAAY,KAAK,oBAAoB,CAAC,KAAK;YAC/F,sFAAsF;YACtF,0CAA0C;YAC1C,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe;YACrC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC;QAC1C,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,YAAY,KAAK,oBAAoB,CAAC,KAAK;YACjG,sFAAsF;YACtF,0CAA0C;YAC1C,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe;YACrC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC;QAE1C,MAAM,cAAc,GAAG,MAAM,GAAG,cAAc,GAAG,WAAW,GAAG,WAAW,CAAC;QAC3E,MAAM,gBAAgB,GAAG,MAAM,GAAG,cAAc,GAAG,WAAW,GAAG,aAAa,CAAC;QAE/E,OAAO,cAAc,GAAG,gBAAgB,CAAC;IAC7C,CAAC;IAED;;;;;;OAMG;IACI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,YAA0B,EAAE,KAAa;QAC9D,IAAI,aAAa,GAAG,KAAK,CAAC;QAC1B,IAAI,aAAa,IAAI,IAAI;YACrB,aAAa,GAAG,MAAM,sBAAsB,EAAE,CAAC;QAEnD,OAAO,IAAI,YAAY,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;IACzD,CAAC;CACJ;AAED,SAAS,eAAe,CAAC,UAAmB;IAGxC,IAAI,UAAU,IAAI,IAAI;QAClB,OAAO,EAAC,WAAW,EAAE,SAAS,EAAC,CAAC;IAEpC,MAAM,iBAAiB,GAAG,MAAM,CAAC;IACjC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,iBAAiB,CAAC;QACzC,OAAO,EAAC,WAAW,EAAE,SAAS,EAAC,CAAC;IAEpC,MAAM,QAAQ,GAAG,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,iBAAiB,CAAC,MAAM,CAAC,CAAC;IACnE,MAAM,iBAAiB,GAAG,UAAU,CAAC,KAAK,CACtC,iBAAiB,CAAC,MAAM,EACxB,QAAQ,GAAG,CAAC;QACR,CAAC,CAAC,UAAU,CAAC,MAAM;QACnB,CAAC,CAAC,QAAQ,CACjB,CAAC;IAEF,MAAM,WAAW,GAAG,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IAChD,IAAI,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC;QAC5B,OAAO,EAAC,WAAW,EAAC,CAAC;IAEzB,OAAO,EAAC,WAAW,EAAE,SAAS,EAAC,CAAC;AACpC,CAAC;AAED,SAAS,oBAAoB,CAAC,WAA6B,EAAE,KAAY;IACrE,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,UAAU,IAAI,WAAW;QAChC,IAAI,IAAI,mBAAmB,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IAEnD,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAsB,EAAE,KAAY;IAC7D,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAS,CAAC,sBAAsB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACzE,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC,uBAAuB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC3E,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC;IAE9C,IAAI,QAAQ,IAAI,IAAI,IAAI,SAAS,IAAI,IAAI;QACrC,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;IAElD,MAAM,EAAC,EAAE,EAAE,EAAE,EAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,EAAC,QAAQ,EAAE,SAAS,EAAE,WAAW,EAAC,CAAC,CAAC;IAE9E,IAAI,SAAS,KAAK,CAAC,EAAE;QACjB,IAAI,UAAU,GAAG,QAAQ,CAAC;QAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE;YAClC,UAAU,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;SACrC;QAED,OAAO,UAAU,CAAC;KACrB;SAAM;QACH,IAAI,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;QACzD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE;YAClC,UAAU,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;SACrC;QAED,OAAO,UAAU,CAAC;KACrB;AACL,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAsB,EAAE,EAC9C,QAAQ,EAAE,SAAS,EAAE,WAAW,EAGnC;IACG,qBAAqB;IACrB,6CAA6C;IAC7C,MAAM,EAAE,GAAG;QACP,GAAG,MAAM,CAAC,UAAU;QACpB,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KAC1E,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;IAExB,kBAAkB;IAClB,6CAA6C;IAC7C,MAAM,EAAE,GAAG;QACP,QAAQ;QACR,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;QAC1C,GAAG,KAAK,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;KACpC,CAAC;IACF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE;QAClC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;KACjC;IAED,OAAO;QACH,EAAE;QACF,EAAE;KACL,CAAC;AACN,CAAC"}

package/dist/gguf/insights/GgufInsightsConfigurationResolver.d.ts

@@ -0,0 +1,87 @@
+import { BuildGpu } from "../../bindings/types.js";
+import { LlamaModelOptions } from "../../evaluator/LlamaModel.js";
+import { LlamaContextOptions } from "../../evaluator/LlamaContext/types.js";
+import type { GgufInsights } from "./GgufInsights.js";
+export declare class GgufInsightsConfigurationResolver {
+    private constructor();
+    get ggufInsights(): GgufInsights;
+    /**
+     * Score the compatibility of the model configuration with the current GPU and VRAM state.
+     * Assumes a model is loaded with the default `"auto"` configurations.
+     * Scored based on the following criteria:
+     * - The number of GPU layers that can be offloaded to the GPU (only if there's a GPU. If there's no GPU then by how small the model is)
+     * - Whether all layers can be offloaded to the GPU (gives additional points)
+     * - Whether the resolved context size is at least as large as the specified `contextSize`
+     *
+     * IF the resolved context size is larger than the specified context size, for each multiplier of the specified `contextSize`
+     * that the resolved context size is larger by, 1 bonus point is given in the `bonusScore`.
+     *
+     * `contextSize` defaults to `4096` (if the model train context size is lower than this, the model train context size is used instead).
+     */
+    scoreModelConfigurationCompatibility({ contextSize, embeddingContext }?: {
+        contextSize?: number;
+        embeddingContext?: boolean;
+    }, { getVramState, getRamState, llamaVramPaddingSize, llamaGpu, llamaSupportsGpuOffloading }?: {
+        getVramState?(): {
+            total: number;
+            free: number;
+        };
+        getRamState?(): {
+            total: number;
+            free: number;
+        };
+        llamaVramPaddingSize?: number;
+        llamaGpu?: BuildGpu;
+        llamaSupportsGpuOffloading?: boolean;
+    }): {
+        /**
+         * A number between `0` (inclusive) and `1` (inclusive) representing the compatibility score.
+         */
+        compatibilityScore: number;
+        /**
+         * A number starting at `0` with no upper limit representing the bonus score.
+         * For each multiplier of the specified `contextSize` that the resolved context size is larger by, 1 bonus point is given.
+         */
+        bonusScore: number;
+        /**
+         * The total score, which is the sum of the compatibility and bonus scores.
+         */
+        totalScore: number;
+        /**
+         * The resolved values used to calculate the scores.
+         */
+        resolvedValues: {
+            gpuLayers: number;
+            contextSize: number;
+            modelRamUsage: number;
+            contextRamUsage: number;
+            totalRamUsage: number;
+            modelVramUsage: number;
+            contextVramUsage: number;
+            totalVramUsage: number;
+        };
+    };
+    resolveModelGpuLayers(gpuLayers: LlamaModelOptions["gpuLayers"], { ignoreMemorySafetyChecks, getVramState, llamaVramPaddingSize, llamaGpu, llamaSupportsGpuOffloading }?: {
+        ignoreMemorySafetyChecks?: boolean;
+        getVramState?(): {
+            total: number;
+            free: number;
+        };
+        llamaVramPaddingSize?: number;
+        llamaGpu?: BuildGpu;
+        llamaSupportsGpuOffloading?: boolean;
+    }): number;
+    resolveContextContextSize(contextSize: LlamaContextOptions["contextSize"], { modelGpuLayers, batchSize, sequences, modelTrainContextSize, getVramState, llamaGpu, ignoreMemorySafetyChecks, isEmbeddingContext }: {
+        modelGpuLayers: number;
+        modelTrainContextSize: number;
+        batchSize?: LlamaContextOptions["batchSize"];
+        sequences?: number;
+        getVramState?(): {
+            total: number;
+            free: number;
+        };
+        llamaGpu?: BuildGpu;
+        ignoreMemorySafetyChecks?: boolean;
+        isEmbeddingContext?: boolean;
+    }): number;
+}