node-llama-cpp 3.0.0-beta.9 → 3.0.0

package/dist/gguf/fileReaders/GgufFsFileReader.js

@@ -0,0 +1,62 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { withLock } from "lifecycle-utils";
+import { GgufReadOffset } from "../utils/GgufReadOffset.js";
+import { defaultExtraAllocationSize } from "../consts.js";
+import { GgufFileReader } from "./GgufFileReader.js";
+export class GgufFsFileReader extends GgufFileReader {
+    filePath;
+    _signal;
+    constructor({ filePath, signal }) {
+        super();
+        this.filePath = path.resolve(process.cwd(), filePath);
+        this._signal = signal;
+    }
+    readByteRange(offset, length) {
+        const readOffset = GgufReadOffset.resolveReadOffset(offset);
+        const endOffset = readOffset.offset + length;
+        if (endOffset >= this._buffer.length)
+            return this._readToExpandBufferUpToOffset(endOffset)
+                .then(() => {
+                const res = this._buffer.subarray(readOffset.offset, endOffset);
+                readOffset.moveBy(length);
+                return res;
+            });
+        const res = this._buffer.subarray(readOffset.offset, endOffset);
+        readOffset.moveBy(length);
+        return res;
+    }
+    ensureHasByteRange(offset, length) {
+        const readOffset = GgufReadOffset.resolveReadOffset(offset);
+        const endOffset = readOffset.offset + length;
+        if (endOffset >= this._buffer.length)
+            return this._readToExpandBufferUpToOffset(endOffset)
+                .then(() => {
+                if (endOffset >= this._buffer.length)
+                    throw new Error("Expected buffer to be long enough for the requested byte range");
+            });
+        return undefined;
+    }
+    async _readToExpandBufferUpToOffset(endOffset, extraAllocationSize = defaultExtraAllocationSize) {
+        return await withLock(this, "modifyBuffer", this._signal, async () => {
+            if (endOffset < this._buffer.length)
+                return;
+            const missingBytesBuffer = await this._readByteRange(this._buffer.length, endOffset + extraAllocationSize - this._buffer.length);
+            this._addToBuffer(missingBytesBuffer);
+        });
+    }
+    async _readByteRange(start, length) {
+        const fd = await fs.open(this.filePath, "r");
+        try {
+            if (this._signal?.aborted)
+                throw this._signal.reason;
+            const buffer = Buffer.alloc(length);
+            await fd.read(buffer, 0, length, start);
+            return buffer;
+        }
+        finally {
+            await fd.close();
+        }
+    }
+}
+//# sourceMappingURL=GgufFsFileReader.js.map

package/dist/gguf/fileReaders/GgufFsFileReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GgufFsFileReader.js","sourceRoot":"","sources":["../../../src/gguf/fileReaders/GgufFsFileReader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAC,QAAQ,EAAC,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAC,cAAc,EAAC,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAC,0BAA0B,EAAC,MAAM,cAAc,CAAC;AACxD,OAAO,EAAC,cAAc,EAAC,MAAM,qBAAqB,CAAC;AAOnD,MAAM,OAAO,gBAAiB,SAAQ,cAAc;IAChC,QAAQ,CAAS;IAChB,OAAO,CAAe;IAEvC,YAAmB,EAAC,QAAQ,EAAE,MAAM,EAA0B;QAC1D,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,QAAQ,CAAC,CAAC;QACtD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IAC1B,CAAC;IAEM,aAAa,CAAC,MAA+B,EAAE,MAAc;QAChE,MAAM,UAAU,GAAG,cAAc,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC;QAE7C,IAAI,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM;YAChC,OAAO,IAAI,CAAC,6BAA6B,CAAC,SAAS,CAAC;iBAC/C,IAAI,CAAC,GAAG,EAAE;gBACP,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;gBAChE,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC1B,OAAO,GAAG,CAAC;YACf,CAAC,CAAC,CAAC;QAEX,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAChE,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC1B,OAAO,GAAG,CAAC;IACf,CAAC;IAES,kBAAkB,CAAC,MAA+B,EAAE,MAAc;QACxE,MAAM,UAAU,GAAG,cAAc,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC;QAE7C,IAAI,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM;YAChC,OAAO,IAAI,CAAC,6BAA6B,CAAC,SAAS,CAAC;iBAC/C,IAAI,CAAC,GAAG,EAAE;gBACP,IAAI,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM;oBAChC,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;YAC1F,CAAC,CAAC,CAAC;QAEX,OAAO,SAAS,CAAC;IACrB,CAAC;IAEO,KAAK,CAAC,6BAA6B,CAAC,SAAiB,EAAE,sBAA8B,0BAA0B;QACnH,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;YACjE,IAAI,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM;gBAC/B,OAAO;YAEX,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,cAAc,CAChD,IAAI,CAAC,OAAO,CAAC,MAAM,EACnB,SAAS,GAAG,mBAAmB,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CACxD,CAAC;YAEF,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,KAAa,EAAE,MAAc;QACtD,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAC7C,IAAI,CAAC;YACD,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO;gBACrB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAE9B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACpC,MAAM,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;YACxC,OAAO,MAAM,CAAC;QAClB,CAAC;gBAAS,CAAC;YACP,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACL,CAAC;CACJ"}

package/dist/gguf/fileReaders/GgufNetworkFetchFileReader.d.ts

@@ -0,0 +1,25 @@
+import retry from "async-retry";
+import { GgufReadOffset } from "../utils/GgufReadOffset.js";
+import { ModelFileAccessTokens } from "../../utils/modelFileAccesTokens.js";
+import { GgufFileReader } from "./GgufFileReader.js";
+type GgufFetchFileReaderOptions = {
+    url: string;
+    retryOptions?: retry.Options;
+    headers?: Record<string, string>;
+    signal?: AbortSignal;
+    tokens?: ModelFileAccessTokens;
+};
+export declare class GgufNetworkFetchFileReader extends GgufFileReader {
+    readonly url: string;
+    readonly retryOptions: retry.Options;
+    readonly headers: Record<string, string>;
+    readonly tokens?: ModelFileAccessTokens;
+    private readonly _signal?;
+    private _tryHeaders;
+    constructor({ url, retryOptions, headers, tokens, signal }: GgufFetchFileReaderOptions);
+    readByteRange(offset: number | GgufReadOffset, length: number): Buffer | Promise<Buffer>;
+    protected ensureHasByteRange(offset: number | GgufReadOffset, length: number): Promise<void> | undefined;
+    private _fetchToExpandBufferUpToOffset;
+    private _fetchByteRange;
+}
+export {};

package/dist/gguf/fileReaders/GgufNetworkFetchFileReader.js

@@ -0,0 +1,92 @@
+import retry from "async-retry";
+import { withLock } from "lifecycle-utils";
+import { GgufReadOffset } from "../utils/GgufReadOffset.js";
+import { defaultExtraAllocationSize, ggufDefaultFetchRetryOptions } from "../consts.js";
+import { resolveModelFileAccessTokensTryHeaders } from "../../utils/modelFileAccesTokens.js";
+import { GgufFileReader } from "./GgufFileReader.js";
+export class GgufNetworkFetchFileReader extends GgufFileReader {
+    url;
+    retryOptions;
+    headers;
+    tokens;
+    _signal;
+    _tryHeaders = undefined;
+    constructor({ url, retryOptions = ggufDefaultFetchRetryOptions, headers, tokens, signal }) {
+        super();
+        this.url = url;
+        this.retryOptions = retryOptions;
+        this.headers = headers ?? {};
+        this.tokens = tokens;
+        this._signal = signal;
+    }
+    readByteRange(offset, length) {
+        const readOffset = GgufReadOffset.resolveReadOffset(offset);
+        const endOffset = readOffset.offset + length;
+        if (endOffset >= this._buffer.length)
+            return this._fetchToExpandBufferUpToOffset(endOffset)
+                .then(() => {
+                const res = this._buffer.subarray(readOffset.offset, endOffset);
+                readOffset.moveBy(length);
+                return res;
+            });
+        const res = this._buffer.subarray(readOffset.offset, endOffset);
+        readOffset.moveBy(length);
+        return res;
+    }
+    ensureHasByteRange(offset, length) {
+        const readOffset = GgufReadOffset.resolveReadOffset(offset);
+        const endOffset = readOffset.offset + length;
+        if (endOffset >= this._buffer.length)
+            return this._fetchToExpandBufferUpToOffset(endOffset)
+                .then(() => {
+                if (endOffset >= this._buffer.length)
+                    throw new Error("Expected buffer to be long enough for the requested byte range");
+            });
+        return undefined;
+    }
+    async _fetchToExpandBufferUpToOffset(endOffset, extraAllocationSize = defaultExtraAllocationSize) {
+        await withLock(this, "modifyBuffer", this._signal, async () => {
+            if (endOffset < this._buffer.length)
+                return;
+            const missingBytesBuffer = await retry(async (bail) => {
+                try {
+                    return await this._fetchByteRange(this._buffer.length, endOffset + extraAllocationSize - this._buffer.length);
+                }
+                catch (err) {
+                    if (this._signal?.aborted) {
+                        bail(this._signal.reason);
+                        throw this._signal.reason;
+                    }
+                    throw err;
+                }
+            }, this.retryOptions);
+            if (this._signal?.aborted)
+                throw this._signal.reason;
+            this._addToBuffer(missingBytesBuffer);
+        });
+    }
+    async _fetchByteRange(start, length) {
+        if (this._tryHeaders == null)
+            this._tryHeaders = await resolveModelFileAccessTokensTryHeaders(this.url, this.tokens, this.headers);
+        const headersToTry = [this.headers, ...this._tryHeaders];
+        while (headersToTry.length > 0) {
+            const headers = headersToTry.shift();
+            const response = await fetch(this.url, {
+                headers: {
+                    ...headers,
+                    Range: `bytes=${start}-${start + length}`,
+                    accept: "*/*"
+                },
+                signal: this._signal
+            });
+            if ((response.status >= 500 || response.status === 429) && headersToTry.length > 0)
+                continue;
+            if (!response.ok)
+                throw new Error(`Failed to fetch byte range: ${response.status} ${response.statusText}`);
+            const arrayBuffer = await response.arrayBuffer();
+            return Buffer.from(arrayBuffer);
+        }
+        throw new Error("Failed to fetch byte range: no more headers to try");
+    }
+}
+//# sourceMappingURL=GgufNetworkFetchFileReader.js.map

package/dist/gguf/fileReaders/GgufNetworkFetchFileReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GgufNetworkFetchFileReader.js","sourceRoot":"","sources":["../../../src/gguf/fileReaders/GgufNetworkFetchFileReader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,aAAa,CAAC;AAChC,OAAO,EAAC,QAAQ,EAAC,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAC,cAAc,EAAC,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAC,0BAA0B,EAAE,4BAA4B,EAAC,MAAM,cAAc,CAAC;AACtF,OAAO,EAAwB,sCAAsC,EAAC,MAAM,qCAAqC,CAAC;AAClH,OAAO,EAAC,cAAc,EAAC,MAAM,qBAAqB,CAAC;AAUnD,MAAM,OAAO,0BAA2B,SAAQ,cAAc;IAC1C,GAAG,CAAS;IACZ,YAAY,CAAgB;IAC5B,OAAO,CAAyB;IAChC,MAAM,CAAyB;IAC9B,OAAO,CAAe;IAC/B,WAAW,GAAyC,SAAS,CAAC;IAEtE,YAAmB,EAAC,GAAG,EAAE,YAAY,GAAG,4BAA4B,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAA6B;QACtH,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,OAAO,GAAG,OAAO,IAAI,EAAE,CAAC;QAC7B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IAC1B,CAAC;IAEM,aAAa,CAAC,MAA+B,EAAE,MAAc;QAChE,MAAM,UAAU,GAAG,cAAc,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC;QAE7C,IAAI,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM;YAChC,OAAO,IAAI,CAAC,8BAA8B,CAAC,SAAS,CAAC;iBAChD,IAAI,CAAC,GAAG,EAAE;gBACP,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;gBAChE,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC1B,OAAO,GAAG,CAAC;YACf,CAAC,CAAC,CAAC;QAEX,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAChE,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC1B,OAAO,GAAG,CAAC;IACf,CAAC;IAES,kBAAkB,CAAC,MAA+B,EAAE,MAAc;QACxE,MAAM,UAAU,GAAG,cAAc,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC;QAE7C,IAAI,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM;YAChC,OAAO,IAAI,CAAC,8BAA8B,CAAC,SAAS,CAAC;iBAChD,IAAI,CAAC,GAAG,EAAE;gBACP,IAAI,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM;oBAChC,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;YAC1F,CAAC,CAAC,CAAC;QAEX,OAAO,SAAS,CAAC;IACrB,CAAC;IAEO,KAAK,CAAC,8BAA8B,CAAC,SAAiB,EAAE,sBAA8B,0BAA0B;QACpH,MAAM,QAAQ,CAAC,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;YAC1D,IAAI,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM;gBAC/B,OAAO;YAEX,MAAM,kBAAkB,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;gBAClD,IAAI,CAAC;oBACD,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,SAAS,GAAG,mBAAmB,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAClH,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC;wBACxB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;wBAC1B,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;oBAC9B,CAAC;oBAED,MAAM,GAAG,CAAC;gBACd,CAAC;YACL,CAAC,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;YAEtB,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO;gBACrB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAE9B,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,KAAa,EAAE,MAAc;QACvD,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI;YACxB,IAAI,CAAC,WAAW,GAAG,MAAM,sCAAsC,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAEzG,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC;QAEzD,OAAO,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,EAAE,CAAC;YAErC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE;gBACnC,OAAO,EAAE;oBACL,GAAG,OAAO;oBACV,KAAK,EAAE,SAAS,KAAK,IAAI,KAAK,GAAG,MAAM,EAAE;oBACzC,MAAM,EAAE,KAAK;iBAChB;gBACD,MAAM,EAAE,IAAI,CAAC,OAAO;aACvB,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC;gBAC9E,SAAS;YAEb,IAAI,CAAC,QAAQ,CAAC,EAAE;gBACZ,MAAM,IAAI,KAAK,CAAC,+BAA+B,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YAE7F,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;YACjD,OAAO,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACpC,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IAC1E,CAAC;CACJ"}

package/dist/gguf/insights/GgufInsights.d.ts

@@ -0,0 +1,50 @@
+import { Llama } from "../../bindings/Llama.js";
+import { GgufFileInfo } from "../types/GgufFileInfoTypes.js";
+import { GgufInsightsConfigurationResolver } from "./GgufInsightsConfigurationResolver.js";
+export type GgufInsightsResourceRequirements = {
+    cpuRam: number;
+    gpuVram: number;
+};
+export declare class GgufInsights {
+    private constructor();
+    /**
+     * Get warnings about the model file that would affect its usage.
+     *
+     * Most of these warnings are also generated by `llama.cpp`
+     */
+    getWarnings(modelFilePath?: string): string[];
+    get ggufFileInfo(): GgufFileInfo;
+    get configurationResolver(): GgufInsightsConfigurationResolver;
+    /** The context size the model was trained on */
+    get trainContextSize(): number | undefined;
+    /** The size of an embedding vector the model can produce */
+    get embeddingVectorSize(): number | undefined;
+    get totalLayers(): number;
+    get modelSize(): number;
+    get flashAttentionSupported(): boolean;
+    estimateModelResourceRequirements({ gpuLayers }: {
+        gpuLayers: number;
+    }): GgufInsightsResourceRequirements;
+    /**
+     * Estimates the memory required to create a context of the given parameters based on the implementation details of `llama.cpp`.
+     * The calculation doesn't include a precise estimation of the graph overhead memory, so it uses a rough estimate for that.
+     * The estimation for the graph overhead memory will be improved in the future to be more precise, but it's good enough for now.
+     */
+    estimateContextResourceRequirements({ contextSize, modelGpuLayers, batchSize, sequences, isEmbeddingContext, includeGraphOverhead, flashAttention }: {
+        contextSize: number;
+        modelGpuLayers: number;
+        batchSize?: number;
+        sequences?: number;
+        isEmbeddingContext?: boolean;
+        flashAttention?: boolean;
+        includeGraphOverhead?: boolean;
+    }): GgufInsightsResourceRequirements;
+    /**
+     * @param ggufFileInfo
+     * @param llama - If you already have a `Llama` instance, pass it to reuse it for the `GgufInsights` instance.
+     * If you don't pass a `Llama` instance, a basic `Llama` instance is created as a fallback - it's a slim instance that
+     * doesn't instantiate a `llama.cpp` backend, so it won't utilize the GPU at all, and be shared with other `GgufInsights` instances
+     * that need a fallback `Llama` instance.
+     */
+    static from(ggufFileInfo: GgufFileInfo, llama?: Llama): Promise<GgufInsights>;
+}

package/dist/gguf/insights/GgufInsights.js

@@ -0,0 +1,401 @@
+import { getLlamaWithoutBackend } from "../../bindings/utils/getLlamaWithoutBackend.js";
+import { getDefaultContextBatchSize, getDefaultContextSequences } from "../../evaluator/LlamaContext/LlamaContext.js";
+import { GgufArchitectureType } from "../types/GgufMetadataTypes.js";
+import { getReadablePath } from "../../cli/utils/getReadablePath.js";
+import { GgufInsightsConfigurationResolver } from "./GgufInsightsConfigurationResolver.js";
+export class GgufInsights {
+    /** @internal */ _llama;
+    /** @internal */ _modelSize;
+    /** @internal */ _totalLayers = null;
+    /** @internal */ _ggufFileInfo;
+    /** @internal */ _configurationResolver;
+    constructor(ggufFileInfo, llama) {
+        this._llama = llama;
+        this._ggufFileInfo = ggufFileInfo;
+        this._modelSize = calculateTensorsSize(ggufFileInfo.fullTensorInfo ?? [], llama);
+        this._configurationResolver = GgufInsightsConfigurationResolver._create(this);
+    }
+    /**
+     * Get warnings about the model file that would affect its usage.
+     *
+     * Most of these warnings are also generated by `llama.cpp`
+     */
+    getWarnings(modelFilePath) {
+        const warnings = [];
+        const modelFilePathText = (modelFilePath != null && modelFilePath !== "")
+            ? ` ("${getReadablePath(modelFilePath)}")`
+            : "";
+        if (this._ggufFileInfo?.metadata?.tokenizer?.ggml?.model === "gpt2" &&
+            this._ggufFileInfo?.metadata?.tokenizer?.ggml?.model == null) {
+            // equivalent to the warning in `llama.cpp` under `llm_load_vocab`: "missing pre-tokenizer type, using: 'default'"
+            warnings.push(`This model file${modelFilePathText} is missing a pre-tokenizer configuration. ` +
+                "This may cause incorrect tokenization and thus degrade the generation quality. " +
+                "Consider using a newer model or regenerating this GGUF model file");
+        }
+        return warnings;
+    }
+    get ggufFileInfo() {
+        return this._ggufFileInfo;
+    }
+    get configurationResolver() {
+        return this._configurationResolver;
+    }
+    /** The context size the model was trained on */
+    get trainContextSize() {
+        return this._ggufFileInfo.architectureMetadata.context_length;
+    }
+    /** The size of an embedding vector the model can produce */
+    get embeddingVectorSize() {
+        return this._ggufFileInfo.architectureMetadata.embedding_length;
+    }
+    get totalLayers() {
+        if (this._totalLayers != null)
+            return this._totalLayers;
+        const outputLayers = 1;
+        this._totalLayers = this._getFileLayers() + outputLayers;
+        return this._totalLayers;
+    }
+    get modelSize() {
+        return this._modelSize;
+    }
+    get flashAttentionSupported() {
+        // source: `llama_new_context_with_model` in `llama.cpp`
+        if (this._ggufFileInfo.metadata?.general?.architecture === GgufArchitectureType.grok)
+            return false;
+        else if (this._ggufFileInfo.metadata?.general?.architecture === GgufArchitectureType.gemma2)
+            return false;
+        else {
+            const nHead = this._ggufFileInfo.architectureMetadata?.attention?.head_count ?? 0;
+            const nEmbd = this._ggufFileInfo.architectureMetadata?.embedding_length ?? 0;
+            const nEmbdHeadK = this._ggufFileInfo.architectureMetadata?.attention?.key_length ?? ((nHead == 0) ? 0 : (nEmbd / nHead));
+            const nEmbdHeadV = this._ggufFileInfo.architectureMetadata?.attention?.value_length ?? ((nHead == 0) ? 0 : nEmbd / nHead);
+            if (nEmbdHeadK !== nEmbdHeadV)
+                return false;
+        }
+        return true;
+    }
+    estimateModelResourceRequirements({ gpuLayers }) {
+        const { cpu, gpu } = this._getTensorResourceSplit(gpuLayers);
+        return {
+            cpuRam: calculateTensorsSize(cpu, this._llama),
+            gpuVram: calculateTensorsSize(gpu, this._llama)
+        };
+    }
+    /**
+     * Estimates the memory required to create a context of the given parameters based on the implementation details of `llama.cpp`.
+     * The calculation doesn't include a precise estimation of the graph overhead memory, so it uses a rough estimate for that.
+     * The estimation for the graph overhead memory will be improved in the future to be more precise, but it's good enough for now.
+     */
+    estimateContextResourceRequirements({ contextSize, modelGpuLayers, batchSize, sequences, isEmbeddingContext = false, includeGraphOverhead = true, flashAttention = false }) {
+        if (sequences == null)
+            sequences = getDefaultContextSequences();
+        if (batchSize == null)
+            batchSize = getDefaultContextBatchSize({ contextSize, sequences });
+        const actualContextSize = contextSize * sequences;
+        const totalLayers = this.totalLayers;
+        const finalGpuLayers = Math.max(0, Math.min(modelGpuLayers ?? totalLayers, totalLayers));
+        const finalCpuLayers = totalLayers - finalGpuLayers;
+        const llmData = this._ggufFileInfo.architectureMetadata;
+        const vocabularySize = llmData.vocab_size ?? this._ggufFileInfo.metadata.tokenizer?.ggml?.tokens?.length ?? 0;
+        const logitsSize = vocabularySize * batchSize;
+        const embedSize = isEmbeddingContext
+            ? (llmData.embedding_length ?? 0) * batchSize
+            : 0;
+        const sizeTBytes = 8; // sizeof(size_t)
+        const floatBytes = 4; // sizeof(float)
+        const uint32TBytes = 4; // sizeof(uint32_t)
+        const int32TBytes = 4; // sizeof(int32_t)
+        // source: `llama_state_get_size` in `llama.cpp`
+        const sRngSize = sizeTBytes;
+        const sRng = 64 * 1024; // LLAMA_MAX_RNG_STATE
+        const sNOutputs = sizeTBytes;
+        const sNOutputPos = batchSize * int32TBytes;
+        const sLogitsSize = sizeTBytes;
+        const sLogits = logitsSize * floatBytes;
+        const sEmbeddingSize = sizeTBytes;
+        const sEmbedding = embedSize * floatBytes;
+        const sKvBufSize = sizeTBytes;
+        const sKvHead = uint32TBytes;
+        const sKvSize = uint32TBytes;
+        const sKvUsed = uint32TBytes;
+        const sKv = 2 * int32TBytes * modelGpuLayers * this._llama._consts.ggmlTensorOverhead;
+        const sKvCell = this._llama._consts.llamaPosSize + sizeTBytes + this._llama._consts.llamaSeqIdSize;
+        const kvSelfLength = this._ggufFileInfo.metadata.general?.architecture === GgufArchitectureType.mamba
+            ? Math.max(1, sequences)
+            : actualContextSize;
+        const sKvCells = kvSelfLength * sKvCell;
+        const overheadMemory = (sRngSize +
+            sRng +
+            sNOutputs +
+            sNOutputPos +
+            sLogitsSize +
+            sLogits +
+            sEmbeddingSize +
+            sEmbedding +
+            sKvBufSize +
+            sKvHead +
+            sKvSize +
+            sKvUsed +
+            sKv +
+            sKvCells);
+        // Estimates the memory allocated by `ggml_backend_sched_reserve` in `llama_new_context_with_model` in `llama.cpp`.
+        // If you read this line and have better insights on how to estimate this memory, please open a PR to improve it :)
+        const estimateGraphOverheadMemory = () => {
+            const s1MB = Math.pow(1024, 2);
+            const tensorInfo = this._ggufFileInfo.fullTensorInfo ?? [];
+            let defaultCalculationAdjustment = 0;
+            if (batchSize == null)
+                return 0;
+            if (this._ggufFileInfo.metadata.general?.architecture === GgufArchitectureType.llama) {
+                const expertCount = this._ggufFileInfo.architectureMetadata.expert_count ?? 0;
+                const headCount = this._ggufFileInfo.architectureMetadata.attention?.head_count ?? 0;
+                const embeddingLength = llmData.embedding_length ?? 0;
+                if (expertCount > 0) {
+                    const expertsUsedCount = this._ggufFileInfo.architectureMetadata.expert_used_count ?? 2;
+                    return int32TBytes * batchSize * (((expertsUsedCount + 1) * embeddingLength) + (actualContextSize * headCount));
+                }
+                return int32TBytes * batchSize * (embeddingLength + (actualContextSize * headCount));
+            }
+            else if (this._ggufFileInfo.metadata.general?.architecture === GgufArchitectureType.qwen2) {
+                if (modelGpuLayers === this.totalLayers) {
+                    defaultCalculationAdjustment -= (s1MB * 340) * (this.trainContextSize == null
+                        ? 1
+                        : actualContextSize / this.trainContextSize);
+                }
+                else {
+                    defaultCalculationAdjustment -= (s1MB * 250) + ((s1MB * 50) * (this.trainContextSize == null
+                        ? 1
+                        : actualContextSize / this.trainContextSize));
+                }
+            }
+            else if (this._ggufFileInfo.metadata.general?.architecture === GgufArchitectureType.gemma) {
+                // only works properly when all layers are on the GPU, which is why it's commented out:
+                // return int32TBytes * batchSize * ((llmData.embedding_length ?? 0));
+                if (modelGpuLayers === this.totalLayers) {
+                    defaultCalculationAdjustment += (s1MB * 40) - ((s1MB * 270) * (this.trainContextSize == null
+                        ? 1
+                        : actualContextSize / this.trainContextSize));
+                }
+                else {
+                    defaultCalculationAdjustment += -(s1MB * 550) + ((s1MB * 150) * (this.trainContextSize == null
+                        ? 1
+                        : Math.max(0, (1 - (actualContextSize / this.trainContextSize)))));
+                }
+            }
+            else if (this._ggufFileInfo.metadata.general?.architecture === GgufArchitectureType.stablelm) {
+                const headCount = this._ggufFileInfo.architectureMetadata.attention?.head_count ?? 0;
+                return (int32TBytes * batchSize * actualContextSize * headCount) - (50 * s1MB);
+                // if (modelGpuLayers === this.totalLayers) {
+                //     defaultCalculationAdjustment += -(s1MB * 20) + (
+                //         (s1MB * 250) * (
+                //             this.trainContextSize == null
+                //                 ? 1
+                //                 : actualContextSize / this.trainContextSize
+                //         )
+                //     );
+                // } else {
+                //     defaultCalculationAdjustment += -(s1MB * 40) + (
+                //         (s1MB * 300) * (
+                //             this.trainContextSize == null
+                //                 ? 1
+                //                 : actualContextSize / this.trainContextSize
+                //         )
+                //     );
+                // }
+            }
+            const totalElements = tensorInfo.length === 0
+                ? this.totalLayers * (((llmData.embedding_length ?? 0) +
+                    (llmData.feed_forward_length ?? 0)) / 2)
+                : tensorInfo.reduce((res, tensor) => {
+                    return res + tensor.dimensions.reduce((res, dim) => res + Number(dim), 0);
+                }, 0);
+            if (this._ggufFileInfo.metadata.general?.architecture === GgufArchitectureType.phi3) {
+                // magic numbers for estimation. will be improved in the future
+                return (totalElements * 123 * (actualContextSize / 4096)) + defaultCalculationAdjustment;
+            }
+            // magic numbers for estimation. will be improved in the future
+            return (totalElements * 77.655 * (actualContextSize / 4096)) + defaultCalculationAdjustment;
+        };
+        const graphOverheadMemory = (flashAttention || !includeGraphOverhead)
+            ? 0
+            : estimateGraphOverheadMemory();
+        const usingGpu = finalGpuLayers !== 0;
+        const cpuRam = (!usingGpu
+            ? (overheadMemory + graphOverheadMemory)
+            : 0) +
+            this._estimateKvMemorySizeInBytes(actualContextSize, finalCpuLayers);
+        const gpuVram = usingGpu
+            ? (overheadMemory +
+                graphOverheadMemory +
+                this._estimateKvMemorySizeInBytes(actualContextSize, finalGpuLayers < totalLayers
+                    ? (finalGpuLayers + 1)
+                    : finalGpuLayers))
+            : 0;
+        return {
+            cpuRam,
+            gpuVram
+        };
+    }
+    /**
+     * Get the split tensor resources for CPU and GPU based on the number of GPU layers
+     * @internal
+     */
+    _getTensorResourceSplit(gpuLayers) {
+        const tensorInfo = this._ggufFileInfo.fullTensorInfo ?? [];
+        if (gpuLayers === 0) {
+            return {
+                cpu: tensorInfo,
+                gpu: []
+            };
+        }
+        const fileLayers = this._getFileLayers();
+        const startGpuLayer = Math.max(0, fileLayers - gpuLayers);
+        const gpuTensors = [];
+        const cpuTensors = [];
+        for (const singleTensorInfo of tensorInfo) {
+            const { layerNumber } = parseTensorName(singleTensorInfo.name);
+            if (gpuLayers !== this.totalLayers) {
+                const architecture = this._ggufFileInfo.metadata?.general?.architecture;
+                if (architecture === GgufArchitectureType.qwen2 || architecture === GgufArchitectureType.gemma) {
+                    if (layerNumber != null && layerNumber >= startGpuLayer)
+                        gpuTensors.push(singleTensorInfo);
+                    else
+                        cpuTensors.push(singleTensorInfo);
+                    continue;
+                }
+            }
+            if (layerNumber == null || layerNumber >= startGpuLayer)
+                gpuTensors.push(singleTensorInfo);
+            else
+                cpuTensors.push(singleTensorInfo);
+        }
+        return {
+            cpu: cpuTensors,
+            gpu: gpuTensors
+        };
+    }
+    /** @internal */
+    _determineNumberOfLayersFromTensorInfo() {
+        const layerNumbers = new Set();
+        for (const singleTensorInfo of (this._ggufFileInfo.fullTensorInfo ?? [])) {
+            const { layerNumber } = parseTensorName(singleTensorInfo.name);
+            if (layerNumber != null)
+                layerNumbers.add(layerNumber);
+        }
+        return layerNumbers.size;
+    }
+    /** @internal */
+    _getFileLayers() {
+        return this._ggufFileInfo.architectureMetadata.block_count ?? this._determineNumberOfLayersFromTensorInfo();
+    }
+    /** @internal */
+    _estimateKvMemorySizeInBytes(contextSize, layers) {
+        // source: `llama_kv_cache_init` in `llama.cpp`
+        const nHead = this._ggufFileInfo.architectureMetadata.attention?.head_count ?? 0;
+        const nEmbd = this._ggufFileInfo.architectureMetadata.embedding_length ?? 0;
+        const nEmbdHeadK = this._ggufFileInfo.architectureMetadata.attention?.key_length ?? ((nHead == 0) ? 0 : (nEmbd / nHead));
+        const nHeadKv = this._ggufFileInfo.architectureMetadata.attention?.head_count_kv ?? nHead;
+        const modelNEmbdKGqa = nEmbdHeadK * nHeadKv;
+        const ssmDConv = this._ggufFileInfo.architectureMetadata.ssm?.conv_kernel ?? 0;
+        const ssmDInner = this._ggufFileInfo.architectureMetadata.ssm?.inner_size ?? 0;
+        const modelNEmbdKS = (ssmDConv > 0 ? (ssmDConv - 1) : 0) * ssmDInner;
+        const nEmbdHeadV = this._ggufFileInfo.architectureMetadata.attention?.value_length ?? ((nHead == 0) ? 0 : nEmbd / nHead);
+        const modelNEmbdVGqa = nEmbdHeadV * nHeadKv;
+        const ssmDState = this._ggufFileInfo.architectureMetadata.ssm?.state_size ?? 0;
+        const modelNEmbdVS = ssmDState * ssmDInner;
+        const totalNEmbdKGqa = modelNEmbdKGqa + modelNEmbdKS;
+        const totalNEmbdVGqa = modelNEmbdVGqa + modelNEmbdVS;
+        const keyTypeSize = this._ggufFileInfo.metadata.general?.architecture === GgufArchitectureType.mamba
+            // if `type_k` of `llama_context_params` changes to be configurable in `LlamaContext`,
+            // this would have to depend on that value
+            ? this._llama._consts.ggmlTypeF32Size
+            : this._llama._consts.ggmlTypeF16Size;
+        const valueTypeSize = this._ggufFileInfo.metadata.general?.architecture === GgufArchitectureType.mamba
+            // if `type_v` of `llama_context_params` changes to be configurable in `LlamaContext`,
+            // this would have to depend on that value
+            ? this._llama._consts.ggmlTypeF32Size
+            : this._llama._consts.ggmlTypeF16Size;
+        const keyTensorsSize = layers * totalNEmbdKGqa * contextSize * keyTypeSize;
+        const valueTensorsSize = layers * totalNEmbdVGqa * contextSize * valueTypeSize;
+        return keyTensorsSize + valueTensorsSize;
+    }
+    /**
+     * @param ggufFileInfo
+     * @param llama - If you already have a `Llama` instance, pass it to reuse it for the `GgufInsights` instance.
+     * If you don't pass a `Llama` instance, a basic `Llama` instance is created as a fallback - it's a slim instance that
+     * doesn't instantiate a `llama.cpp` backend, so it won't utilize the GPU at all, and be shared with other `GgufInsights` instances
+     * that need a fallback `Llama` instance.
+     */
+    static async from(ggufFileInfo, llama) {
+        let resolvedLlama = llama;
+        if (resolvedLlama == null)
+            resolvedLlama = await getLlamaWithoutBackend();
+        return new GgufInsights(ggufFileInfo, resolvedLlama);
+    }
+}
+function parseTensorName(tensorName) {
+    if (tensorName == null)
+        return { layerNumber: undefined };
+    const layerTensorPrefix = "blk.";
+    if (!tensorName.startsWith(layerTensorPrefix))
+        return { layerNumber: undefined };
+    const dotIndex = tensorName.indexOf(".", layerTensorPrefix.length);
+    const layerNumberString = tensorName.slice(layerTensorPrefix.length, dotIndex < 0
+        ? tensorName.length
+        : dotIndex);
+    const layerNumber = parseInt(layerNumberString);
+    if (Number.isFinite(layerNumber))
+        return { layerNumber };
+    return { layerNumber: undefined };
+}
+function calculateTensorsSize(tensorsInfo, llama) {
+    let size = 0;
+    for (const tensorInfo of tensorsInfo)
+        size += calculateTensorSize(tensorInfo, llama);
+    return size;
+}
+function calculateTensorSize(tensor, llama) {
+    const typeSize = llama._bindings.getTypeSizeForGgmlType(tensor.ggmlType);
+    const blockSize = llama._bindings.getBlockSizeForGgmlType(tensor.ggmlType);
+    const ggmlMaxDims = llama._consts.ggmlMaxDims;
+    if (typeSize == null || blockSize == null)
+        throw new Error("Invalid type or block size");
+    const { ne, nb } = getTensorNeAndNb(tensor, { typeSize, blockSize, ggmlMaxDims });
+    if (blockSize === 1) {
+        let totalBytes = typeSize;
+        for (let i = 0; i < ggmlMaxDims; i++) {
+            totalBytes += (ne[i] - 1) * nb[i];
+        }
+        return totalBytes;
+    }
+    else {
+        let totalBytes = Math.floor((ne[0] * nb[0]) / blockSize);
+        for (let i = 1; i < ggmlMaxDims; i++) {
+            totalBytes += (ne[i] - 1) * nb[i];
+        }
+        return totalBytes;
+    }
+}
+function getTensorNeAndNb(tensor, { typeSize, blockSize, ggmlMaxDims }) {
+    // number of elements
+    // source: `ggml_new_tensor_impl` in `ggml.c`
+    const ne = [
+        ...tensor.dimensions,
+        ...(Array(Math.max(0, ggmlMaxDims - tensor.dimensions.length)).fill(1))
+    ].slice(0, ggmlMaxDims);
+    // number of bytes
+    // source: `ggml_new_tensor_impl` in `ggml.c`
+    const nb = [
+        typeSize,
+        Math.floor(typeSize * (ne[0] / blockSize)),
+        ...Array(ggmlMaxDims - 2).fill(0)
+    ];
+    for (let i = 2; i < ggmlMaxDims; i++) {
+        nb[i] = nb[i - 1] * ne[i - 1];
+    }
+    return {
+        ne,
+        nb
+    };
+}
+//# sourceMappingURL=GgufInsights.js.map