@realtimex/node-llama-cpp 0.1.0

package/dist/gguf/fileReaders/GgufFsFileReader.js

@@ -0,0 +1,62 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { withLock } from "lifecycle-utils";
+import { GgufReadOffset } from "../utils/GgufReadOffset.js";
+import { defaultExtraAllocationSize } from "../consts.js";
+import { GgufFileReader } from "./GgufFileReader.js";
+export class GgufFsFileReader extends GgufFileReader {
+    filePath;
+    _signal;
+    constructor({ filePath, signal }) {
+        super();
+        this.filePath = path.resolve(process.cwd(), filePath);
+        this._signal = signal;
+    }
+    readByteRange(offset, length) {
+        const readOffset = GgufReadOffset.resolveReadOffset(offset);
+        const endOffset = readOffset.offset + length;
+        if (endOffset >= this._buffer.length)
+            return this._readToExpandBufferUpToOffset(endOffset)
+                .then(() => {
+                const res = this._buffer.subarray(readOffset.offset, endOffset);
+                readOffset.moveBy(length);
+                return res;
+            });
+        const res = this._buffer.subarray(readOffset.offset, endOffset);
+        readOffset.moveBy(length);
+        return res;
+    }
+    ensureHasByteRange(offset, length) {
+        const readOffset = GgufReadOffset.resolveReadOffset(offset);
+        const endOffset = readOffset.offset + length;
+        if (endOffset >= this._buffer.length)
+            return this._readToExpandBufferUpToOffset(endOffset)
+                .then(() => {
+                if (endOffset >= this._buffer.length)
+                    throw new Error("Expected buffer to be long enough for the requested byte range");
+            });
+        return undefined;
+    }
+    async _readToExpandBufferUpToOffset(endOffset, extraAllocationSize = defaultExtraAllocationSize) {
+        return await withLock([this, "modifyBuffer"], this._signal, async () => {
+            if (endOffset < this._buffer.length)
+                return;
+            const missingBytesBuffer = await this._readByteRange(this._buffer.length, endOffset + extraAllocationSize - this._buffer.length);
+            this._addToBuffer(missingBytesBuffer);
+        });
+    }
+    async _readByteRange(start, length) {
+        const fd = await fs.open(this.filePath, "r");
+        try {
+            if (this._signal?.aborted)
+                throw this._signal.reason;
+            const buffer = Buffer.alloc(length);
+            await fd.read(buffer, 0, length, start);
+            return buffer;
+        }
+        finally {
+            await fd.close();
+        }
+    }
+}
+//# sourceMappingURL=GgufFsFileReader.js.map

package/dist/gguf/fileReaders/GgufFsFileReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GgufFsFileReader.js","sourceRoot":"","sources":["../../../src/gguf/fileReaders/GgufFsFileReader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAC,QAAQ,EAAC,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAC,cAAc,EAAC,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAC,0BAA0B,EAAC,MAAM,cAAc,CAAC;AACxD,OAAO,EAAC,cAAc,EAAC,MAAM,qBAAqB,CAAC;AAOnD,MAAM,OAAO,gBAAiB,SAAQ,cAAc;IAChC,QAAQ,CAAS;IAChB,OAAO,CAAe;IAEvC,YAAmB,EAAC,QAAQ,EAAE,MAAM,EAA0B;QAC1D,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,QAAQ,CAAC,CAAC;QACtD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IAC1B,CAAC;IAEM,aAAa,CAAC,MAA+B,EAAE,MAAc;QAChE,MAAM,UAAU,GAAG,cAAc,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC;QAE7C,IAAI,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM;YAChC,OAAO,IAAI,CAAC,6BAA6B,CAAC,SAAS,CAAC;iBAC/C,IAAI,CAAC,GAAG,EAAE;gBACP,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;gBAChE,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC1B,OAAO,GAAG,CAAC;YACf,CAAC,CAAC,CAAC;QAEX,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAChE,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC1B,OAAO,GAAG,CAAC;IACf,CAAC;IAES,kBAAkB,CAAC,MAA+B,EAAE,MAAc;QACxE,MAAM,UAAU,GAAG,cAAc,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC;QAE7C,IAAI,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM;YAChC,OAAO,IAAI,CAAC,6BAA6B,CAAC,SAAS,CAAC;iBAC/C,IAAI,CAAC,GAAG,EAAE;gBACP,IAAI,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM;oBAChC,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;YAC1F,CAAC,CAAC,CAAC;QAEX,OAAO,SAAS,CAAC;IACrB,CAAC;IAEO,KAAK,CAAC,6BAA6B,CAAC,SAAiB,EAAE,sBAA8B,0BAA0B;QACnH,OAAO,MAAM,QAAQ,CAAC,CAAC,IAAwB,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;YACvF,IAAI,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM;gBAC/B,OAAO;YAEX,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,cAAc,CAChD,IAAI,CAAC,OAAO,CAAC,MAAM,EACnB,SAAS,GAAG,mBAAmB,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CACxD,CAAC;YAEF,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,KAAa,EAAE,MAAc;QACtD,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAC7C,IAAI,CAAC;YACD,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO;gBACrB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAE9B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACpC,MAAM,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;YACxC,OAAO,MAAM,CAAC;QAClB,CAAC;gBAAS,CAAC;YACP,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACL,CAAC;CACJ"}

package/dist/gguf/fileReaders/GgufNetworkFetchFileReader.d.ts

@@ -0,0 +1,28 @@
+import retry from "async-retry";
+import { GgufReadOffset } from "../utils/GgufReadOffset.js";
+import { ModelFileAccessTokens } from "../../utils/modelFileAccessTokens.js";
+import { ModelDownloadEndpoints } from "../../utils/modelDownloadEndpoints.js";
+import { GgufFileReader } from "./GgufFileReader.js";
+type GgufFetchFileReaderOptions = {
+    url: string;
+    retryOptions?: retry.Options;
+    headers?: Record<string, string>;
+    signal?: AbortSignal;
+    tokens?: ModelFileAccessTokens;
+    endpoints?: ModelDownloadEndpoints;
+};
+export declare class GgufNetworkFetchFileReader extends GgufFileReader {
+    readonly url: string;
+    readonly retryOptions: retry.Options;
+    readonly headers: Record<string, string>;
+    readonly tokens?: ModelFileAccessTokens;
+    readonly endpoints?: ModelDownloadEndpoints;
+    private readonly _signal?;
+    private _tryHeaders;
+    constructor({ url, retryOptions, headers, tokens, endpoints, signal }: GgufFetchFileReaderOptions);
+    readByteRange(offset: number | GgufReadOffset, length: number): Buffer<ArrayBuffer> | Promise<Buffer<ArrayBuffer>>;
+    protected ensureHasByteRange(offset: number | GgufReadOffset, length: number): Promise<void> | undefined;
+    private _fetchToExpandBufferUpToOffset;
+    private _fetchByteRange;
+}
+export {};

package/dist/gguf/fileReaders/GgufNetworkFetchFileReader.js

@@ -0,0 +1,94 @@
+import retry from "async-retry";
+import { withLock } from "lifecycle-utils";
+import { GgufReadOffset } from "../utils/GgufReadOffset.js";
+import { defaultExtraAllocationSize, ggufDefaultFetchRetryOptions } from "../consts.js";
+import { resolveModelFileAccessTokensTryHeaders } from "../../utils/modelFileAccessTokens.js";
+import { GgufFileReader } from "./GgufFileReader.js";
+export class GgufNetworkFetchFileReader extends GgufFileReader {
+    url;
+    retryOptions;
+    headers;
+    tokens;
+    endpoints;
+    _signal;
+    _tryHeaders = undefined;
+    constructor({ url, retryOptions = ggufDefaultFetchRetryOptions, headers, tokens, endpoints, signal }) {
+        super();
+        this.url = url;
+        this.retryOptions = retryOptions;
+        this.headers = headers ?? {};
+        this.tokens = tokens;
+        this.endpoints = endpoints;
+        this._signal = signal;
+    }
+    readByteRange(offset, length) {
+        const readOffset = GgufReadOffset.resolveReadOffset(offset);
+        const endOffset = readOffset.offset + length;
+        if (endOffset >= this._buffer.length)
+            return this._fetchToExpandBufferUpToOffset(endOffset)
+                .then(() => {
+                const res = this._buffer.subarray(readOffset.offset, endOffset);
+                readOffset.moveBy(length);
+                return res;
+            });
+        const res = this._buffer.subarray(readOffset.offset, endOffset);
+        readOffset.moveBy(length);
+        return res;
+    }
+    ensureHasByteRange(offset, length) {
+        const readOffset = GgufReadOffset.resolveReadOffset(offset);
+        const endOffset = readOffset.offset + length;
+        if (endOffset >= this._buffer.length)
+            return this._fetchToExpandBufferUpToOffset(endOffset)
+                .then(() => {
+                if (endOffset >= this._buffer.length)
+                    throw new Error("Expected buffer to be long enough for the requested byte range");
+            });
+        return undefined;
+    }
+    async _fetchToExpandBufferUpToOffset(endOffset, extraAllocationSize = defaultExtraAllocationSize) {
+        await withLock([this, "modifyBuffer"], this._signal, async () => {
+            if (endOffset < this._buffer.length)
+                return;
+            const missingBytesBuffer = await retry(async (bail) => {
+                try {
+                    return await this._fetchByteRange(this._buffer.length, endOffset + extraAllocationSize - this._buffer.length);
+                }
+                catch (err) {
+                    if (this._signal?.aborted) {
+                        bail(this._signal.reason);
+                        throw this._signal.reason;
+                    }
+                    throw err;
+                }
+            }, this.retryOptions);
+            if (this._signal?.aborted)
+                throw this._signal.reason;
+            this._addToBuffer(missingBytesBuffer);
+        });
+    }
+    async _fetchByteRange(start, length) {
+        if (this._tryHeaders == null)
+            this._tryHeaders = await resolveModelFileAccessTokensTryHeaders(this.url, this.tokens, this.endpoints, this.headers);
+        const headersToTry = [this.headers, ...this._tryHeaders];
+        while (headersToTry.length > 0) {
+            const headers = headersToTry.shift();
+            const response = await fetch(this.url, {
+                headers: {
+                    ...headers,
+                    Range: `bytes=${start}-${start + length}`,
+                    accept: "*/*"
+                },
+                signal: this._signal
+            });
+            if ((response.status >= 500 || response.status === 429 || response.status === 401) && headersToTry.length > 0)
+                continue;
+            if (!response.ok)
+                throw new Error(`Failed to fetch byte range: ${response.status} ${response.statusText}`);
+            const arrayBuffer = await response.arrayBuffer();
+            return Buffer.from(arrayBuffer);
+        }
+        throw new Error("Failed to fetch byte range: no more headers to try");
+    }
+}
+//# sourceMappingURL=GgufNetworkFetchFileReader.js.map

package/dist/gguf/fileReaders/GgufNetworkFetchFileReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GgufNetworkFetchFileReader.js","sourceRoot":"","sources":["../../../src/gguf/fileReaders/GgufNetworkFetchFileReader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,aAAa,CAAC;AAChC,OAAO,EAAC,QAAQ,EAAC,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAC,cAAc,EAAC,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAC,0BAA0B,EAAE,4BAA4B,EAAC,MAAM,cAAc,CAAC;AACtF,OAAO,EAAwB,sCAAsC,EAAC,MAAM,sCAAsC,CAAC;AAEnH,OAAO,EAAC,cAAc,EAAC,MAAM,qBAAqB,CAAC;AAWnD,MAAM,OAAO,0BAA2B,SAAQ,cAAc;IAC1C,GAAG,CAAS;IACZ,YAAY,CAAgB;IAC5B,OAAO,CAAyB;IAChC,MAAM,CAAyB;IAC/B,SAAS,CAA0B;IAClC,OAAO,CAAe;IAC/B,WAAW,GAAyC,SAAS,CAAC;IAEtE,YAAmB,EAAC,GAAG,EAAE,YAAY,GAAG,4BAA4B,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAA6B;QACjI,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,OAAO,GAAG,OAAO,IAAI,EAAE,CAAC;QAC7B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IAC1B,CAAC;IAEM,aAAa,CAAC,MAA+B,EAAE,MAAc;QAChE,MAAM,UAAU,GAAG,cAAc,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC;QAE7C,IAAI,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM;YAChC,OAAO,IAAI,CAAC,8BAA8B,CAAC,SAAS,CAAC;iBAChD,IAAI,CAAC,GAAG,EAAE;gBACP,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;gBAChE,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC1B,OAAO,GAAG,CAAC;YACf,CAAC,CAAC,CAAC;QAEX,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAChE,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC1B,OAAO,GAAG,CAAC;IACf,CAAC;IAES,kBAAkB,CAAC,MAA+B,EAAE,MAAc;QACxE,MAAM,UAAU,GAAG,cAAc,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC;QAE7C,IAAI,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM;YAChC,OAAO,IAAI,CAAC,8BAA8B,CAAC,SAAS,CAAC;iBAChD,IAAI,CAAC,GAAG,EAAE;gBACP,IAAI,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM;oBAChC,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;YAC1F,CAAC,CAAC,CAAC;QAEX,OAAO,SAAS,CAAC;IACrB,CAAC;IAEO,KAAK,CAAC,8BAA8B,CAAC,SAAiB,EAAE,sBAA8B,0BAA0B;QACpH,MAAM,QAAQ,CAAC,CAAC,IAAkC,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;YAC1F,IAAI,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM;gBAC/B,OAAO;YAEX,MAAM,kBAAkB,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;gBAClD,IAAI,CAAC;oBACD,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,SAAS,GAAG,mBAAmB,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAClH,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC;wBACxB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;wBAC1B,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;oBAC9B,CAAC;oBAED,MAAM,GAAG,CAAC;gBACd,CAAC;YACL,CAAC,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;YAEtB,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO;gBACrB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAE9B,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,KAAa,EAAE,MAAc;QACvD,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI;YACxB,IAAI,CAAC,WAAW,GAAG,MAAM,sCAAsC,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAEzH,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC;QAEzD,OAAO,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,EAAE,CAAC;YAErC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE;gBACnC,OAAO,EAAE;oBACL,GAAG,OAAO;oBACV,KAAK,EAAE,SAAS,KAAK,IAAI,KAAK,GAAG,MAAM,EAAE;oBACzC,MAAM,EAAE,KAAK;iBAChB;gBACD,MAAM,EAAE,IAAI,CAAC,OAAO;aACvB,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC;gBACzG,SAAS;YAEb,IAAI,CAAC,QAAQ,CAAC,EAAE;gBACZ,MAAM,IAAI,KAAK,CAAC,+BAA+B,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YAE7F,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;YACjD,OAAO,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACpC,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IAC1E,CAAC;CACJ"}

package/dist/gguf/insights/GgufInsights.d.ts

@@ -0,0 +1,78 @@
+import { Llama } from "../../bindings/Llama.js";
+import { GgufFileInfo } from "../types/GgufFileInfoTypes.js";
+import { GgmlType, GgufTensorInfo } from "../types/GgufTensorInfoTypes.js";
+import { GgufInsightsConfigurationResolver } from "./GgufInsightsConfigurationResolver.js";
+import { GgufInsightsTokens } from "./GgufInsightsTokens.js";
+export type GgufInsightsResourceRequirements = {
+    cpuRam: number;
+    gpuVram: number;
+};
+export declare class GgufInsights {
+    private constructor();
+    /**
+     * Get warnings about the model file that would affect its usage.
+     *
+     * Most of these warnings are also generated by `llama.cpp`
+     */
+    getWarnings(modelFilePath?: string): string[];
+    get ggufFileInfo(): GgufFileInfo;
+    get configurationResolver(): GgufInsightsConfigurationResolver;
+    get tokens(): GgufInsightsTokens;
+    /** The context size the model was trained on */
+    get trainContextSize(): number | undefined;
+    /** The size of an embedding vector the model can produce */
+    get embeddingVectorSize(): number | undefined;
+    get totalLayers(): number;
+    get modelSize(): number;
+    get flashAttentionSupported(): boolean;
+    get hasEncoder(): boolean;
+    get hasDecoder(): boolean;
+    get isRecurrent(): boolean;
+    get isHybrid(): boolean;
+    /**
+     * Get the dominant tensor type used in the model file
+     */
+    get dominantTensorType(): GgmlType | undefined;
+    get supportsRanking(): boolean;
+    /**
+     * The size of the SWA (Sliding Window Attention).
+     *
+     * When `undefined`, the model does not use sliding window attention.
+     */
+    get swaSize(): number | undefined;
+    estimateModelResourceRequirements({ gpuLayers, useMmap, gpuSupportsMmap }: {
+        gpuLayers: number;
+        useMmap?: boolean;
+        gpuSupportsMmap?: boolean;
+    }): GgufInsightsResourceRequirements;
+    /**
+     * Estimates the memory required to create a context of the given parameters based on the implementation details of `llama.cpp`.
+     * The calculation doesn't include a precise estimation of the graph overhead memory, so it uses a rough estimate for that.
+     * The estimation for the graph overhead memory will be improved in the future to be more precise, but it's good enough for now.
+     */
+    estimateContextResourceRequirements({ contextSize, modelGpuLayers, batchSize, sequences, isEmbeddingContext, includeGraphOverhead, flashAttention, swaFullCache, kvCacheKeyType, kvCacheValueType }: {
+        contextSize: number;
+        modelGpuLayers: number;
+        batchSize?: number;
+        sequences?: number;
+        isEmbeddingContext?: boolean;
+        flashAttention?: boolean;
+        includeGraphOverhead?: boolean;
+        swaFullCache?: boolean;
+        kvCacheKeyType?: GgmlType;
+        kvCacheValueType?: GgmlType;
+    }): GgufInsightsResourceRequirements;
+    private _estimateContextCacheMemorySplitInBytes;
+    private _estimateRecurrentStateMemorySizeInBytes;
+    /**
+     * @param ggufFileInfo
+     * @param llama - If you already have a `Llama` instance, pass it to reuse it for the `GgufInsights` instance.
+     * If you don't pass a `Llama` instance, a basic `Llama` instance is created as a fallback - it's a slim instance that
+     * doesn't instantiate a `llama.cpp` backend, so it won't utilize the GPU at all, and be shared with other `GgufInsights` instances
+     * that need a fallback `Llama` instance.
+     */
+    static from(ggufFileInfo: GgufFileInfo, llama?: Llama): Promise<GgufInsights>;
+}
+export declare function parseRankingTemplate(template: string | undefined | null): string | undefined;
+export declare function isRankingTemplateValid(template: string | undefined | null): boolean;
+export declare function getDominantTensorType(tensorInfo: GgufTensorInfo[]): GgmlType | undefined;