node-llama-cpp 3.0.0-beta.11 → 3.0.0-beta.13

package/dist/gguf/ggufParser/GGUFParser.js

@@ -0,0 +1,123 @@
+import InvalidGGUFMagicError from "../errors/InvalidGGUFMagicError.js";
+import UnsupportedMetadataTypeError from "../errors/UnsupportedMetadataTypeError.js";
+import { fileTypeIntToString } from "./checkArchitecture.js";
+import GGUFBaseStream, { METHOD_TO_BYTE_COUNT } from "./stream/GGUFBaseStream.js";
+const METADATA_VALUE_TO_METHOD = {
+    0: "readUint8",
+    1: "readInt8",
+    2: "readUint16",
+    3: "readInt16",
+    4: "readUint32",
+    5: "readInt32",
+    6: "readFloat32",
+    7: "readBool"
+};
+const METADATA_STRING = 8;
+const METADATA_ARRAY = 9;
+const GGUF_MAGIC = "GGUF";
+const DEFAULT_IGNORE_METADATA_KEYS = ["tokenizer.ggml.tokens", "tokenizer.ggml.scores", "tokenizer.ggml.token_type", "tokenizer.ggml.merges"];
+export default class GGUFParser {
+    _stream;
+    ignoreKeys = DEFAULT_IGNORE_METADATA_KEYS;
+    constructor(_stream, ignoreKeys = DEFAULT_IGNORE_METADATA_KEYS) {
+        this.ignoreKeys = ignoreKeys;
+        this._stream = _stream;
+    }
+    async _readMetadataValue(type, offset) {
+        const numberMethod = METADATA_VALUE_TO_METHOD[type];
+        if (numberMethod) {
+            return {
+                value: await this._stream[numberMethod](offset),
+                newOffset: offset + METHOD_TO_BYTE_COUNT[numberMethod]
+            };
+        }
+        if (METADATA_STRING === type) {
+            const { string, newOffset } = await this._stream.readString(offset);
+            return { value: string, newOffset };
+        }
+        if (METADATA_ARRAY === type) {
+            const arrayType = await this._stream.readUint32(offset);
+            offset += METHOD_TO_BYTE_COUNT.readUint32;
+            const arrayLength = await this._stream.readUint64(offset);
+            offset += METHOD_TO_BYTE_COUNT.readUint64;
+            const arrayValues = [];
+            for (let i = 0; i < arrayLength; i++) {
+                const { value, newOffset } = await this._readMetadataValue(arrayType, offset);
+                arrayValues.push(value);
+                offset = newOffset;
+            }
+            return { value: arrayValues, newOffset: offset };
+        }
+        throw new UnsupportedMetadataTypeError(type);
+    }
+    async _parseMetadataRaw() {
+        let offset = 0;
+        const readMagicBytesLength = METHOD_TO_BYTE_COUNT.readUint8 * GGUF_MAGIC.length;
+        const magicBytes = await this._stream.readNBytes(readMagicBytesLength);
+        offset += readMagicBytesLength;
+        const magicText = String.fromCharCode(...magicBytes);
+        if (magicText !== GGUF_MAGIC) {
+            throw new InvalidGGUFMagicError();
+        }
+        const version = await this._stream.readUint32(offset);
+        offset += METHOD_TO_BYTE_COUNT.readUint32;
+        const tensorCount = await this._stream.readUint64(offset);
+        offset += METHOD_TO_BYTE_COUNT.readUint64;
+        const metadataKVCount = await this._stream.readUint64(offset);
+        offset += METHOD_TO_BYTE_COUNT.readUint64;
+        const metadata = {
+            version,
+            tensorCount: GGUFBaseStream.castNumber(tensorCount)
+        };
+        for (let i = 0; i < Number(metadataKVCount); i++) {
+            // read key
+            const keyResult = await this._stream.readString(offset);
+            offset = keyResult.newOffset;
+            // read the value type
+            const valueType = await this._stream.readUint32(offset);
+            offset += METHOD_TO_BYTE_COUNT.readUint32;
+            // read value
+            const valueResult = await this._readMetadataValue(valueType, offset);
+            offset = valueResult.newOffset;
+            metadata[keyResult.string] = valueResult.value;
+        }
+        return {
+            metadata: metadata,
+            metadataSize: offset
+        };
+    }
+    async parseMetadata() {
+        const metadataRaw = await this._parseMetadataRaw();
+        const metadata = {};
+        for (const [key, value] of Object.entries(metadataRaw.metadata)) {
+            if (this.ignoreKeys.includes(key)) {
+                continue;
+            }
+            const { lastObject, lastKey } = GGUFParser._getNestedObject(key, metadata);
+            lastObject[lastKey] = value;
+        }
+        if (typeof metadata?.general?.file_type === "number") {
+            metadata.general["file_type"] = fileTypeIntToString(metadata.general.file_type) || metadata.general.file_type;
+        }
+        return {
+            metadata: metadata,
+            metadataSize: metadataRaw.metadataSize
+        };
+    }
+    static _getNestedObject(key, currentNestedObject) {
+        const nestedKey = key.split(".");
+        const lastKey = nestedKey.pop();
+        while (nestedKey.length > 0) {
+            const currentKey = nestedKey.shift();
+            if (!currentNestedObject[currentKey]) {
+                currentNestedObject[currentKey] = {};
+            }
+            currentNestedObject = currentNestedObject[currentKey];
+        }
+        return {
+            lastObject: currentNestedObject,
+            lastKey
+        };
+    }
+}
+//# sourceMappingURL=GGUFParser.js.map

package/dist/gguf/ggufParser/GGUFParser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GGUFParser.js","sourceRoot":"","sources":["../../../src/gguf/ggufParser/GGUFParser.ts"],"names":[],"mappings":"AAAA,OAAO,qBAAqB,MAAM,oCAAoC,CAAC;AACvE,OAAO,4BAA4B,MAAM,2CAA2C,CAAC;AACrF,OAAO,EAAC,mBAAmB,EAAC,MAAM,wBAAwB,CAAC;AAE3D,OAAO,cAAc,EAAE,EAAC,oBAAoB,EAAC,MAAM,4BAA4B,CAAC;AAEhF,MAAM,wBAAwB,GAAyD;IACnF,CAAC,EAAE,WAAW;IACd,CAAC,EAAE,UAAU;IACb,CAAC,EAAE,YAAY;IACf,CAAC,EAAE,WAAW;IACd,CAAC,EAAE,YAAY;IACf,CAAC,EAAE,WAAW;IACd,CAAC,EAAE,aAAa;IAChB,CAAC,EAAE,UAAU;CAChB,CAAC;AAEF,MAAM,eAAe,GAAG,CAAC,CAAC;AAC1B,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB,MAAM,UAAU,GAAG,MAAM,CAAC;AAE1B,MAAM,4BAA4B,GAAG,CAAC,uBAAuB,EAAE,uBAAuB,EAAE,2BAA2B,EAAE,uBAAuB,CAAC,CAAC;AAO9I,MAAM,CAAC,OAAO,OAAO,UAAU;IACR,OAAO,CAAiB;IAEpC,UAAU,GAAG,4BAA4B,CAAC;IAEjD,YAAmB,OAAuB,EAAE,UAAU,GAAG,4BAA4B;QACjF,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,IAAmD,EAAE,MAAc;QAIhG,MAAM,YAAY,GAAG,wBAAwB,CAAC,IAAI,CAAC,CAAC;QACpD,IAAI,YAAY,EAAE;YACd,OAAO;gBACH,KAAK,EAAE,MAAM,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC;gBAC/C,SAAS,EAAE,MAAM,GAAG,oBAAoB,CAAC,YAAY,CAAC;aACzD,CAAC;SACL;QAED,IAAI,eAAe,KAAK,IAAI,EAAE;YAC1B,MAAM,EAAC,MAAM,EAAE,SAAS,EAAC,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAClE,OAAO,EAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAC,CAAC;SACrC;QAED,IAAI,cAAc,KAAK,IAAI,EAAE;YACzB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YACxD,MAAM,IAAI,oBAAoB,CAAC,UAAU,CAAC;YAE1C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAC1D,MAAM,IAAI,oBAAoB,CAAC,UAAU,CAAC;YAE1C,MAAM,WAAW,GAAG,EAAE,CAAC;YACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE;gBAClC,MAAM,EAAC,KAAK,EAAE,SAAS,EAAC,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;gBAC5E,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACxB,MAAM,GAAG,SAAS,CAAC;aACtB;YACD,OAAO,EAAC,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,EAAC,CAAC;SAClD;QAED,MAAM,IAAI,4BAA4B,CAAC,IAAI,CAAC,CAAC;IACjD,CAAC;IAEO,KAAK,CAAC,iBAAiB;QAC3B,IAAI,MAAM,GAAG,CAAC,CAAC;QAEf,MAAM,oBAAoB,GAAG,oBAAoB,CAAC,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC;QAChF,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;QACvE,MAAM,IAAI,oBAAoB,CAAC;QAE/B,MAAM,SAAS,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,UAAU,CAAC,CAAC;QACrD,IAAI,SAAS,KAAK,UAAU,EAAE;YAC1B,MAAM,IAAI,qBAAqB,EAAE,CAAC;SACrC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QACtD,MAAM,IAAI,oBAAoB,CAAC,UAAU,CAAC;QAE1C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAC1D,MAAM,IAAI,oBAAoB,CAAC,UAAU,CAAC;QAE1C,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAC9D,MAAM,IAAI,oBAAoB,CAAC,UAAU,CAAC;QAE1C,MAAM,QAAQ,GAA2B;YACrC,OAAO;YACP,WAAW,EAAE,cAAc,CAAC,UAAU,CAAC,WAAqB,CAAC;SAChE,CAAC;QAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC,EAAE,EAAE;YAC9C,WAAW;YACX,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YACxD,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC;YAE7B,sBAAsB;YACtB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YACxD,MAAM,IAAI,oBAAoB,CAAC,UAAU,CAAC;YAE1C,aAAa;YACb,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YACrE,MAAM,GAAG,WAAW,CAAC,SAAS,CAAC;YAC/B,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC;SAClD;QAED,OAAO;YACH,QAAQ,EAAE,QAAQ;YAClB,YAAY,EAAE,MAAM;SACvB,CAAC;IACN,CAAC;IAEM,KAAK,CAAC,aAAa;QACtB,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACnD,MAAM,QAAQ,GAA2B,EAAE,CAAC;QAE5C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE;YAC7D,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;gBAC/B,SAAS;aACZ;YACD,MAAM,EAAC,UAAU,EAAE,OAAO,EAAC,GAAG,UAAU,CAAC,gBAAgB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;YACzE,UAAU,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC;SAC/B;QAED,IAAI,OAAO,QAAQ,EAAE,OAAO,EAAE,SAAS,KAAK,QAAQ,EAAE;YAClD,QAAQ,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,mBAAmB,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC;SACjH;QAED,OAAO;YACH,QAAQ,EAAE,QAA2B;YACrC,YAAY,EAAE,WAAW,CAAC,YAAY;SACzC,CAAC;IACN,CAAC;IAES,MAAM,CAAC,gBAAgB,CAAC,GAAW,EAAE,mBAAwB;QACnE,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACjC,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,EAAG,CAAC;QAEjC,OAAO,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE;YACzB,MAAM,UAAU,GAAG,SAAS,CAAC,KAAK,EAAG,CAAC;YACtC,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,EAAE;gBAClC,mBAAmB,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;aACxC;YACD,mBAAmB,GAAG,mBAAmB,CAAC,UAAU,CAAC,CAAC;SACzD;QAED,OAAO;YACH,UAAU,EAAE,mBAAmB;YAC/B,OAAO;SACV,CAAC;IACN,CAAC;CACJ"}

package/dist/gguf/ggufParser/GGUFTypes.d.ts

@@ -0,0 +1,257 @@
+import { fileTypeIntToString } from "./checkArchitecture.js";
+export type GGUFArchitectureType = "llama" | "falcon" | "mpt" | "gptneox" | "gptj" | "gpt2" | "bloom" | "rwkv" | "whisper";
+export type GGUFMetadataArchitectureProperties = {
+    context_length: number;
+    embedding_length: number;
+    block_count: number;
+    feed_forward_length: number;
+    use_parallel_residual: boolean;
+    tensor_data_layout: string;
+    expert_count: number;
+    expert_used_count: number;
+    attention: {
+        head_count: number;
+        head_count_kv: number;
+        max_alibi_bias: number;
+        clamp_kqv: number;
+        key_length: number;
+        value_length: number;
+        layer_norm_epsilon: number;
+        layer_norm_rms_epsilon: number;
+    };
+    rope: {
+        dimension_count: number;
+        freq_base: number;
+        scaling: {
+            type: string;
+            factor: number;
+            original_context_length: number;
+            finetuned: boolean;
+        };
+    };
+};
+export type GGUFMetadataGeneralProperties = {
+    architecture: GGUFArchitectureType;
+    /**
+     * The version of the quantization format. Not required if the model is not
+     * quantized (i.e. no tensors are quantized). If any tensors are quantized,
+     * this must be present. This is separate to the quantization scheme of the
+     * tensors itself; the quantization version may change without changing the
+     * scheme's name (e.g. the quantization scheme is Q5_K, and the quantization
+     * version is 4).
+     */
+    quantization_version: string;
+    /**
+     * the global alignment to use, as described above. This can vary to allow
+     * for different alignment schemes, but it must be a multiple of 8. Some
+     * writers may not write the alignment. If the alignment is not specified,
+     * assume it is `32`.
+     */
+    alignment: string;
+    /**
+     * The name of the model. This should be a human-readable name that can be
+     * used to identify the model. It should be unique within the community
+     * that the model is defined in.
+     */
+    name: string;
+    author: string;
+    /**
+     * URL to the model's homepage. This can be a GitHub repo, a paper, etc.
+     */
+    url: string;
+    /**
+     * free-form description of the model including anything that isn't
+     * covered by the other fields
+     */
+    description: string;
+    /**
+     * License of the model, expressed as a SPDX license expression
+     * (e.g. `MIT OR Apache-2.0`). *Should not* include any other information,
+     * such as the license text or the URL to the license.
+     */
+    license: string;
+    /**
+     * Information about where this model came from. This is useful for tracking
+     * the provenance of the model, and for finding the original source if the
+     * model is modified. For a model that was converted from GGML, for
+     * example, these keys would point to the model that was converted from.
+     */
+    source: {
+        /**
+         * URL to the source of the model. Can be a GitHub repo, a paper, etc.
+         */
+        url: string;
+        huggingface: {
+            repository: string;
+        };
+    };
+    /**
+     * An enumerated value describing the type of the majority of the tensors
+     * in the file. Optional; can be inferred from the tensor types.
+     */
+    file_type: ReturnType<typeof fileTypeIntToString>;
+};
+export type GGUFMetadataAny = {
+    general: GGUFMetadataGeneralProperties;
+} & {
+    [key in GGUFArchitectureType]: GGUFMetadataArchitectureProperties;
+};
+export type GGUFMetadataLLAMA = {
+    general: GGUFMetadataGeneralProperties & {
+        architecture: "llama";
+    };
+    llama: {
+        context_length: number;
+        embedding_length: number;
+        block_count: number;
+        feed_forward_length: number;
+        attention: {
+            head_count: number;
+            layer_norm_rms_epsilon: number;
+            head_count_kv?: number;
+        };
+        rope: {
+            dimension_count: number;
+            scale?: number;
+        };
+        expert_count?: number;
+        expert_used_count?: number;
+        tensor_data_layout?: string;
+    };
+};
+export type GGUFMetadataFalcon = {
+    general: GGUFMetadataGeneralProperties & {
+        architecture: "falcon";
+    };
+    falcon: {
+        context_length: number;
+        embedding_length: number;
+        block_count: number;
+        attention: {
+            head_count: number;
+            head_count_kv: number;
+            use_norm: boolean;
+            layer_norm_epsilon: number;
+        };
+        tensor_data_layout?: string;
+    };
+};
+export type GGUFMetadataMPT = {
+    general: GGUFMetadataGeneralProperties & {
+        architecture: "mpt";
+    };
+    mpt: {
+        context_length: number;
+        embedding_length: number;
+        block_count: number;
+        attention: {
+            head_count: number;
+            alibi_bias_max: number;
+            clip_kqv: number;
+            layer_norm_epsilon: number;
+        };
+    };
+};
+export type GGUFMetadataGPTNeoX = {
+    general: GGUFMetadataGeneralProperties & {
+        architecture: "gptneox";
+    };
+    gptneox: {
+        context_length: number;
+        embedding_length: number;
+        block_count: number;
+        use_parallel_residual: boolean;
+        rope: {
+            dimension_count: number;
+            freq_base: number;
+            scale?: number;
+        };
+        attention: {
+            head_count: number;
+            layer_norm_epsilon: number;
+        };
+    };
+};
+export type GGUFMetadataGPTJ = {
+    general: GGUFMetadataGeneralProperties & {
+        architecture: "gptj";
+    };
+    gptj: {
+        context_length: number;
+        embedding_length: number;
+        block_count: number;
+        rope: {
+            dimension_count: number;
+            scale?: number;
+        };
+        attention: {
+            head_count: number;
+            layer_norm_epsilon: number;
+        };
+    };
+};
+export type GGUFMetadataGPT2 = {
+    general: GGUFMetadataGeneralProperties & {
+        architecture: "gpt2";
+    };
+    gpt2: {
+        context_length: number;
+        embedding_length: number;
+        block_count: number;
+        attention: {
+            head_count: number;
+            layer_norm_epsilon: number;
+        };
+    };
+};
+export type GGUFMetadataBloom = {
+    general: GGUFMetadataGeneralProperties & {
+        architecture: "bloom";
+    };
+    bloom: {
+        context_length: number;
+        embedding_length: number;
+        block_count: number;
+        feed_forward_length: number;
+        attention: {
+            head_count: number;
+            layer_norm_epsilon: number;
+        };
+    };
+};
+export type GGUFMetadataRWKV = {
+    general: GGUFMetadataGeneralProperties & {
+        architecture: "rwkv";
+    };
+    rwkv: {
+        context_length: number;
+        block_count: number;
+        embedding_length: number;
+        feed_forward_length: number;
+    };
+};
+export type GGUFMetadataWhisper = {
+    general: GGUFMetadataGeneralProperties & {
+        architecture: "whisper";
+    };
+    whisper: {
+        encoder: {
+            context_length: number;
+            embedding_length: number;
+            block_count: number;
+            mels_count: number;
+            attention: {
+                head_count: number;
+            };
+        };
+        decoder: {
+            context_length: number;
+            embedding_length: number;
+            block_count: number;
+            attention: {
+                head_count: number;
+            };
+        };
+    };
+};
+export type GGUFMetadata = GGUFMetadataLLAMA | GGUFMetadataFalcon | GGUFMetadataMPT | GGUFMetadataGPTNeoX | GGUFMetadataGPTJ | GGUFMetadataGPT2 | GGUFMetadataBloom | GGUFMetadataRWKV | GGUFMetadataWhisper;

package/dist/gguf/ggufParser/GGUFTypes.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=GGUFTypes.js.map

package/dist/gguf/ggufParser/GGUFTypes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GGUFTypes.js","sourceRoot":"","sources":["../../../src/gguf/ggufParser/GGUFTypes.ts"],"names":[],"mappings":""}

package/dist/gguf/ggufParser/checkArchitecture.d.ts

@@ -0,0 +1,14 @@
+import { GGUFMetadata, GGUFMetadataBloom, GGUFMetadataFalcon, GGUFMetadataGPT2, GGUFMetadataGPTJ, GGUFMetadataGPTNeoX, GGUFMetadataLLAMA, GGUFMetadataMPT, GGUFMetadataRWKV } from "./GGUFTypes.js";
+export declare function isLlamaMetadata(metadata: GGUFMetadata): metadata is GGUFMetadataLLAMA;
+export declare function isMPTMetadata(metadata: GGUFMetadata): metadata is GGUFMetadataMPT;
+export declare function isGPTNeoXMetadata(metadata: GGUFMetadata): metadata is GGUFMetadataGPTNeoX;
+export declare function isGPTJMetadata(metadata: GGUFMetadata): metadata is GGUFMetadataGPTJ;
+export declare function isGPT2Metadata(metadata: GGUFMetadata): metadata is GGUFMetadataGPT2;
+export declare function isBloomMetadata(metadata: GGUFMetadata): metadata is GGUFMetadataBloom;
+export declare function isFalconMetadata(metadata: GGUFMetadata): metadata is GGUFMetadataFalcon;
+export declare function isRWKVMetadata(metadata: GGUFMetadata): metadata is GGUFMetadataRWKV;
+/**
+ * https://github.com/ggerganov/ggml/blob/master/docs/gguf.md#general-metadata
+ * Convert file type from string to int
+ */
+export declare function fileTypeIntToString(fileType?: number): "ALL_F32" | "MOSTLY_F16" | "MOSTLY_Q4_0" | "MOSTLY_Q4_1" | "MOSTLY_Q4_1_SOME_F16" | "MOSTLY_Q4_2" | "MOSTLY_Q4_3" | "MOSTLY_Q8_0" | "MOSTLY_Q5_0" | "MOSTLY_Q5_1" | "MOSTLY_Q2_K" | "MOSTLY_Q3_K_S" | "MOSTLY_Q3_K_M" | "MOSTLY_Q3_K_L" | "MOSTLY_Q4_K_S" | "MOSTLY_Q4_K_M" | "MOSTLY_Q5_K_S" | "MOSTLY_Q5_K_M" | "MOSTLY_Q6_K" | undefined;

package/dist/gguf/ggufParser/checkArchitecture.js

@@ -0,0 +1,74 @@
+export function isLlamaMetadata(metadata) {
+    return metadata.general.architecture === "llama";
+}
+export function isMPTMetadata(metadata) {
+    return metadata.general.architecture === "mpt";
+}
+export function isGPTNeoXMetadata(metadata) {
+    return metadata.general.architecture === "gptneox";
+}
+export function isGPTJMetadata(metadata) {
+    return metadata.general.architecture === "gptj";
+}
+export function isGPT2Metadata(metadata) {
+    return metadata.general.architecture === "gpt2";
+}
+export function isBloomMetadata(metadata) {
+    return metadata.general.architecture === "bloom";
+}
+export function isFalconMetadata(metadata) {
+    return metadata.general.architecture === "falcon";
+}
+export function isRWKVMetadata(metadata) {
+    return metadata.general.architecture === "rwkv";
+}
+/**
+ * https://github.com/ggerganov/ggml/blob/master/docs/gguf.md#general-metadata
+ * Convert file type from string to int
+ */
+export function fileTypeIntToString(fileType) {
+    if (fileType == null)
+        return;
+    switch (fileType) {
+        case 0:
+            return "ALL_F32";
+        case 1:
+            return "MOSTLY_F16";
+        case 2:
+            return "MOSTLY_Q4_0";
+        case 3:
+            return "MOSTLY_Q4_1";
+        case 4:
+            return "MOSTLY_Q4_1_SOME_F16";
+        case 5:
+            return "MOSTLY_Q4_2";
+        case 6:
+            return "MOSTLY_Q4_3";
+        case 7:
+            return "MOSTLY_Q8_0";
+        case 8:
+            return "MOSTLY_Q5_0";
+        case 9:
+            return "MOSTLY_Q5_1";
+        case 10:
+            return "MOSTLY_Q2_K";
+        case 11:
+            return "MOSTLY_Q3_K_S";
+        case 12:
+            return "MOSTLY_Q3_K_M";
+        case 13:
+            return "MOSTLY_Q3_K_L";
+        case 14:
+            return "MOSTLY_Q4_K_S";
+        case 15:
+            return "MOSTLY_Q4_K_M";
+        case 16:
+            return "MOSTLY_Q5_K_S";
+        case 17:
+            return "MOSTLY_Q5_K_M";
+        case 18:
+            return "MOSTLY_Q6_K";
+    }
+    return;
+}
+//# sourceMappingURL=checkArchitecture.js.map

package/dist/gguf/ggufParser/checkArchitecture.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"checkArchitecture.js","sourceRoot":"","sources":["../../../src/gguf/ggufParser/checkArchitecture.ts"],"names":[],"mappings":"AAWA,MAAM,UAAU,eAAe,CAAE,QAAsB;IACnD,OAAO,QAAQ,CAAC,OAAO,CAAC,YAAY,KAAK,OAAO,CAAC;AACrD,CAAC;AAED,MAAM,UAAU,aAAa,CAAE,QAAsB;IACjD,OAAO,QAAQ,CAAC,OAAO,CAAC,YAAY,KAAK,KAAK,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAE,QAAsB;IACrD,OAAO,QAAQ,CAAC,OAAO,CAAC,YAAY,KAAK,SAAS,CAAC;AACvD,CAAC;AAED,MAAM,UAAU,cAAc,CAAE,QAAsB;IAClD,OAAO,QAAQ,CAAC,OAAO,CAAC,YAAY,KAAK,MAAM,CAAC;AACpD,CAAC;AAED,MAAM,UAAU,cAAc,CAAE,QAAsB;IAClD,OAAO,QAAQ,CAAC,OAAO,CAAC,YAAY,KAAK,MAAM,CAAC;AACpD,CAAC;AAED,MAAM,UAAU,eAAe,CAAE,QAAsB;IACnD,OAAO,QAAQ,CAAC,OAAO,CAAC,YAAY,KAAK,OAAO,CAAC;AACrD,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAE,QAAsB;IACpD,OAAO,QAAQ,CAAC,OAAO,CAAC,YAAY,KAAK,QAAQ,CAAC;AACtD,CAAC;AAED,MAAM,UAAU,cAAc,CAAE,QAAsB;IAClD,OAAO,QAAQ,CAAC,OAAO,CAAC,YAAY,KAAK,MAAM,CAAC;AACpD,CAAC;AAGD;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,QAAiB;IACjD,IAAI,QAAQ,IAAI,IAAI;QAAE,OAAO;IAC7B,QAAQ,QAAQ,EAAE;QACd,KAAK,CAAC;YACF,OAAO,SAAS,CAAC;QACrB,KAAK,CAAC;YACF,OAAO,YAAY,CAAC;QACxB,KAAK,CAAC;YACF,OAAO,aAAa,CAAC;QACzB,KAAK,CAAC;YACF,OAAO,aAAa,CAAC;QACzB,KAAK,CAAC;YACF,OAAO,sBAAsB,CAAC;QAClC,KAAK,CAAC;YACF,OAAO,aAAa,CAAC;QACzB,KAAK,CAAC;YACF,OAAO,aAAa,CAAC;QACzB,KAAK,CAAC;YACF,OAAO,aAAa,CAAC;QACzB,KAAK,CAAC;YACF,OAAO,aAAa,CAAC;QACzB,KAAK,CAAC;YACF,OAAO,aAAa,CAAC;QACzB,KAAK,EAAE;YACH,OAAO,aAAa,CAAC;QACzB,KAAK,EAAE;YACH,OAAO,eAAe,CAAC;QAC3B,KAAK,EAAE;YACH,OAAO,eAAe,CAAC;QAC3B,KAAK,EAAE;YACH,OAAO,eAAe,CAAC;QAC3B,KAAK,EAAE;YACH,OAAO,eAAe,CAAC;QAC3B,KAAK,EAAE;YACH,OAAO,eAAe,CAAC;QAC3B,KAAK,EAAE;YACH,OAAO,eAAe,CAAC;QAC3B,KAAK,EAAE;YACH,OAAO,eAAe,CAAC;QAC3B,KAAK,EAAE;YACH,OAAO,aAAa,CAAC;KAC5B;IAED,OAAO;AACX,CAAC"}

package/dist/gguf/ggufParser/stream/GGUFBaseStream.d.ts

@@ -0,0 +1,38 @@
+/// <reference types="node" />
+import { Buffer } from "buffer";
+export declare const METHOD_TO_BYTE_COUNT: {
+    readUint8: number;
+    readUint16: number;
+    readUint32: number;
+    readUint64: number;
+    readInt8: number;
+    readInt16: number;
+    readInt32: number;
+    readInt64: number;
+    readFloat32: number;
+    readFloat64: number;
+    readBool: number;
+};
+export declare const ALLOCATION_SIZE: number;
+export default abstract class GGUFBaseStream {
+    protected _buffer: Buffer;
+    protected constructor();
+    abstract readNBytes(numBytes: number, offset?: number): Promise<Buffer>;
+    readUint8(offset: number): Promise<number>;
+    readUint16(offset: number): Promise<number>;
+    readUint32(offset: number): Promise<number>;
+    readUint64(offset: number): Promise<bigint>;
+    readInt8(offset: number): Promise<number>;
+    readInt16(offset: number): Promise<number>;
+    readInt32(offset: number): Promise<number>;
+    readInt64(offset: number): Promise<bigint>;
+    readFloat32(offset: number): Promise<number>;
+    readFloat64(offset: number): Promise<number>;
+    readBool(offset: number): Promise<boolean>;
+    readString(offset: number): Promise<{
+        string: string;
+        newOffset: number;
+    }>;
+    protected _addToBuffer(buffer: Buffer): void;
+    static castNumber(value: bigint): number | bigint;
+}

package/dist/gguf/ggufParser/stream/GGUFBaseStream.js

@@ -0,0 +1,83 @@
+import { Buffer } from "buffer";
+export const METHOD_TO_BYTE_COUNT = {
+    readUint8: 1,
+    readUint16: 2,
+    readUint32: 4,
+    readUint64: 8,
+    readInt8: 1,
+    readInt16: 2,
+    readInt32: 4,
+    readInt64: 8,
+    readFloat32: 4,
+    readFloat64: 8,
+    readBool: 1
+};
+export const ALLOCATION_SIZE = 1024 * 1024 * 1.5; // 1.5MB
+export default class GGUFBaseStream {
+    _buffer = Buffer.alloc(0);
+    constructor() {
+    }
+    async readUint8(offset) {
+        const response = await this.readNBytes(METHOD_TO_BYTE_COUNT.readUint8, offset);
+        return response.readUInt8();
+    }
+    async readUint16(offset) {
+        const response = await this.readNBytes(METHOD_TO_BYTE_COUNT.readUint16, offset);
+        return response.readUInt16LE();
+    }
+    async readUint32(offset) {
+        const response = await this.readNBytes(METHOD_TO_BYTE_COUNT.readUint32, offset);
+        return response.readUInt32LE();
+    }
+    async readUint64(offset) {
+        const response = await this.readNBytes(METHOD_TO_BYTE_COUNT.readUint64, offset);
+        return response.readBigUInt64LE();
+    }
+    async readInt8(offset) {
+        const response = await this.readNBytes(METHOD_TO_BYTE_COUNT.readInt8, offset);
+        return response.readInt8();
+    }
+    async readInt16(offset) {
+        const response = await this.readNBytes(METHOD_TO_BYTE_COUNT.readInt16, offset);
+        return response.readInt16LE();
+    }
+    async readInt32(offset) {
+        const response = await this.readNBytes(METHOD_TO_BYTE_COUNT.readInt32, offset);
+        return response.readInt32LE();
+    }
+    async readInt64(offset) {
+        const response = await this.readNBytes(METHOD_TO_BYTE_COUNT.readInt64, offset);
+        return response.readBigInt64LE();
+    }
+    async readFloat32(offset) {
+        const response = await this.readNBytes(METHOD_TO_BYTE_COUNT.readFloat32, offset);
+        return response.readFloatLE();
+    }
+    async readFloat64(offset) {
+        const response = await this.readNBytes(METHOD_TO_BYTE_COUNT.readFloat64, offset);
+        return response.readDoubleLE();
+    }
+    async readBool(offset) {
+        const response = await this.readNBytes(METHOD_TO_BYTE_COUNT.readUint8, offset);
+        return response.readUInt8() === 1;
+    }
+    async readString(offset) {
+        const length = Number(await this.readUint64(offset));
+        offset += METHOD_TO_BYTE_COUNT.readUint64;
+        const readLength = METHOD_TO_BYTE_COUNT.readUint8 * length;
+        const stringBytes = await this.readNBytes(readLength, offset);
+        return {
+            string: String.fromCharCode(...stringBytes),
+            newOffset: offset + readLength
+        };
+    }
+    _addToBuffer(buffer) {
+        this._buffer = Buffer.concat([this._buffer, buffer]);
+    }
+    static castNumber(value) {
+        if (value > Number.MAX_SAFE_INTEGER)
+            return value;
+        return Number(value);
+    }
+}
+//# sourceMappingURL=GGUFBaseStream.js.map

package/dist/gguf/ggufParser/stream/GGUFBaseStream.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GGUFBaseStream.js","sourceRoot":"","sources":["../../../../src/gguf/ggufParser/stream/GGUFBaseStream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAC,MAAM,QAAQ,CAAC;AAE9B,MAAM,CAAC,MAAM,oBAAoB,GAAG;IAChC,SAAS,EAAE,CAAC;IACZ,UAAU,EAAE,CAAC;IACb,UAAU,EAAE,CAAC;IACb,UAAU,EAAE,CAAC;IACb,QAAQ,EAAE,CAAC;IACX,SAAS,EAAE,CAAC;IACZ,SAAS,EAAE,CAAC;IACZ,SAAS,EAAE,CAAC;IACZ,WAAW,EAAE,CAAC;IACd,WAAW,EAAE,CAAC;IACd,QAAQ,EAAE,CAAC;CACd,CAAC;AAEF,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,GAAG,IAAI,GAAG,GAAG,CAAC,CAAC,QAAQ;AAE1D,MAAM,CAAC,OAAO,OAAgB,cAAc;IAC9B,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACpC;IACA,CAAC;IAIM,KAAK,CAAC,SAAS,CAAC,MAAc;QACjC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,oBAAoB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC/E,OAAO,QAAQ,CAAC,SAAS,EAAE,CAAC;IAChC,CAAC;IAEM,KAAK,CAAC,UAAU,CAAC,MAAc;QAClC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,oBAAoB,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAChF,OAAO,QAAQ,CAAC,YAAY,EAAE,CAAC;IACnC,CAAC;IAEM,KAAK,CAAC,UAAU,CAAC,MAAc;QAClC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,oBAAoB,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAChF,OAAO,QAAQ,CAAC,YAAY,EAAE,CAAC;IACnC,CAAC;IAEM,KAAK,CAAC,UAAU,CAAC,MAAc;QAClC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,oBAAoB,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAChF,OAAO,QAAQ,CAAC,eAAe,EAAE,CAAC;IACtC,CAAC;IAEM,KAAK,CAAC,QAAQ,CAAC,MAAc;QAChC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,oBAAoB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC9E,OAAO,QAAQ,CAAC,QAAQ,EAAE,CAAC;IAC/B,CAAC;IAEM,KAAK,CAAC,SAAS,CAAC,MAAc;QACjC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,oBAAoB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC/E,OAAO,QAAQ,CAAC,WAAW,EAAE,CAAC;IAClC,CAAC;IAEM,KAAK,CAAC,SAAS,CAAC,MAAc;QACjC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,oBAAoB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC/E,OAAO,QAAQ,CAAC,WAAW,EAAE,CAAC;IAClC,CAAC;IAEM,KAAK,CAAC,SAAS,CAAC,MAAc;QACjC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,oBAAoB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC/E,OAAO,QAAQ,CAAC,cAAc,EAAE,CAAC;IACrC,CAAC;IAEM,KAAK,CAAC,WAAW,CAAC,MAAc;QACnC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,oBAAoB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QACjF,OAAO,QAAQ,CAAC,WAAW,EAAE,CAAC;IAClC,CAAC;IAEM,KAAK,CAAC,WAAW,CAAC,MAAc;QACnC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,oBAAoB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QACjF,OAAO,QAAQ,CAAC,YAAY,EAAE,CAAC;IACnC,CAAC;IAEM,KAAK,CAAC,QAAQ,CAAC,MAAc;QAChC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,oBAAoB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC/E,OAAO,QAAQ,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IACtC,CAAC;IAEM,KAAK,CAAC,UAAU,CAAC,MAAc;QAClC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;QACrD,MAAM,IAAI,oBAAoB,CAAC,UAAU,CAAC;QAE1C,MAAM,UAAU,GAAG,oBAAoB,CAAC,SAAS,GAAG,MAAM,CAAC;QAC3D,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAE9D,OAAO;YACH,MAAM,EAAE,MAAM,CAAC,YAAY,CAAC,GAAG,WAAW,CAAC;YAC3C,SAAS,EAAE,MAAM,GAAG,UAAU;SACjC,CAAC;IACN,CAAC;IAES,YAAY,CAAC,MAAc;QACjC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;IACzD,CAAC;IAEM,MAAM,CAAC,UAAU,CAAC,KAAa;QAClC,IAAI,KAAK,GAAG,MAAM,CAAC,gBAAgB;YAAE,OAAO,KAAK,CAAC;QAClD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACzB,CAAC;CACJ"}

package/dist/gguf/ggufParser/stream/GGUFFetchStream.d.ts

@@ -0,0 +1,14 @@
+/// <reference types="node" />
+import retry from "async-retry";
+import GgufBaseStream from "./GGUFBaseStream.js";
+type GGUFFetchStreamOptions = {
+    retry: retry.Options;
+};
+export default class GGUFFetchStream extends GgufBaseStream {
+    readonly url: string;
+    readonly options: Partial<GGUFFetchStreamOptions>;
+    constructor(url: string, options?: Partial<GGUFFetchStreamOptions>);
+    readNBytes(numBytes: number, offset?: number): Promise<Buffer>;
+    private _fetchBytesWithoutRetry;
+}
+export {};

package/dist/gguf/ggufParser/stream/GGUFFetchStream.js

@@ -0,0 +1,35 @@
+import retry from "async-retry";
+import { withLock } from "lifecycle-utils";
+import GgufBaseStream, { ALLOCATION_SIZE } from "./GGUFBaseStream.js";
+export default class GGUFFetchStream extends GgufBaseStream {
+    url;
+    options = {};
+    constructor(url, options = {}) {
+        super();
+        this.options = options;
+        this.url = url;
+    }
+    async readNBytes(numBytes, offset = 0) {
+        return await withLock(this, "_lock", async function readNBytesWithoutLock() {
+            if (offset + numBytes < this._buffer.length) {
+                return this._buffer.subarray(offset, offset + numBytes);
+            }
+            const fetchMissingBytes = await retry(async () => {
+                return await this._fetchBytesWithoutRetry(this._buffer.length, offset + numBytes + ALLOCATION_SIZE);
+            }, this.options.retry);
+            this._addToBuffer(fetchMissingBytes);
+            return await readNBytesWithoutLock.call(this);
+        });
+    }
+    async _fetchBytesWithoutRetry(start, end) {
+        const response = await fetch(this.url, {
+            headers: {
+                Range: `bytes=${start}-${end}`,
+                accept: "*/*"
+            }
+        });
+        const arrayBuffer = await response.arrayBuffer();
+        return Buffer.from(arrayBuffer);
+    }
+}
+//# sourceMappingURL=GGUFFetchStream.js.map