node-llama-cpp 3.0.0-beta.2 → 3.0.0-beta.21

package/dist/evaluator/TokenBias.js

@@ -0,0 +1,33 @@
+import { tokenizeInput } from "../utils/tokenizeInput.js";
+export class TokenBias {
+    /** @internal */ _model;
+    /** @internal */ _biases = new Map();
+    constructor(model) {
+        this._model = model;
+    }
+    /**
+     * Adjust the bias of the given token(s).
+     * If a text is provided, the bias will be applied to each individual token in the text.
+     * Setting a bias to `"never"` will prevent the token from being generated, unless it is required to comply with a grammar.
+     * Setting the bias of the EOS or EOT tokens to `"never"` has no effect and will be ignored.
+     * @param input - The token(s) to apply the bias to
+     * @param bias - The bias to apply to the token(s).
+     * Setting to a positive number increases the probability of the token(s) being generated.
+     * Setting to a negative number decreases the probability of the token(s) being generated.
+     * Setting to `0` has no effect.
+     * Setting to `"never"` will prevent the token from being generated, unless it is required to comply with a grammar.
+     * Try to play around with values between `10` and `-10` to see what works for your use case.
+     * Fractional values are allowed and can be used to fine-tune the bias (for example, `1.123`).
+     */
+    set(input, bias) {
+        for (const token of tokenizeInput(input, this._model.tokenizer))
+            this._biases.set(token, bias === "never" ? -Infinity : bias);
+        for (const token of tokenizeInput(input, this._model.tokenizer, "trimLeadingSpace"))
+            this._biases.set(token, bias === "never" ? -Infinity : bias);
+        return this;
+    }
+    static for(model) {
+        return new TokenBias(model);
+    }
+}
+//# sourceMappingURL=TokenBias.js.map

package/dist/evaluator/TokenBias.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TokenBias.js","sourceRoot":"","sources":["../../src/evaluator/TokenBias.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,aAAa,EAAC,MAAM,2BAA2B,CAAC;AAGxD,MAAM,OAAO,SAAS;IAClB,gBAAgB,CAAiB,MAAM,CAAa;IACpD,gBAAgB,CAAiB,OAAO,GAAG,IAAI,GAAG,EAAiB,CAAC;IAEpE,YAAmB,KAAiB;QAChC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACxB,CAAC;IAED;;;;;;;;;;;;;OAaG;IACI,GAAG,CAAC,KAA2C,EAAE,IAAsB;QAC1E,KAAK,MAAM,KAAK,IAAI,aAAa,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;YAC3D,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAEjE,KAAK,MAAM,KAAK,IAAI,aAAa,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,kBAAkB,CAAC;YAC/E,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAEjE,OAAO,IAAI,CAAC;IAChB,CAAC;IAEM,MAAM,CAAC,GAAG,CAAC,KAAiB;QAC/B,OAAO,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;CACJ"}

package/dist/evaluator/TokenMeter.d.ts

@@ -0,0 +1,54 @@
+/**
+ * Tracks the evaluation usage of tokens.
+ */
+export declare class TokenMeter {
+    private _inputTokens;
+    private _outputTokens;
+    private _restoreStateTokens;
+    /**
+     * The number of input tokens used
+     */
+    get usedInputTokens(): number;
+    /**
+     * The number of tokens generated by a model
+     */
+    get usedOutputTokens(): number;
+    /**
+     * The number of tokens used as input to restore a context sequence state to continue previous evaluation.
+     * This may be consumed by virtual context sequences.
+     */
+    get usedRestoreStateTokens(): number;
+    /**
+     * Get the current state of the token meter
+     */
+    getState(): TokenMeterState;
+    /**
+     * Log the usage of tokens
+     */
+    useTokens(tokens: number, type: "input" | "output" | "restoreState"): void;
+    /**
+     * Get the difference between the current meter and another meter
+     */
+    diff(meter: TokenMeter | TokenMeterState): {
+        usedInputTokens: number;
+        usedOutputTokens: number;
+        usedRestoreStateTokens: number;
+    };
+    /**
+     * Log the usage of tokens on multiple meters
+     */
+    static useTokens(meters: null | undefined | TokenMeter | readonly TokenMeter[] | ReadonlySet<TokenMeter>, tokens: number, type: "input" | "output" | "restoreState"): void;
+    /**
+     * Get the difference between two meters
+     */
+    static diff(meter1: TokenMeter | TokenMeterState, meter2: TokenMeter | TokenMeterState): {
+        usedInputTokens: number;
+        usedOutputTokens: number;
+        usedRestoreStateTokens: number;
+    };
+}
+export type TokenMeterState = {
+    usedInputTokens: number;
+    usedOutputTokens: number;
+    usedRestoreStateTokens: number;
+};

package/dist/evaluator/TokenMeter.js

@@ -0,0 +1,86 @@
+/**
+ * Tracks the evaluation usage of tokens.
+ */
+export class TokenMeter {
+    _inputTokens = 0;
+    _outputTokens = 0;
+    _restoreStateTokens = 0;
+    /**
+     * The number of input tokens used
+     */
+    get usedInputTokens() {
+        return this._inputTokens;
+    }
+    /**
+     * The number of tokens generated by a model
+     */
+    get usedOutputTokens() {
+        return this._outputTokens;
+    }
+    /**
+     * The number of tokens used as input to restore a context sequence state to continue previous evaluation.
+     * This may be consumed by virtual context sequences.
+     */
+    get usedRestoreStateTokens() {
+        return this._restoreStateTokens;
+    }
+    /**
+     * Get the current state of the token meter
+     */
+    getState() {
+        return {
+            usedInputTokens: this.usedInputTokens,
+            usedOutputTokens: this.usedOutputTokens,
+            usedRestoreStateTokens: this.usedRestoreStateTokens
+        };
+    }
+    /**
+     * Log the usage of tokens
+     */
+    useTokens(tokens, type) {
+        if (tokens < 0)
+            throw new RangeError("Tokens cannot be negative");
+        else if (tokens === 0)
+            return;
+        if (type === "input")
+            this._inputTokens += tokens;
+        else if (type === "output")
+            this._outputTokens += tokens;
+        else if (type === "restoreState")
+            this._restoreStateTokens += tokens;
+        else {
+            void (type);
+            throw new TypeError(`Unknown token type: ${type}`);
+        }
+    }
+    /**
+     * Get the difference between the current meter and another meter
+     */
+    diff(meter) {
+        return TokenMeter.diff(this, meter);
+    }
+    /**
+     * Log the usage of tokens on multiple meters
+     */
+    static useTokens(meters, tokens, type) {
+        if (meters == null)
+            return;
+        if (meters instanceof TokenMeter)
+            meters.useTokens(tokens, type);
+        else {
+            for (const meter of meters)
+                meter.useTokens(tokens, type);
+        }
+    }
+    /**
+     * Get the difference between two meters
+     */
+    static diff(meter1, meter2) {
+        return {
+            usedInputTokens: meter1.usedInputTokens - meter2.usedInputTokens,
+            usedOutputTokens: meter1.usedOutputTokens - meter2.usedOutputTokens,
+            usedRestoreStateTokens: meter1.usedRestoreStateTokens - meter2.usedRestoreStateTokens
+        };
+    }
+}
+//# sourceMappingURL=TokenMeter.js.map

package/dist/evaluator/TokenMeter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TokenMeter.js","sourceRoot":"","sources":["../../src/evaluator/TokenMeter.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,OAAO,UAAU;IACX,YAAY,GAAW,CAAC,CAAC;IACzB,aAAa,GAAW,CAAC,CAAC;IAC1B,mBAAmB,GAAW,CAAC,CAAC;IAExC;;OAEG;IACH,IAAW,eAAe;QACtB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,IAAW,gBAAgB;QACvB,OAAO,IAAI,CAAC,aAAa,CAAC;IAC9B,CAAC;IAED;;;OAGG;IACH,IAAW,sBAAsB;QAC7B,OAAO,IAAI,CAAC,mBAAmB,CAAC;IACpC,CAAC;IAED;;OAEG;IACI,QAAQ;QACX,OAAO;YACH,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,sBAAsB,EAAE,IAAI,CAAC,sBAAsB;SACtD,CAAC;IACN,CAAC;IAED;;OAEG;IACI,SAAS,CAAC,MAAc,EAAE,IAAyC;QACtE,IAAI,MAAM,GAAG,CAAC;YACV,MAAM,IAAI,UAAU,CAAC,2BAA2B,CAAC,CAAC;aACjD,IAAI,MAAM,KAAK,CAAC;YACjB,OAAO;QAEX,IAAI,IAAI,KAAK,OAAO;YAChB,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC;aAC3B,IAAI,IAAI,KAAK,QAAQ;YACtB,IAAI,CAAC,aAAa,IAAI,MAAM,CAAC;aAC5B,IAAI,IAAI,KAAK,cAAc;YAC5B,IAAI,CAAC,mBAAmB,IAAI,MAAM,CAAC;aAClC;YACD,KAAK,CAAC,IAAoB,CAAC,CAAC;YAC5B,MAAM,IAAI,SAAS,CAAC,uBAAuB,IAAI,EAAE,CAAC,CAAC;SACtD;IACL,CAAC;IAED;;OAEG;IACI,IAAI,CAAC,KAAmC;QAC3C,OAAO,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACxC,CAAC;IAED;;OAEG;IACI,MAAM,CAAC,SAAS,CACnB,MAAuF,EACvF,MAAc,EACd,IAAyC;QAEzC,IAAI,MAAM,IAAI,IAAI;YACd,OAAO;QAEX,IAAI,MAAM,YAAY,UAAU;YAC5B,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;aAC9B;YACD,KAAK,MAAM,KAAK,IAAI,MAAM;gBACtB,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;SACrC;IACL,CAAC;IAED;;OAEG;IACI,MAAM,CAAC,IAAI,CACd,MAAoC,EACpC,MAAoC;QAEpC,OAAO;YACH,eAAe,EAAE,MAAM,CAAC,eAAe,GAAG,MAAM,CAAC,eAAe;YAChE,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB;YACnE,sBAAsB,EAAE,MAAM,CAAC,sBAAsB,GAAG,MAAM,CAAC,sBAAsB;SACxF,CAAC;IACN,CAAC;CACJ"}

package/dist/gguf/consts.d.ts

@@ -0,0 +1,3 @@
+import retry from "async-retry";
+export declare const ggufDefaultFetchRetryOptions: retry.Options;
+export declare const defaultExtraAllocationSize: number;

package/dist/gguf/consts.js

@@ -0,0 +1,8 @@
+export const ggufDefaultFetchRetryOptions = {
+    retries: 10,
+    factor: 2,
+    minTimeout: 1000,
+    maxTimeout: 1000 * 16
+};
+export const defaultExtraAllocationSize = 1024 * 1024 * 1.5; // 1.5MB
+//# sourceMappingURL=consts.js.map

package/dist/gguf/consts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"consts.js","sourceRoot":"","sources":["../../src/gguf/consts.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,4BAA4B,GAAkB;IACvD,OAAO,EAAE,EAAE;IACX,MAAM,EAAE,CAAC;IACT,UAAU,EAAE,IAAI;IAChB,UAAU,EAAE,IAAI,GAAG,EAAE;CACf,CAAC;AAEX,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,GAAG,IAAI,GAAG,GAAG,CAAC,CAAC,QAAQ"}

package/dist/gguf/errors/InvalidGgufMagicError.d.ts

@@ -0,0 +1,3 @@
+export declare class InvalidGgufMagicError extends Error {
+    constructor(expectedGgufMagic: string, actualGgufMagic: string);
+}

package/dist/gguf/errors/InvalidGgufMagicError.js

@@ -0,0 +1,6 @@
+export class InvalidGgufMagicError extends Error {
+    constructor(expectedGgufMagic, actualGgufMagic) {
+        super(`Invalid GGUF magic. Expected "${expectedGgufMagic}" but got "${actualGgufMagic}".`);
+    }
+}
+//# sourceMappingURL=InvalidGgufMagicError.js.map

package/dist/gguf/errors/InvalidGgufMagicError.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"InvalidGgufMagicError.js","sourceRoot":"","sources":["../../../src/gguf/errors/InvalidGgufMagicError.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,qBAAsB,SAAQ,KAAK;IAC5C,YAAmB,iBAAyB,EAAE,eAAuB;QACjE,KAAK,CAAC,iCAAiC,iBAAiB,cAAc,eAAe,IAAI,CAAC,CAAC;IAC/F,CAAC;CACJ"}

package/dist/gguf/errors/UnsupportedGgufValueTypeError.d.ts

@@ -0,0 +1,4 @@
+export declare class UnsupportedGgufValueTypeError extends Error {
+    readonly ggufValueType: number;
+    constructor(ggufValueType: number);
+}

package/dist/gguf/errors/UnsupportedGgufValueTypeError.js

@@ -0,0 +1,9 @@
+export class UnsupportedGgufValueTypeError extends Error {
+    ggufValueType;
+    constructor(ggufValueType) {
+        super(`Unsupported GGUF value type "${ggufValueType}"`);
+        Object.defineProperty(this, "ggufValueType", { enumerable: false });
+        this.ggufValueType = ggufValueType;
+    }
+}
+//# sourceMappingURL=UnsupportedGgufValueTypeError.js.map

package/dist/gguf/errors/UnsupportedGgufValueTypeError.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"UnsupportedGgufValueTypeError.js","sourceRoot":"","sources":["../../../src/gguf/errors/UnsupportedGgufValueTypeError.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,6BAA8B,SAAQ,KAAK;IACpC,aAAa,CAAS;IAEtC,YAAmB,aAAqB;QACpC,KAAK,CAAC,gCAAgC,aAAa,GAAG,CAAC,CAAC;QAExD,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,eAAoC,EAAE,EAAC,UAAU,EAAE,KAAK,EAAC,CAAC,CAAC;QAEvF,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACvC,CAAC;CACJ"}

package/dist/gguf/fileReaders/GgufFileReader.d.ts

@@ -0,0 +1,33 @@
+/// <reference types="node" />
+import { GgufReadOffset } from "../utils/GgufReadOffset.js";
+export declare const valueTypeToBytesToRead: {
+    readonly uint8: 1;
+    readonly uint16: 2;
+    readonly uint32: 4;
+    readonly uint64: 8;
+    readonly int8: 1;
+    readonly int16: 2;
+    readonly int32: 4;
+    readonly int64: 8;
+    readonly float32: 4;
+    readonly float64: 8;
+    readonly bool: 1;
+};
+export declare abstract class GgufFileReader {
+    protected _buffer: Buffer;
+    abstract readByteRange(offset: number | GgufReadOffset, length: number): Promise<Buffer>;
+    readUint8(offset: number | GgufReadOffset): Promise<number>;
+    readUint16(offset: number | GgufReadOffset): Promise<number>;
+    readUint32(offset: number | GgufReadOffset): Promise<number>;
+    readUint64(offset: number | GgufReadOffset): Promise<bigint>;
+    readInt8(offset: number | GgufReadOffset): Promise<number>;
+    readInt16(offset: number | GgufReadOffset): Promise<number>;
+    readInt32(offset: number | GgufReadOffset): Promise<number>;
+    readInt64(offset: number | GgufReadOffset): Promise<bigint>;
+    readFloat32(offset: number | GgufReadOffset): Promise<number>;
+    readFloat64(offset: number | GgufReadOffset): Promise<number>;
+    readBool(offset: number | GgufReadOffset): Promise<boolean>;
+    protected _addToBuffer(buffer: Buffer): void;
+    private _readByteRangeAndUpdateOffset;
+    static castNumberIfSafe(value: bigint): number | bigint;
+}

package/dist/gguf/fileReaders/GgufFileReader.js

@@ -0,0 +1,76 @@
+import { GgufReadOffset } from "../utils/GgufReadOffset.js";
+export const valueTypeToBytesToRead = {
+    uint8: 1,
+    uint16: 2,
+    uint32: 4,
+    uint64: 8,
+    int8: 1,
+    int16: 2,
+    int32: 4,
+    int64: 8,
+    float32: 4,
+    float64: 8,
+    bool: 1
+};
+export class GgufFileReader {
+    _buffer = Buffer.alloc(0);
+    async readUint8(offset) {
+        const response = await this._readByteRangeAndUpdateOffset(offset, valueTypeToBytesToRead.uint8);
+        return response.readUInt8();
+    }
+    async readUint16(offset) {
+        const response = await this._readByteRangeAndUpdateOffset(offset, valueTypeToBytesToRead.uint16);
+        return response.readUInt16LE();
+    }
+    async readUint32(offset) {
+        const response = await this._readByteRangeAndUpdateOffset(offset, valueTypeToBytesToRead.uint32);
+        return response.readUInt32LE();
+    }
+    async readUint64(offset) {
+        const response = await this._readByteRangeAndUpdateOffset(offset, valueTypeToBytesToRead.uint64);
+        return response.readBigUInt64LE();
+    }
+    async readInt8(offset) {
+        const response = await this._readByteRangeAndUpdateOffset(offset, valueTypeToBytesToRead.int8);
+        return response.readInt8();
+    }
+    async readInt16(offset) {
+        const response = await this._readByteRangeAndUpdateOffset(offset, valueTypeToBytesToRead.int16);
+        return response.readInt16LE();
+    }
+    async readInt32(offset) {
+        const response = await this._readByteRangeAndUpdateOffset(offset, valueTypeToBytesToRead.int32);
+        return response.readInt32LE();
+    }
+    async readInt64(offset) {
+        const response = await this._readByteRangeAndUpdateOffset(offset, valueTypeToBytesToRead.int64);
+        return response.readBigInt64LE();
+    }
+    async readFloat32(offset) {
+        const response = await this._readByteRangeAndUpdateOffset(offset, valueTypeToBytesToRead.float32);
+        return response.readFloatLE();
+    }
+    async readFloat64(offset) {
+        const response = await this._readByteRangeAndUpdateOffset(offset, valueTypeToBytesToRead.float64);
+        return response.readDoubleLE();
+    }
+    async readBool(offset) {
+        const response = await this._readByteRangeAndUpdateOffset(offset, valueTypeToBytesToRead.uint8);
+        return response.readUInt8() === 1;
+    }
+    _addToBuffer(buffer) {
+        this._buffer = Buffer.concat([this._buffer, buffer]);
+    }
+    async _readByteRangeAndUpdateOffset(offset, length) {
+        const readOffset = GgufReadOffset.resolveReadOffset(offset);
+        const response = await this.readByteRange(readOffset.offset, length);
+        readOffset.moveBy(length);
+        return response;
+    }
+    static castNumberIfSafe(value) {
+        if (value > Number.MAX_SAFE_INTEGER)
+            return value;
+        return Number(value);
+    }
+}
+//# sourceMappingURL=GgufFileReader.js.map

package/dist/gguf/fileReaders/GgufFileReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GgufFileReader.js","sourceRoot":"","sources":["../../../src/gguf/fileReaders/GgufFileReader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,cAAc,EAAC,MAAM,4BAA4B,CAAC;AAE1D,MAAM,CAAC,MAAM,sBAAsB,GAAG;IAClC,KAAK,EAAE,CAAC;IACR,MAAM,EAAE,CAAC;IACT,MAAM,EAAE,CAAC;IACT,MAAM,EAAE,CAAC;IACT,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC;IACR,KAAK,EAAE,CAAC;IACR,KAAK,EAAE,CAAC;IACR,OAAO,EAAE,CAAC;IACV,OAAO,EAAE,CAAC;IACV,IAAI,EAAE,CAAC;CACD,CAAC;AAEX,MAAM,OAAgB,cAAc;IACtB,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAI7B,KAAK,CAAC,SAAS,CAAC,MAA+B;QAClD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,MAAM,EAAE,sBAAsB,CAAC,KAAK,CAAC,CAAC;QAChG,OAAO,QAAQ,CAAC,SAAS,EAAE,CAAC;IAChC,CAAC;IAEM,KAAK,CAAC,UAAU,CAAC,MAA+B;QACnD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,MAAM,EAAE,sBAAsB,CAAC,MAAM,CAAC,CAAC;QACjG,OAAO,QAAQ,CAAC,YAAY,EAAE,CAAC;IACnC,CAAC;IAEM,KAAK,CAAC,UAAU,CAAC,MAA+B;QACnD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,MAAM,EAAE,sBAAsB,CAAC,MAAM,CAAC,CAAC;QACjG,OAAO,QAAQ,CAAC,YAAY,EAAE,CAAC;IACnC,CAAC;IAEM,KAAK,CAAC,UAAU,CAAC,MAA+B;QACnD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,MAAM,EAAE,sBAAsB,CAAC,MAAM,CAAC,CAAC;QACjG,OAAO,QAAQ,CAAC,eAAe,EAAE,CAAC;IACtC,CAAC;IAEM,KAAK,CAAC,QAAQ,CAAC,MAA+B;QACjD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,MAAM,EAAE,sBAAsB,CAAC,IAAI,CAAC,CAAC;QAC/F,OAAO,QAAQ,CAAC,QAAQ,EAAE,CAAC;IAC/B,CAAC;IAEM,KAAK,CAAC,SAAS,CAAC,MAA+B;QAClD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,MAAM,EAAE,sBAAsB,CAAC,KAAK,CAAC,CAAC;QAChG,OAAO,QAAQ,CAAC,WAAW,EAAE,CAAC;IAClC,CAAC;IAEM,KAAK,CAAC,SAAS,CAAC,MAA+B;QAClD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,MAAM,EAAE,sBAAsB,CAAC,KAAK,CAAC,CAAC;QAChG,OAAO,QAAQ,CAAC,WAAW,EAAE,CAAC;IAClC,CAAC;IAEM,KAAK,CAAC,SAAS,CAAC,MAA+B;QAClD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,MAAM,EAAE,sBAAsB,CAAC,KAAK,CAAC,CAAC;QAChG,OAAO,QAAQ,CAAC,cAAc,EAAE,CAAC;IACrC,CAAC;IAEM,KAAK,CAAC,WAAW,CAAC,MAA+B;QACpD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,MAAM,EAAE,sBAAsB,CAAC,OAAO,CAAC,CAAC;QAClG,OAAO,QAAQ,CAAC,WAAW,EAAE,CAAC;IAClC,CAAC;IAEM,KAAK,CAAC,WAAW,CAAC,MAA+B;QACpD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,MAAM,EAAE,sBAAsB,CAAC,OAAO,CAAC,CAAC;QAClG,OAAO,QAAQ,CAAC,YAAY,EAAE,CAAC;IACnC,CAAC;IAEM,KAAK,CAAC,QAAQ,CAAC,MAA+B;QACjD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,MAAM,EAAE,sBAAsB,CAAC,KAAK,CAAC,CAAC;QAChG,OAAO,QAAQ,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IACtC,CAAC;IAES,YAAY,CAAC,MAAc;QACjC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;IACzD,CAAC;IAEO,KAAK,CAAC,6BAA6B,CAAC,MAA+B,EAAE,MAAc;QACvF,MAAM,UAAU,GAAG,cAAc,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAE5D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACrE,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAE1B,OAAO,QAAQ,CAAC;IACpB,CAAC;IAEM,MAAM,CAAC,gBAAgB,CAAC,KAAa;QACxC,IAAI,KAAK,GAAG,MAAM,CAAC,gBAAgB;YAC/B,OAAO,KAAK,CAAC;QAEjB,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACzB,CAAC;CACJ"}

package/dist/gguf/fileReaders/GgufFsFileReader.d.ts

@@ -0,0 +1,17 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import { GgufReadOffset } from "../utils/GgufReadOffset.js";
+import { GgufFileReader } from "./GgufFileReader.js";
+type GgufFsFileReaderOptions = {
+    filePath: string;
+    signal?: AbortSignal;
+};
+export declare class GgufFsFileReader extends GgufFileReader {
+    readonly filePath: string;
+    private readonly _signal?;
+    constructor({ filePath, signal }: GgufFsFileReaderOptions);
+    readByteRange(offset: number | GgufReadOffset, length: number): Promise<Buffer>;
+    private _readToExpandBufferUpToOffset;
+    private _readByteRange;
+}
+export {};

package/dist/gguf/fileReaders/GgufFsFileReader.js

@@ -0,0 +1,45 @@
+import fs from "node:fs/promises";
+import { withLock } from "lifecycle-utils";
+import { GgufReadOffset } from "../utils/GgufReadOffset.js";
+import { defaultExtraAllocationSize } from "../consts.js";
+import { GgufFileReader } from "./GgufFileReader.js";
+export class GgufFsFileReader extends GgufFileReader {
+    filePath;
+    _signal;
+    constructor({ filePath, signal }) {
+        super();
+        this.filePath = filePath;
+        this._signal = signal;
+    }
+    async readByteRange(offset, length) {
+        const readOffset = GgufReadOffset.resolveReadOffset(offset);
+        const endOffset = readOffset.offset + length;
+        if (endOffset >= this._buffer.length)
+            await this._readToExpandBufferUpToOffset(endOffset);
+        const res = this._buffer.subarray(readOffset.offset, endOffset);
+        readOffset.moveBy(length);
+        return res;
+    }
+    async _readToExpandBufferUpToOffset(endOffset, extraAllocationSize = defaultExtraAllocationSize) {
+        return await withLock(this, "modifyBuffer", this._signal, async () => {
+            if (endOffset < this._buffer.length)
+                return;
+            const missingBytesBuffer = await this._readByteRange(this._buffer.length, endOffset + extraAllocationSize - this._buffer.length);
+            this._addToBuffer(missingBytesBuffer);
+        });
+    }
+    async _readByteRange(start, length) {
+        const fd = await fs.open(this.filePath, "r");
+        try {
+            if (this._signal?.aborted)
+                throw this._signal.reason;
+            const buffer = Buffer.alloc(length);
+            await fd.read(buffer, 0, length, start);
+            return buffer;
+        }
+        finally {
+            await fd.close();
+        }
+    }
+}
+//# sourceMappingURL=GgufFsFileReader.js.map

package/dist/gguf/fileReaders/GgufFsFileReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GgufFsFileReader.js","sourceRoot":"","sources":["../../../src/gguf/fileReaders/GgufFsFileReader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,EAAC,QAAQ,EAAC,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAC,cAAc,EAAC,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAC,0BAA0B,EAAC,MAAM,cAAc,CAAC;AACxD,OAAO,EAAC,cAAc,EAAC,MAAM,qBAAqB,CAAC;AAOnD,MAAM,OAAO,gBAAiB,SAAQ,cAAc;IAChC,QAAQ,CAAS;IAChB,OAAO,CAAe;IAEvC,YAAmB,EAAC,QAAQ,EAAE,MAAM,EAA0B;QAC1D,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IAC1B,CAAC;IAEM,KAAK,CAAC,aAAa,CAAC,MAA+B,EAAE,MAAc;QACtE,MAAM,UAAU,GAAG,cAAc,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC;QAE7C,IAAI,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM;YAChC,MAAM,IAAI,CAAC,6BAA6B,CAAC,SAAS,CAAC,CAAC;QAExD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAChE,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC1B,OAAO,GAAG,CAAC;IACf,CAAC;IAEO,KAAK,CAAC,6BAA6B,CAAC,SAAiB,EAAE,sBAA8B,0BAA0B;QACnH,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;YACjE,IAAI,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM;gBAC/B,OAAO;YAEX,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,cAAc,CAChD,IAAI,CAAC,OAAO,CAAC,MAAM,EACnB,SAAS,GAAG,mBAAmB,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CACxD,CAAC;YAEF,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,KAAa,EAAE,MAAc;QACtD,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAC7C,IAAI;YACA,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO;gBACrB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAE9B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACpC,MAAM,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;YACxC,OAAO,MAAM,CAAC;SACjB;gBAAS;YACN,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;SACpB;IACL,CAAC;CACJ"}

package/dist/gguf/fileReaders/GgufNetworkFetchFileReader.d.ts

@@ -0,0 +1,22 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import retry from "async-retry";
+import { GgufReadOffset } from "../utils/GgufReadOffset.js";
+import { GgufFileReader } from "./GgufFileReader.js";
+type GgufFetchFileReaderOptions = {
+    url: string;
+    retryOptions?: retry.Options;
+    headers?: Record<string, string>;
+    signal?: AbortSignal;
+};
+export declare class GgufNetworkFetchFileReader extends GgufFileReader {
+    readonly url: string;
+    readonly retryOptions: retry.Options;
+    readonly headers: Record<string, string>;
+    private readonly _signal?;
+    constructor({ url, retryOptions, headers, signal }: GgufFetchFileReaderOptions);
+    readByteRange(offset: number | GgufReadOffset, length: number): Promise<Buffer>;
+    private _fetchToExpandBufferUpToOffset;
+    private _fetchByteRange;
+}
+export {};

package/dist/gguf/fileReaders/GgufNetworkFetchFileReader.js

@@ -0,0 +1,63 @@
+import retry from "async-retry";
+import { withLock } from "lifecycle-utils";
+import { GgufReadOffset } from "../utils/GgufReadOffset.js";
+import { defaultExtraAllocationSize, ggufDefaultFetchRetryOptions } from "../consts.js";
+import { GgufFileReader } from "./GgufFileReader.js";
+export class GgufNetworkFetchFileReader extends GgufFileReader {
+    url;
+    retryOptions;
+    headers;
+    _signal;
+    constructor({ url, retryOptions = ggufDefaultFetchRetryOptions, headers, signal }) {
+        super();
+        this.url = url;
+        this.retryOptions = retryOptions;
+        this.headers = headers ?? {};
+        this._signal = signal;
+    }
+    async readByteRange(offset, length) {
+        const readOffset = GgufReadOffset.resolveReadOffset(offset);
+        const endOffset = readOffset.offset + length;
+        if (endOffset >= this._buffer.length)
+            await this._fetchToExpandBufferUpToOffset(endOffset);
+        const res = this._buffer.subarray(readOffset.offset, endOffset);
+        readOffset.moveBy(length);
+        return res;
+    }
+    async _fetchToExpandBufferUpToOffset(endOffset, extraAllocationSize = defaultExtraAllocationSize) {
+        await withLock(this, "modifyBuffer", this._signal, async () => {
+            if (endOffset < this._buffer.length)
+                return;
+            const missingBytesBuffer = await retry(async (bail) => {
+                try {
+                    return await this._fetchByteRange(this._buffer.length, endOffset + extraAllocationSize - this._buffer.length);
+                }
+                catch (err) {
+                    if (this._signal?.aborted) {
+                        bail(this._signal.reason);
+                        throw this._signal.reason;
+                    }
+                    throw err;
+                }
+            }, this.retryOptions);
+            if (this._signal?.aborted)
+                throw this._signal.reason;
+            this._addToBuffer(missingBytesBuffer);
+        });
+    }
+    async _fetchByteRange(start, length) {
+        const response = await fetch(this.url, {
+            headers: {
+                ...this.headers,
+                Range: `bytes=${start}-${start + length}`,
+                accept: "*/*"
+            },
+            signal: this._signal
+        });
+        if (!response.ok)
+            throw new Error(`Failed to fetch byte range: ${response.status} ${response.statusText}`);
+        const arrayBuffer = await response.arrayBuffer();
+        return Buffer.from(arrayBuffer);
+    }
+}
+//# sourceMappingURL=GgufNetworkFetchFileReader.js.map

package/dist/gguf/fileReaders/GgufNetworkFetchFileReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GgufNetworkFetchFileReader.js","sourceRoot":"","sources":["../../../src/gguf/fileReaders/GgufNetworkFetchFileReader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,aAAa,CAAC;AAChC,OAAO,EAAC,QAAQ,EAAC,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAC,cAAc,EAAC,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAC,0BAA0B,EAAE,4BAA4B,EAAC,MAAM,cAAc,CAAC;AACtF,OAAO,EAAC,cAAc,EAAC,MAAM,qBAAqB,CAAC;AASnD,MAAM,OAAO,0BAA2B,SAAQ,cAAc;IAC1C,GAAG,CAAS;IACZ,YAAY,CAAgB;IAC5B,OAAO,CAAyB;IAC/B,OAAO,CAAe;IAEvC,YAAmB,EAAC,GAAG,EAAE,YAAY,GAAG,4BAA4B,EAAE,OAAO,EAAE,MAAM,EAA6B;QAC9G,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,OAAO,GAAG,OAAO,IAAI,EAAE,CAAC;QAC7B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IAC1B,CAAC;IAEM,KAAK,CAAC,aAAa,CAAC,MAA+B,EAAE,MAAc;QACtE,MAAM,UAAU,GAAG,cAAc,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC;QAE7C,IAAI,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM;YAChC,MAAM,IAAI,CAAC,8BAA8B,CAAC,SAAS,CAAC,CAAC;QAEzD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAChE,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC1B,OAAO,GAAG,CAAC;IACf,CAAC;IAEO,KAAK,CAAC,8BAA8B,CAAC,SAAiB,EAAE,sBAA8B,0BAA0B;QACpH,MAAM,QAAQ,CAAC,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;YAC1D,IAAI,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM;gBAC/B,OAAO;YAEX,MAAM,kBAAkB,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;gBAClD,IAAI;oBACA,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,SAAS,GAAG,mBAAmB,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;iBACjH;gBAAC,OAAO,GAAG,EAAE;oBACV,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE;wBACvB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;wBAC1B,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;qBAC7B;oBAED,MAAM,GAAG,CAAC;iBACb;YACL,CAAC,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;YAEtB,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO;gBACrB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAE9B,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,KAAa,EAAE,MAAc;QACvD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE;YACnC,OAAO,EAAE;gBACL,GAAG,IAAI,CAAC,OAAO;gBACf,KAAK,EAAE,SAAS,KAAK,IAAI,KAAK,GAAG,MAAM,EAAE;gBACzC,MAAM,EAAE,KAAK;aAChB;YACD,MAAM,EAAE,IAAI,CAAC,OAAO;SACvB,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE;YACZ,MAAM,IAAI,KAAK,CAAC,+BAA+B,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QAE7F,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;QACjD,OAAO,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACpC,CAAC;CACJ"}

package/dist/gguf/insights/GgufInsights.d.ts

@@ -0,0 +1,42 @@
+import { Llama } from "../../bindings/Llama.js";
+import { GgufFileInfo } from "../types/GgufFileInfoTypes.js";
+import { GgufInsightsConfigurationResolver } from "./GgufInsightsConfigurationResolver.js";
+export type GgufInsightsResourceRequirements = {
+    cpuRam: number;
+    gpuVram: number;
+};
+export declare class GgufInsights {
+    private constructor();
+    get ggufFileInfo(): GgufFileInfo;
+    get configurationResolver(): GgufInsightsConfigurationResolver;
+    /** The context size the model was trained on */
+    get trainContextSize(): number | undefined;
+    /** The size of an embedding vector the model can produce */
+    get embeddingVectorSize(): number | undefined;
+    get totalLayers(): number;
+    get modelSize(): number;
+    estimateModelResourceRequirements({ gpuLayers }: {
+        gpuLayers: number;
+    }): GgufInsightsResourceRequirements;
+    /**
+     * Estimates the memory required to create a context of the given parameters based on the implementation details of `llama.cpp`.
+     * The calculation doesn't include a precise estimation of the graph overhead memory, so it uses a rough estimate for that.
+     * The estimation for the graph overhead memory will be improved in the future to be more precise, but it's good enough for now.
+     */
+    estimateContextResourceRequirements({ contextSize, modelGpuLayers, batchSize, sequences, isEmbeddingContext, includeGraphOverhead }: {
+        contextSize: number;
+        modelGpuLayers: number;
+        batchSize?: number;
+        sequences?: number;
+        isEmbeddingContext?: boolean;
+        includeGraphOverhead?: boolean;
+    }): GgufInsightsResourceRequirements;
+    /**
+     * @param ggufFileInfo
+     * @param llama - If you already have a `Llama` instance, pass it to reuse it for the `GgufInsights` instance.
+     * If you don't pass a `Llama` instance, a basic `Llama` instance is created as a fallback - it's a slim instance that
+     * doesn't instantiate a `llama.cpp` backend, so it won't utilize the GPU at all, and be shared with other `GgufInsights` instances
+     * that need a fallback `Llama` instance.
+     */
+    static from(ggufFileInfo: GgufFileInfo, llama?: Llama): Promise<GgufInsights>;
+}