node-llama-cpp 3.0.0-beta.2 → 3.0.0-beta.21

package/dist/evaluator/LlamaModel.d.ts

@@ -0,0 +1,230 @@
+/// <reference types="node" />
+import { EventRelay } from "lifecycle-utils";
+import { Token, Tokenizer } from "../types.js";
+import { ModelTypeDescription } from "../bindings/AddonTypes.js";
+import { LlamaVocabularyType } from "../bindings/types.js";
+import { GgufFileInfo } from "../gguf/types/GgufFileInfoTypes.js";
+import { GgufInsights } from "../gguf/insights/GgufInsights.js";
+import { GgufMetadataTokenizerTokenType } from "../gguf/types/GgufMetadataTypes.js";
+import { LlamaContextOptions } from "./LlamaContext/types.js";
+import { LlamaContext } from "./LlamaContext/LlamaContext.js";
+import { LlamaEmbeddingContext, LlamaEmbeddingContextOptions } from "./LlamaEmbeddingContext.js";
+import type { BuiltinSpecialTokenValue } from "../utils/LlamaText.js";
+export type LlamaModelOptions = {
+    /** path to the model on the filesystem */
+    modelPath: string;
+    /**
+     * Number of layers to store in VRAM.
+     * - **`"auto"`** - adapt to the current VRAM state and try to fit as many layers as possible in it.
+     * Takes into account the VRAM required to create a context with a `contextSize` set to `"auto"`.
+     * - **`"max"`** - store all layers in VRAM. If there's not enough VRAM, an error will be thrown. Use with caution.
+     * - **`number`** - store the specified number of layers in VRAM. If there's not enough VRAM, an error will be thrown. Use with caution.
+     * - **`{min?: number, max?: number, fitContext?: {contextSize: number}}`** - adapt to the current VRAM state and try to fit as
+     * many layers as possible in it, but at least `min` and at most `max` layers. Set `fitContext` to the parameters of a context you
+     * intend to create with the model, so it'll take it into account in the calculations and leave enough memory for such a context.
+     *
+     * If GPU support is disabled, will be set to `0` automatically.
+     *
+     * Defaults to `"auto"`.
+     */
+    gpuLayers?: "auto" | "max" | number | {
+        min?: number;
+        max?: number;
+        fitContext?: {
+            contextSize?: number;
+            /**
+             * Defaults to `false`.
+             */
+            embeddingContext?: boolean;
+        };
+    };
+    /** only load the vocabulary, no weights */
+    vocabOnly?: boolean;
+    /**
+     * Use mmap if possible.
+     * Defaults to `true`.
+     * If LoRA is used, this will always be set to `false`.
+     */
+    useMmap?: boolean;
+    /**
+     * Force the system to keep the model in the RAM/VRAM.
+     * Use with caution as this can crash your system if the available resources are insufficient.
+     */
+    useMlock?: boolean;
+    /**
+     * Check for tensor validity before actually loading the model.
+     * Using it increases the time it takes to load the model.
+     * Defaults to `false`.
+     */
+    checkTensors?: boolean;
+    /**
+     * Load the provided LoRA adapters onto the model after loading the model.
+     * LoRA adapters are used to modify the weights of a pretrained model to adapt to new tasks or domains
+     * without the need for extensive retraining from scratch.
+     *
+     * If a string is provided, it will be treated as a path to a single LoRA adapter file.
+     */
+    lora?: string | {
+        adapters: Array<{
+            loraFilePath: string;
+            baseModelPath?: string;
+            /**
+             * Defaults to `1`.
+             */
+            scale?: number;
+        }>;
+        /**
+         * The number of threads to use when loading the LoRA adapters.
+         * set to 0 to use the maximum threads supported by the current machine hardware.
+         *
+         * Defaults to `6`.
+         */
+        threads?: number;
+        /**
+         * Called with the LoRA adapters load percentage when the LoRA adapters are being loaded.
+         * @param loadProgress - a number between 0 (exclusive) and 1 (inclusive).
+         */
+        onLoadProgress?(loadProgress: number): void;
+    };
+    /**
+     * Called with the load percentage when the model is being loaded.
+     * > **Note:** This progress does not include the progress of loading the provided LoRA adapters (when `lora` is used)
+     * @param loadProgress - a number between 0 (exclusive) and 1 (inclusive).
+     */
+    onLoadProgress?(loadProgress: number): void;
+    /** An abort signal to abort the model load */
+    loadSignal?: AbortSignal;
+    /**
+     * Ignore insufficient memory errors and continue with the model load.
+     * Can cause the process to crash if there's not enough VRAM to fit the model.
+     *
+     * Defaults to `false`.
+     */
+    ignoreMemorySafetyChecks?: boolean;
+};
+export declare class LlamaModel {
+    readonly tokenizer: Tokenizer;
+    readonly onDispose: EventRelay<void>;
+    private constructor();
+    dispose(): Promise<void>;
+    /** @hidden */
+    [Symbol.asyncDispose](): Promise<void>;
+    get disposed(): boolean;
+    get tokens(): LlamaModelTokens;
+    get filename(): string | undefined;
+    get fileInfo(): GgufFileInfo;
+    get fileInsights(): GgufInsights;
+    /**
+     * Number of layers offloaded to the GPU.
+     * If GPU support is disabled, this will always be `0`.
+     */
+    get gpuLayers(): number;
+    /**
+     * Total model size in memory in bytes
+     */
+    get size(): number;
+    /**
+     * Transform text into tokens that can be fed to the model
+     * @param text - the text to tokenize
+     * @param [specialTokens] - if set to true, text that correspond to special tokens will be tokenized to those tokens.
+     * For example, `<s>` will be tokenized to the BOS token if `specialTokens` is set to `true`,
+     * otherwise it will be tokenized to tokens that corresponds to the plaintext `<s>` string.
+     * @param [options] - additional options for tokenization.
+     * If set to `"trimLeadingSpace"`, a leading space will be trimmed from the tokenized output if the output has an
+     * additional space at the beginning.
+     */
+    tokenize(text: string, specialTokens?: boolean, options?: "trimLeadingSpace"): Token[];
+    tokenize(text: BuiltinSpecialTokenValue, specialTokens: "builtin"): Token[];
+    /**
+     * Transform tokens into text
+     * @param tokens - the tokens to detokenize.
+     * @param [specialTokens] - if set to `true`, special tokens will be detokenized to their corresponding token text representation.
+     * Recommended for debugging purposes only.
+     * Defaults to `false`.
+     */
+    detokenize(tokens: readonly Token[], specialTokens?: boolean): string;
+    getTokenType(token: Token): GgufMetadataTokenizerTokenType | null;
+    /** Check whether the given token is a special token (a control-type token) */
+    isSpecialToken(token: Token): boolean;
+    /** Check whether the given token is an EOG (End Of Generation) token, like EOS or EOT. */
+    isEogToken(token: Token): boolean;
+    createContext(options?: LlamaContextOptions): Promise<LlamaContext>;
+    createEmbeddingContext(options?: LlamaEmbeddingContextOptions): Promise<LlamaEmbeddingContext>;
+    /** @hidden `ModelTypeDescription` type alias is too long in the documentation */
+    get typeDescription(): ModelTypeDescription;
+    /** The context size the model was trained on */
+    get trainContextSize(): number;
+    /** The size of an embedding vector the model can produce */
+    get embeddingVectorSize(): number;
+    get vocabularyType(): LlamaVocabularyType;
+}
+export declare class LlamaModelTokens {
+    private constructor();
+    /**
+     * @returns infill tokens
+     */
+    get infill(): LlamaModelInfillTokens;
+    /**
+     * @returns The BOS (Beginning Of Sequence) token.
+     */
+    get bos(): Token | null;
+    /**
+     * @returns The EOS (End Of Sequence) token.
+     */
+    get eos(): Token | null;
+    /**
+     * @returns The NL (New Line) token.
+     */
+    get nl(): Token | null;
+    /**
+     * @returns The BOS (Beginning Of Sequence) token as a string.
+     */
+    get bosString(): string | null;
+    /**
+     * @returns The EOS (End Of Sequence) token as a string.
+     */
+    get eosString(): string | null;
+    /**
+     * @returns The NL (New Line) token as a string.
+     */
+    get nlString(): string | null;
+    /**
+     * @returns Whether we should prepend a BOS (Beginning Of Sequence) token for evaluations with this model.
+     */
+    get shouldPrependBosToken(): boolean;
+}
+export declare class LlamaModelInfillTokens {
+    private constructor();
+    /**
+     * @returns The beginning of infill prefix token.
+     */
+    get prefix(): Token | null;
+    /**
+     * @returns The beginning of infill middle token.
+     */
+    get middle(): Token | null;
+    /**
+     * @returns The beginning of infill suffix token.
+     */
+    get suffix(): Token | null;
+    /**
+     * @returns End of infill middle token (End Of Text).
+     */
+    get eot(): Token | null;
+    /**
+     * @returns The beginning of infill prefix token as a string.
+     */
+    get prefixString(): string | null;
+    /**
+     * @returns The beginning of infill middle token as a string.
+     */
+    get middleString(): string | null;
+    /**
+     * @returns The beginning of infill suffix token as a string.
+     */
+    get suffixString(): string | null;
+    /**
+     * @returns End of infill middle token (End Of Text) as a string.
+     */
+    get eotString(): string | null;
+}