node-llama-cpp 2.8.6 → 3.0.0-beta.10

package/dist/evaluator/LlamaChat/LlamaChat.d.ts

@@ -0,0 +1,185 @@
+/// <reference types="node" />
+import { EventRelay } from "lifecycle-utils";
+import { ChatWrapper } from "../../ChatWrapper.js";
+import { LlamaContextSequence } from "../LlamaContext/LlamaContext.js";
+import { ChatHistoryItem, ChatModelFunctions, Token } from "../../types.js";
+import { GbnfJsonSchemaToType } from "../../utils/gbnfJson/types.js";
+import { LlamaGrammar } from "../LlamaGrammar.js";
+import { EvaluationPriority } from "../LlamaContext/types.js";
+export type LlamaChatOptions = {
+    contextSequence: LlamaContextSequence;
+    /** `"auto"` is used by default */
+    chatWrapper?: "auto" | ChatWrapper;
+    /** Automatically dispose the sequence when the session is disposed */
+    autoDisposeSequence?: boolean;
+};
+export type LLamaChatGenerateResponseOptions<Functions extends ChatModelFunctions | undefined = undefined> = {
+    onToken?: (tokens: Token[]) => void;
+    signal?: AbortSignal;
+    maxTokens?: number;
+    /**
+     * Temperature is a hyperparameter that controls the randomness of the generated text.
+     * It affects the probability distribution of the model's output tokens.
+     * A higher temperature (e.g., 1.5) makes the output more random and creative,
+     * while a lower temperature (e.g., 0.5) makes the output more focused, deterministic, and conservative.
+     * The suggested temperature is 0.8, which provides a balance between randomness and determinism.
+     * At the extreme, a temperature of 0 will always pick the most likely next token, leading to identical outputs in each run.
+     *
+     * Set to `0` to disable.
+     * Disabled by default (set to `0`).
+     */
+    temperature?: number;
+    /**
+     * From the next token candidates, discard the percentage of tokens with the lowest probability.
+     * For example, if set to `0.05`, 5% of the lowest probability tokens will be discarded.
+     * This is useful for generating more high-quality results when using a high temperature.
+     * Set to a value between `0` and `1` to enable.
+     *
+     * Only relevant when `temperature` is set to a value greater than `0`.
+     * Disabled by default.
+     */
+    minP?: number;
+    /**
+     * Limits the model to consider only the K most likely next tokens for sampling at each step of sequence generation.
+     * An integer number between `1` and the size of the vocabulary.
+     * Set to `0` to disable (which uses the full vocabulary).
+     *
+     * Only relevant when `temperature` is set to a value greater than 0.
+     */
+    topK?: number;
+    /**
+     * Dynamically selects the smallest set of tokens whose cumulative probability exceeds the threshold P,
+     * and samples the next token only from this set.
+     * A float number between `0` and `1`.
+     * Set to `1` to disable.
+     *
+     * Only relevant when `temperature` is set to a value greater than `0`.
+     */
+    topP?: number;
+    /**
+     * Trim whitespace from the end of the generated text
+     * Disabled by default.
+     */
+    trimWhitespaceSuffix?: boolean;
+    repeatPenalty?: false | LLamaChatRepeatPenalty;
+    /**
+     * See the parameter `evaluationPriority` on the `LlamaContextSequence.evaluate()` function for more information.
+     */
+    evaluationPriority?: EvaluationPriority;
+    contextShift?: LLamaChatContextShiftOptions;
+    /**
+     * The evaluation context window returned from the last evaluation.
+     * This is an optimization to utilize existing context sequence state better when possible.
+     */
+    lastEvaluationContextWindow?: {
+        /** The history of the last evaluation. */
+        history?: ChatHistoryItem[];
+        /**
+         * Minimum overlap percentage with existing context sequence state to use the last evaluation context window.
+         * If the last evaluation context window is not used, a new context will be generated based on the full history,
+         * which will decrease the likelihood of another context shift happening so soon.
+         */
+        minimumOverlapPercentageToPreventContextShift?: number;
+    };
+} & ({
+    grammar?: LlamaGrammar;
+    functions?: never;
+    documentFunctionParams?: never;
+} | {
+    grammar?: never;
+    functions?: Functions | ChatModelFunctions;
+    documentFunctionParams?: boolean;
+});
+export type LLamaChatContextShiftOptions = {
+    /**
+     * The number of tokens to delete from the context window to make space for new ones.
+     * Defaults to 10% of the context size.
+     */
+    size?: number | ((sequence: LlamaContextSequence) => number | Promise<number>);
+    /**
+     * The strategy to use when deleting tokens from the context window.
+     * Defaults to `"eraseFirstResponseAndKeepFirstSystem"`.
+     */
+    strategy?: "eraseFirstResponseAndKeepFirstSystem" | ((options: {
+        chatHistory: ChatHistoryItem[];
+        maxTokensCount: number;
+        tokenizer(text: string, specialTokens?: boolean): Token[];
+        chatWrapper: ChatWrapper;
+        lastShiftMetadata?: object | null;
+    }) => {
+        chatHistory: ChatHistoryItem[];
+        metadata?: object | null;
+    } | Promise<{
+        chatHistory: ChatHistoryItem[];
+        metadata?: object | null;
+    }>);
+    /**
+     * The `contextShiftMetadata` returned from the last evaluation.
+     * This is an optimization to utilize the existing context state better when possible.
+     */
+    lastEvaluationMetadata?: object | undefined | null;
+};
+export type LLamaChatRepeatPenalty = {
+    /**
+     * Number of recent tokens generated by the model to apply penalties to repetition of.
+     * Defaults to `64`.
+     */
+    lastTokens?: number;
+    punishTokensFilter?: (tokens: Token[]) => Token[];
+    /**
+     * Penalize new line tokens.
+     * Enabled by default.
+     */
+    penalizeNewLine?: boolean;
+    /**
+     * The relative amount to lower the probability of the tokens in `punishTokens` by
+     * Defaults to `1.1`.
+     * Set to `1` to disable.
+     */
+    penalty?: number;
+    /**
+     * For n time a token is in the `punishTokens` array, lower its probability by `n * frequencyPenalty`
+     * Disabled by default (`0`).
+     * Set to a value between `0` and `1` to enable.
+     */
+    frequencyPenalty?: number;
+    /**
+     * Lower the probability of all the tokens in the `punishTokens` array by `presencePenalty`
+     * Disabled by default (`0`).
+     * Set to a value between `0` and `1` to enable.
+     */
+    presencePenalty?: number;
+};
+export declare class LlamaChat {
+    readonly onDispose: EventRelay<void>;
+    constructor({ contextSequence, chatWrapper, autoDisposeSequence }: LlamaChatOptions);
+    dispose({ disposeSequence }?: {
+        disposeSequence?: boolean;
+    }): void;
+    /** @hidden */
+    [Symbol.dispose](): void;
+    get disposed(): boolean;
+    get chatWrapper(): ChatWrapper;
+    get sequence(): LlamaContextSequence;
+    get context(): import("../LlamaContext/LlamaContext.js").LlamaContext;
+    get model(): import("../LlamaModel.js").LlamaModel;
+    generateResponse<const Functions extends ChatModelFunctions | undefined = undefined>(history: ChatHistoryItem[], { onToken, signal, maxTokens, temperature, minP, topK, topP, grammar, trimWhitespaceSuffix, repeatPenalty, evaluationPriority, functions, documentFunctionParams, contextShift, lastEvaluationContextWindow: { history: lastEvaluationContextWindowHistory, minimumOverlapPercentageToPreventContextShift } }?: LLamaChatGenerateResponseOptions<Functions>): Promise<LlamaChatResponse<Functions>>;
+}
+export type LlamaChatResponse<Functions extends ChatModelFunctions | undefined = undefined> = {
+    response: string;
+    functionCall?: Functions extends ChatModelFunctions ? LlamaChatResponseFunctionCall<Functions> : never;
+    lastEvaluation: {
+        cleanHistory: ChatHistoryItem[];
+        contextWindow: ChatHistoryItem[];
+        contextShiftMetadata: any;
+    };
+    metadata: {
+        remainingGenerationAfterStop?: string | Token[];
+        stopReason: "eosToken" | "stopGenerationTrigger" | "functionCall" | "maxTokens";
+    };
+};
+export type LlamaChatResponseFunctionCall<Functions extends ChatModelFunctions, FunctionCallName extends keyof Functions & string = string & keyof Functions, Params = Functions[FunctionCallName]["params"] extends undefined | null | void ? undefined : GbnfJsonSchemaToType<Functions[FunctionCallName]["params"]>> = {
+    functionName: FunctionCallName;
+    params: Params;
+    raw: string;
+};