node-llama-cpp 3.0.0-beta.2 → 3.0.0-beta.21

package/dist/{llamaEvaluator → evaluator}/LlamaChat/LlamaChat.d.ts

@@ -2,10 +2,12 @@
 import { EventRelay } from "lifecycle-utils";
 import { ChatWrapper } from "../../ChatWrapper.js";
 import { LlamaContextSequence } from "../LlamaContext/LlamaContext.js";
-import { ChatHistoryItem, ChatModelFunctions, Token } from "../../types.js";
+import { ChatHistoryItem, ChatModelFunctions, LLamaContextualRepeatPenalty, Token } from "../../types.js";
 import { GbnfJsonSchemaToType } from "../../utils/gbnfJson/types.js";
 import { LlamaGrammar } from "../LlamaGrammar.js";
+import { LlamaText } from "../../utils/LlamaText.js";
 import { EvaluationPriority } from "../LlamaContext/types.js";
+import { TokenBias } from "../TokenBias.js";
 export type LlamaChatOptions = {
     contextSequence: LlamaContextSequence;
     /** `"auto"` is used by default */
@@ -16,6 +18,13 @@ export type LlamaChatOptions = {
 export type LLamaChatGenerateResponseOptions<Functions extends ChatModelFunctions | undefined = undefined> = {
     onToken?: (tokens: Token[]) => void;
     signal?: AbortSignal;
+    /**
+     * When a response already started being generated and then the signal is aborted,
+     * the generation will stop and the response will be returned as is instead of throwing an error.
+     *
+     * Defaults to `false`.
+     */
+    stopOnAbortSignal?: boolean;
     maxTokens?: number;
     /**
      * Temperature is a hyperparameter that controls the randomness of the generated text.
@@ -29,6 +38,16 @@ export type LLamaChatGenerateResponseOptions<Functions extends ChatModelFunction
      * Disabled by default (set to `0`).
      */
     temperature?: number;
+    /**
+     * From the next token candidates, discard the percentage of tokens with the lowest probability.
+     * For example, if set to `0.05`, 5% of the lowest probability tokens will be discarded.
+     * This is useful for generating more high-quality results when using a high temperature.
+     * Set to a value between `0` and `1` to enable.
+     *
+     * Only relevant when `temperature` is set to a value greater than `0`.
+     * Disabled by default.
+     */
+    minP?: number;
     /**
      * Limits the model to consider only the K most likely next tokens for sampling at each step of sequence generation.
      * An integer number between `1` and the size of the vocabulary.
@@ -51,12 +70,22 @@ export type LLamaChatGenerateResponseOptions<Functions extends ChatModelFunction
      * Disabled by default.
      */
     trimWhitespaceSuffix?: boolean;
-    repeatPenalty?: false | LLamaChatRepeatPenalty;
+    repeatPenalty?: false | LLamaContextualRepeatPenalty;
+    /**
+     * Adjust the probability of tokens being generated.
+     * Can be used to bias the model to generate tokens that you want it to lean towards,
+     * or to avoid generating tokens that you want it to avoid.
+     */
+    tokenBias?: TokenBias | (() => TokenBias);
     /**
      * See the parameter `evaluationPriority` on the `LlamaContextSequence.evaluate()` function for more information.
      */
     evaluationPriority?: EvaluationPriority;
     contextShift?: LLamaChatContextShiftOptions;
+    /**
+     * Custom stop triggers to stop the generation of the response when any of the provided triggers are found.
+     */
+    customStopTriggers?: (LlamaText | string | (string | Token)[])[];
     /**
      * The evaluation context window returned from the last evaluation.
      * This is an optimization to utilize existing context sequence state better when possible.
@@ -109,37 +138,6 @@ export type LLamaChatContextShiftOptions = {
      */
     lastEvaluationMetadata?: object | undefined | null;
 };
-export type LLamaChatRepeatPenalty = {
-    /**
-     * Number of recent tokens generated by the model to apply penalties to repetition of.
-     * Defaults to `64`.
-     */
-    lastTokens?: number;
-    punishTokensFilter?: (tokens: Token[]) => Token[];
-    /**
-     * Penalize new line tokens.
-     * Enabled by default.
-     */
-    penalizeNewLine?: boolean;
-    /**
-     * The relative amount to lower the probability of the tokens in `punishTokens` by
-     * Defaults to `1.1`.
-     * Set to `1` to disable.
-     */
-    penalty?: number;
-    /**
-     * For n time a token is in the `punishTokens` array, lower its probability by `n * frequencyPenalty`
-     * Disabled by default (`0`).
-     * Set to a value between `0` and `1` to enable.
-     */
-    frequencyPenalty?: number;
-    /**
-     * Lower the probability of all the tokens in the `punishTokens` array by `presencePenalty`
-     * Disabled by default (`0`).
-     * Set to a value between `0` and `1` to enable.
-     */
-    presencePenalty?: number;
-};
 export declare class LlamaChat {
     readonly onDispose: EventRelay<void>;
     constructor({ contextSequence, chatWrapper, autoDisposeSequence }: LlamaChatOptions);
@@ -153,7 +151,7 @@ export declare class LlamaChat {
     get sequence(): LlamaContextSequence;
     get context(): import("../LlamaContext/LlamaContext.js").LlamaContext;
     get model(): import("../LlamaModel.js").LlamaModel;
-    generateResponse<const Functions extends ChatModelFunctions | undefined = undefined>(history: ChatHistoryItem[], { onToken, signal, maxTokens, temperature, topK, topP, grammar, trimWhitespaceSuffix, repeatPenalty, evaluationPriority, functions, documentFunctionParams, contextShift, lastEvaluationContextWindow: { history: lastEvaluationContextWindowHistory, minimumOverlapPercentageToPreventContextShift } }?: LLamaChatGenerateResponseOptions<Functions>): Promise<LlamaChatResponse<Functions>>;
+    generateResponse<const Functions extends ChatModelFunctions | undefined = undefined>(history: ChatHistoryItem[], { onToken, signal, stopOnAbortSignal, maxTokens, temperature, minP, topK, topP, grammar, trimWhitespaceSuffix, repeatPenalty, tokenBias, evaluationPriority, functions, documentFunctionParams, contextShift, customStopTriggers, lastEvaluationContextWindow: { history: lastEvaluationContextWindowHistory, minimumOverlapPercentageToPreventContextShift } }?: LLamaChatGenerateResponseOptions<Functions>): Promise<LlamaChatResponse<Functions>>;
 }
 export type LlamaChatResponse<Functions extends ChatModelFunctions | undefined = undefined> = {
     response: string;
@@ -165,7 +163,11 @@ export type LlamaChatResponse<Functions extends ChatModelFunctions | undefined =
     };
     metadata: {
         remainingGenerationAfterStop?: string | Token[];
-        stopReason: "eosToken" | "stopGenerationTrigger" | "functionCall" | "maxTokens";
+        stopReason: "eogToken" | "stopGenerationTrigger" | "functionCall" | "maxTokens" | "abort";
+    } | {
+        remainingGenerationAfterStop?: string | Token[];
+        stopReason: "customStopTrigger";
+        customStopTrigger: (string | Token)[];
     };
 };
 export type LlamaChatResponseFunctionCall<Functions extends ChatModelFunctions, FunctionCallName extends keyof Functions & string = string & keyof Functions, Params = Functions[FunctionCallName]["params"] extends undefined | null | void ? undefined : GbnfJsonSchemaToType<Functions[FunctionCallName]["params"]>> = {