node-llama-cpp 3.0.0-beta.3 → 3.0.0-beta.30

package/dist/{llamaEvaluator → evaluator}/LlamaContext/LlamaContext.d.ts

@@ -1,33 +1,42 @@
 import { EventRelay } from "lifecycle-utils";
 import { Token } from "../../types.js";
-import { LlamaModel } from "../LlamaModel.js";
 import { LlamaGrammarEvaluationState } from "../LlamaGrammarEvaluationState.js";
-import { ContextShiftOptions, ContextTokensDeleteRange, EvaluationPriority, LlamaContextOptions, LlamaContextSequenceRepeatPenalty } from "./types.js";
+import { TokenMeter } from "../TokenMeter.js";
+import { TokenBias } from "../TokenBias.js";
+import { ContextShiftOptions, ContextTokensDeleteRange, EvaluationPriority, LlamaContextSequenceRepeatPenalty } from "./types.js";
+import type { LlamaModel } from "../LlamaModel/LlamaModel.js";
 export declare class LlamaContext {
     readonly onDispose: EventRelay<void>;
-    /**
-     * @param options
-     */
-    constructor({ model, sequences, seed, contextSize, batchSize, threads, batching: { dispatchSchedule: batchingDispatchSchedule, itemsPrioritizingStrategy: batchingItemsPrioritizingStrategy }, _embedding, _noSeed }: LlamaContextOptions);
-    dispose(): void;
+    private constructor();
+    dispose(): Promise<void>;
     /** @hidden */
-    [Symbol.dispose](): void;
+    [Symbol.asyncDispose](): Promise<void>;
     get disposed(): boolean;
     get model(): LlamaModel;
     get contextSize(): number;
     get batchSize(): number;
+    /**
+     * The actual size of the state in the memory in bytes.
+     * This value is provided by `llama.cpp` and doesn't include all the memory overhead of the context.
+     */
+    get stateSize(): number;
     getAllocatedContextSize(): number;
     get totalSequences(): number;
     get sequencesLeft(): number;
     /**
      * Before calling this method, make sure to call `sequencesLeft` to check if there are any sequences left.
      * When there are no sequences left, this method will throw an error.
-     * @param [options]
      */
-    getSequence({ contextShift: { size: contextShiftSize, strategy: contextShiftStrategy } }?: {
+    getSequence(options?: {
         contextShift?: ContextShiftOptions;
     }): LlamaContextSequence;
     dispatchPendingBatch(): void;
+    /**
+     * Print the timings of token evaluation since that last print for this context.
+     * > **Note:** it prints on the `LlamaLogLevel.info` level, so if you set the level of your `Llama` instance higher than that,
+     * it won't print anything.
+     */
+    printTimings(): Promise<void>;
 }
 export declare class LlamaContextSequence {
     readonly onDispose: EventRelay<void>;
@@ -40,6 +49,7 @@ export declare class LlamaContextSequence {
     get model(): LlamaModel;
     get nextTokenIndex(): number;
     get contextTokens(): Token[];
+    get tokenMeter(): TokenMeter;
     get isLoadedToMemory(): boolean;
     compareContextTokens(tokens: Token[]): {
         firstDifferentIndex: number;
@@ -51,20 +61,23 @@ export declare class LlamaContextSequence {
     clearHistory(): Promise<void>;
     /**
      * Erase context tokens in the provided ranges to free up space for new tokens to be generated.
-     * the start and end of each range are exclusive.
+     * The start of each range is inclusive, and the end of each range is exclusive.
      * For example, the range `{start: 0, end: 1}` will remove the token at the `0` index only.
      */
     eraseContextTokenRanges(ranges: ContextTokensDeleteRange[]): Promise<void>;
-    /**
-     * @param tokens
-     * @param [options]
-     */
-    evaluate(tokens: Token[], { temperature, topK, topP, grammarEvaluationState, repeatPenalty, evaluationPriority, contextShift: { size: contextShiftSize, strategy: contextShiftStrategy }, yieldEosToken }?: {
+    evaluate(tokens: Token[], options?: {
         temperature?: number;
+        minP?: number;
         topK?: number;
         topP?: number;
         grammarEvaluationState?: LlamaGrammarEvaluationState | (() => LlamaGrammarEvaluationState | undefined);
         repeatPenalty?: LlamaContextSequenceRepeatPenalty;
+        /**
+         * Adjust the probability of tokens being generated.
+         * Can be used to bias the model to generate tokens that you want it to lean towards,
+         * or to avoid generating tokens that you want it to avoid.
+         */
+        tokenBias?: TokenBias | (() => TokenBias);
         /**
          * When a lot of tokens are queued for the next batch, more than the configured `batchSize`, the tokens for each sequence will be
          * evaluated based on the strategy chosen for the context.
@@ -78,19 +91,18 @@ export declare class LlamaContextSequence {
         /** Override the sequence context shift options for this evaluation */
         contextShift?: ContextShiftOptions;
         /**
-         * Yield the EOS token when it's generated.
-         * When `false` the generation will stop when the EOS token is generated and the EOS token won't be yielded.
+         * Yield an EOG (End Of Generation) token (like EOS and EOT) when it's generated.
+         * When `false` the generation will stop when an EOG token is generated and the token won't be yielded.
          * Defaults to `false`.
          */
-        yieldEosToken?: boolean;
-    }): AsyncGenerator<Token, void>;
+        yieldEogToken?: boolean;
+    }): AsyncGenerator<Token, void | Token>;
     /**
      * Evaluate the provided tokens into the context sequence without generating new tokens.
      * @param tokens
      * @param [options]
      */
     evaluateWithoutGeneratingNewTokens(tokens: Token[], { evaluationPriority, contextShift: { size: contextShiftSize, strategy: contextShiftStrategy } }?: {
-        grammarEvaluationState?: LlamaGrammarEvaluationState;
         /**
          * When a lot of tokens are queued for the next batch, more than the configured `batchSize`, the tokens for each sequence will be
          * evaluated based on the strategy chosen for the context.
@@ -105,3 +117,11 @@ export declare class LlamaContextSequence {
         contextShift?: ContextShiftOptions;
     }): Promise<void>;
 }
+export declare function getDefaultContextBatchSize({ contextSize, sequences }: {
+    contextSize: number;
+    sequences: number;
+}): number;
+export declare function getDefaultContextSequences(): number;
+export declare function getDefaultModelContextSize({ trainContextSize }: {
+    trainContextSize?: number;
+}): number;