node-llama-cpp 3.3.2 → 3.4.1

package/dist/evaluator/LlamaContext/tokenPredictors/InputLookupTokenPredictor.d.ts

@@ -0,0 +1,58 @@
+import { Token } from "../../../types.js";
+import { TokenPredictor } from "../TokenPredictor.js";
+/**
+ * Attempts to find the last few generated tokens in the input (prompt) tokens to predict the next tokens.
+ *
+ * This is useful in input-grounded tasks (when the model frequently repeats some of the input tokens in the output,
+ * such as in text summarization or modifying code).
+ *
+ * This works in all completion classes, including `LlamaChatSession`, `LlamaChat`, and `LlamaCompletion`.
+ *
+ * Based on https://github.com/apoorvumang/prompt-lookup-decoding.
+ * @see [Using Token Predictors: Input Lookup Token Predictor](https://node-llama-cpp.withcat.ai/guide/token-prediction#input-lookup)
+ */
+export declare class InputLookupTokenPredictor extends TokenPredictor {
+    constructor(options?: {
+        patternLength?: {
+            /**
+             * Min pattern length to look for in the input tokens.
+             *
+             * Defaults to `1`.
+             */
+            min?: number;
+            /**
+             * Max pattern length to look for in the input tokens.
+             *
+             * Set to `0` to disable the max pattern size.
+             *
+             * Defaults to `0`.
+             */
+            max?: number;
+        };
+        predictionLength?: {
+            /**
+             * Minimum number of tokens to predict.
+             *
+             * Defaults to `1`.
+             */
+            min?: number;
+            /**
+             * Maximum number of tokens to predict.
+             *
+             * Defaults to `3`.
+             */
+            max?: number;
+        };
+    });
+    get patternMinLength(): number;
+    get patternMaxLength(): number;
+    get predictionMinLength(): number;
+    get predictionMaxLength(): number;
+    reset({ stateTokens }: {
+        stateTokens: Token[];
+    }): void;
+    updateInputTokens(tokens: Token[]): void;
+    pushTokens(tokens: Token[]): void;
+    predictTokens(): Token[];
+    dispose(): void;
+}

package/dist/evaluator/LlamaContext/tokenPredictors/InputLookupTokenPredictor.js

@@ -0,0 +1,138 @@
+import { DisposedError } from "lifecycle-utils";
+import { pushAll } from "../../../utils/pushAll.js";
+import { TokenPredictor } from "../TokenPredictor.js";
+const defaultPatternMinLength = 1;
+const defaultPatternMaxLength = 0;
+const defaultPredictionMinLength = 1;
+const defaultPredictionMaxLength = 3;
+/**
+ * Attempts to find the last few generated tokens in the input (prompt) tokens to predict the next tokens.
+ *
+ * This is useful in input-grounded tasks (when the model frequently repeats some of the input tokens in the output,
+ * such as in text summarization or modifying code).
+ *
+ * This works in all completion classes, including `LlamaChatSession`, `LlamaChat`, and `LlamaCompletion`.
+ *
+ * Based on https://github.com/apoorvumang/prompt-lookup-decoding.
+ * @see [Using Token Predictors: Input Lookup Token Predictor](https://node-llama-cpp.withcat.ai/guide/token-prediction#input-lookup)
+ */
+export class InputLookupTokenPredictor extends TokenPredictor {
+    /** @internal */ _patternMinLength;
+    /** @internal */ _patternMaxLength;
+    /** @internal */ _predictionMinLength;
+    /** @internal */ _predictionMaxLength;
+    /** @internal */ _lastPredictionMatchStartIndex = undefined;
+    /** @internal */ _lastPredictionMatchLength = undefined;
+    /** @internal */ _stateTokens = [];
+    /** @internal */ _inputTokens = [];
+    /** @internal */ _disposed = false;
+    constructor(options = {}) {
+        super();
+        this._patternMinLength = Math.floor(Math.max(1, options?.patternLength?.min ?? defaultPatternMinLength));
+        this._patternMaxLength = Math.floor(Math.max(0, Math.max(this._patternMinLength, options?.patternLength?.max ?? defaultPatternMaxLength)));
+        this._predictionMinLength = Math.floor(Math.max(1, options.predictionLength?.min ?? defaultPredictionMinLength));
+        this._predictionMaxLength = Math.floor(Math.max(this._patternMinLength, options.predictionLength?.max ?? defaultPredictionMaxLength));
+    }
+    get patternMinLength() {
+        return this._patternMinLength;
+    }
+    get patternMaxLength() {
+        return this._patternMaxLength;
+    }
+    get predictionMinLength() {
+        return this._predictionMinLength;
+    }
+    get predictionMaxLength() {
+        return this._predictionMaxLength;
+    }
+    reset({ stateTokens }) {
+        this._stateTokens = stateTokens.slice();
+        delete this._lastPredictionMatchStartIndex;
+        delete this._lastPredictionMatchLength;
+    }
+    updateInputTokens(tokens) {
+        this._inputTokens = tokens.slice();
+        delete this._lastPredictionMatchStartIndex;
+        delete this._lastPredictionMatchLength;
+    }
+    pushTokens(tokens) {
+        pushAll(this._stateTokens, tokens);
+        if (this._lastPredictionMatchStartIndex != null && this._lastPredictionMatchLength != null) {
+            this._lastPredictionMatchLength += tokens.length;
+        }
+    }
+    predictTokens() {
+        if (this._disposed)
+            throw new DisposedError();
+        if (this._inputTokens.length === 0 || this._stateTokens.length === 0)
+            return [];
+        if (this._lastPredictionMatchStartIndex != null && this._lastPredictionMatchLength != null) {
+            for (let p = this._lastPredictionMatchStartIndex + this._lastPredictionMatchLength - 1, s = this._stateTokens.length - 1; p >= this._lastPredictionMatchStartIndex && s >= 0; p--, s--) {
+                if (this._inputTokens[p] !== this._stateTokens[s]) {
+                    delete this._lastPredictionMatchStartIndex;
+                    delete this._lastPredictionMatchLength;
+                    break;
+                }
+            }
+            if (this._lastPredictionMatchStartIndex != null && this._lastPredictionMatchLength != null) {
+                const predictionEndIndex = this._lastPredictionMatchStartIndex + this._lastPredictionMatchLength;
+                if (predictionEndIndex < this._inputTokens.length) {
+                    return this._inputTokens.slice(predictionEndIndex, predictionEndIndex + this._predictionMaxLength);
+                }
+            }
+        }
+        const [matchStartIndex, matchLength] = this._findLongestPatternIndex(this._inputTokens, this._stateTokens);
+        if (matchStartIndex == null || matchLength == null)
+            return [];
+        const predictionEndIndex = matchStartIndex + matchLength;
+        const res = this._inputTokens.slice(predictionEndIndex, predictionEndIndex + this._predictionMaxLength);
+        if (res.length >= this._predictionMinLength) {
+            this._lastPredictionMatchStartIndex = matchStartIndex;
+            this._lastPredictionMatchLength = matchLength;
+            return res;
+        }
+        return [];
+    }
+    dispose() {
+        this._disposed = true;
+        this._stateTokens = [];
+        this._inputTokens = [];
+        delete this._lastPredictionMatchStartIndex;
+        delete this._lastPredictionMatchLength;
+    }
+    /** @internal */
+    _findLongestPatternIndex(findIn, lookupPattern) {
+        const checkIndexes = [];
+        let bestIndex = -1;
+        let bestIndexDiff = -1;
+        for (let i = findIn.length - this._predictionMinLength; i >= 0; i--) {
+            const token = findIn[i];
+            for (let j = checkIndexes.length - 1; j >= 0; j--) {
+                const startIndex = checkIndexes[j];
+                const indexDiff = startIndex - i;
+                if (lookupPattern[lookupPattern.length - 1 - indexDiff] !== token || (this._patternMaxLength > 0 && indexDiff >= this._patternMaxLength)) {
+                    checkIndexes.splice(j, 1);
+                    if (indexDiff >= this._patternMinLength && indexDiff >= bestIndexDiff) {
+                        bestIndex = startIndex;
+                        bestIndexDiff = indexDiff;
+                    }
+                }
+            }
+            if (token === lookupPattern[lookupPattern.length - 1])
+                checkIndexes.unshift(i);
+        }
+        for (let j = checkIndexes.length - 1; j >= 0; j--) {
+            const startIndex = checkIndexes[j];
+            const indexDiff = startIndex + 1;
+            checkIndexes.splice(j, 1);
+            if (indexDiff >= this._patternMinLength && indexDiff >= bestIndexDiff) {
+                bestIndex = startIndex;
+                bestIndexDiff = indexDiff;
+            }
+        }
+        if (bestIndex >= 0)
+            return [bestIndex - (bestIndexDiff - 1), bestIndexDiff];
+        return [];
+    }
+}
+//# sourceMappingURL=InputLookupTokenPredictor.js.map

package/dist/evaluator/LlamaContext/tokenPredictors/InputLookupTokenPredictor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"InputLookupTokenPredictor.js","sourceRoot":"","sources":["../../../../src/evaluator/LlamaContext/tokenPredictors/InputLookupTokenPredictor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAC,OAAO,EAAC,MAAM,2BAA2B,CAAC;AAClD,OAAO,EAAC,cAAc,EAAC,MAAM,sBAAsB,CAAC;AAEpD,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAClC,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAClC,MAAM,0BAA0B,GAAG,CAAC,CAAC;AACrC,MAAM,0BAA0B,GAAG,CAAC,CAAC;AAErC;;;;;;;;;;GAUG;AACH,MAAM,OAAO,yBAA0B,SAAQ,cAAc;IACzD,gBAAgB,CAAkB,iBAAiB,CAAS;IAC5D,gBAAgB,CAAkB,iBAAiB,CAAS;IAC5D,gBAAgB,CAAkB,oBAAoB,CAAS;IAC/D,gBAAgB,CAAkB,oBAAoB,CAAS;IAC/D,gBAAgB,CAAS,8BAA8B,GAAuB,SAAS,CAAC;IACxF,gBAAgB,CAAS,0BAA0B,GAAuB,SAAS,CAAC;IACpF,gBAAgB,CAAS,YAAY,GAAY,EAAE,CAAC;IACpD,gBAAgB,CAAS,YAAY,GAAY,EAAE,CAAC;IACpD,gBAAgB,CAAS,SAAS,GAAG,KAAK,CAAC;IAE3C,YAAmB,UAkCf,EAAE;QACF,KAAK,EAAE,CAAC;QAER,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,IAAI,uBAAuB,CAAC,CAAC,CAAC;QACzG,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAC/B,IAAI,CAAC,GAAG,CACJ,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,iBAAiB,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,IAAI,uBAAuB,CAAC,CAC3F,CACJ,CAAC;QACF,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,gBAAgB,EAAE,GAAG,IAAI,0BAA0B,CAAC,CAAC,CAAC;QACjH,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,KAAK,CAClC,IAAI,CAAC,GAAG,CACJ,IAAI,CAAC,iBAAiB,EACtB,OAAO,CAAC,gBAAgB,EAAE,GAAG,IAAI,0BAA0B,CAC9D,CACJ,CAAC;IACN,CAAC;IAED,IAAW,gBAAgB;QACvB,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAClC,CAAC;IAED,IAAW,gBAAgB;QACvB,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAClC,CAAC;IAED,IAAW,mBAAmB;QAC1B,OAAO,IAAI,CAAC,oBAAoB,CAAC;IACrC,CAAC;IAED,IAAW,mBAAmB;QAC1B,OAAO,IAAI,CAAC,oBAAoB,CAAC;IACrC,CAAC;IAEM,KAAK,CAAC,EAAC,WAAW,EAExB;QACG,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC,KAAK,EAAE,CAAC;QACxC,OAAO,IAAI,CAAC,8BAA8B,CAAC;QAC3C,OAAO,IAAI,CAAC,0BAA0B,CAAC;IAC3C,CAAC;IAEe,iBAAiB,CAAC,MAAe;QAC7C,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC,8BAA8B,CAAC;QAC3C,OAAO,IAAI,CAAC,0BAA0B,CAAC;IAC3C,CAAC;IAEM,UAAU,CAAC,MAAe;QAC7B,OAAO,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;QAEnC,IAAI,IAAI,CAAC,8BAA8B,IAAI,IAAI,IAAI,IAAI,CAAC,0BAA0B,IAAI,IAAI,EAAE,CAAC;YACzF,IAAI,CAAC,0BAA0B,IAAI,MAAM,CAAC,MAAM,CAAC;QACrD,CAAC;IACL,CAAC;IAEM,aAAa;QAChB,IAAI,IAAI,CAAC,SAAS;YACd,MAAM,IAAI,aAAa,EAAE,CAAC;QAE9B,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAChE,OAAO,EAAE,CAAC;QAEd,IAAI,IAAI,CAAC,8BAA8B,IAAI,IAAI,IAAI,IAAI,CAAC,0BAA0B,IAAI,IAAI,EAAE,CAAC;YACzF,KACI,IAAI,CAAC,GAAG,IAAI,CAAC,8BAA8B,GAAG,IAAI,CAAC,0BAA0B,GAAG,CAAC,EAC7E,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EACpC,CAAC,IAAI,IAAI,CAAC,8BAA8B,IAAI,CAAC,IAAI,CAAC,EAClD,CAAC,EAAE,EAAE,CAAC,EAAE,EACV,CAAC;gBACC,IAAI,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC;oBAChD,OAAO,IAAI,CAAC,8BAA8B,CAAC;oBAC3C,OAAO,IAAI,CAAC,0BAA0B,CAAC;oBACvC,MAAM;gBACV,CAAC;YACL,CAAC;YAED,IAAI,IAAI,CAAC,8BAA8B,IAAI,IAAI,IAAI,IAAI,CAAC,0BAA0B,IAAI,IAAI,EAAE,CAAC;gBACzF,MAAM,kBAAkB,GAAG,IAAI,CAAC,8BAA8B,GAAG,IAAI,CAAC,0BAA0B,CAAC;gBACjG,IAAI,kBAAkB,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;oBAChD,OAAO,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,kBAAkB,EAAE,kBAAkB,GAAG,IAAI,CAAC,oBAAoB,CAAC,CAAC;gBACvG,CAAC;YACL,CAAC;QACL,CAAC;QAED,MAAM,CAAC,eAAe,EAAE,WAAW,CAAC,GAAG,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QAC3G,IAAI,eAAe,IAAI,IAAI,IAAI,WAAW,IAAI,IAAI;YAC9C,OAAO,EAAE,CAAC;QAEd,MAAM,kBAAkB,GAAG,eAAe,GAAG,WAAW,CAAC;QACzD,MAAM,GAAG,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,kBAAkB,EAAE,kBAAkB,GAAG,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAExG,IAAI,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC1C,IAAI,CAAC,8BAA8B,GAAG,eAAe,CAAC;YACtD,IAAI,CAAC,0BAA0B,GAAG,WAAW,CAAC;YAC9C,OAAO,GAAG,CAAC;QACf,CAAC;QAED,OAAO,EAAE,CAAC;IACd,CAAC;IAEe,OAAO;QACnB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC,8BAA8B,CAAC;QAC3C,OAAO,IAAI,CAAC,0BAA0B,CAAC;IAC3C,CAAC;IAED,gBAAgB;IACR,wBAAwB,CAAC,MAAe,EAAE,aAAsB;QACpE,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC;QACnB,IAAI,aAAa,GAAG,CAAC,CAAC,CAAC;QAEvB,KAAK,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAClE,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YAExB,KAAK,IAAI,CAAC,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,MAAM,UAAU,GAAG,YAAY,CAAC,CAAC,CAAE,CAAC;gBACpC,MAAM,SAAS,GAAG,UAAU,GAAG,CAAC,CAAC;gBACjC,IAAI,aAAa,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,GAAG,SAAS,CAAC,KAAK,KAAK,IAAI,CACjE,IAAI,CAAC,iBAAiB,GAAG,CAAC,IAAI,SAAS,IAAI,IAAI,CAAC,iBAAiB,CACpE,EAAE,CAAC;oBACA,YAAY,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;oBAE1B,IAAI,SAAS,IAAI,IAAI,CAAC,iBAAiB,IAAI,SAAS,IAAI,aAAa,EAAE,CAAC;wBACpE,SAAS,GAAG,UAAU,CAAC;wBACvB,aAAa,GAAG,SAAS,CAAC;oBAC9B,CAAC;gBACL,CAAC;YACL,CAAC;YAED,IAAI,KAAK,KAAK,aAAa,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC;gBACjD,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAChC,CAAC;QAED,KAAK,IAAI,CAAC,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAChD,MAAM,UAAU,GAAG,YAAY,CAAC,CAAC,CAAE,CAAC;YACpC,MAAM,SAAS,GAAG,UAAU,GAAG,CAAC,CAAC;YACjC,YAAY,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAE1B,IAAI,SAAS,IAAI,IAAI,CAAC,iBAAiB,IAAI,SAAS,IAAI,aAAa,EAAE,CAAC;gBACpE,SAAS,GAAG,UAAU,CAAC;gBACvB,aAAa,GAAG,SAAS,CAAC;YAC9B,CAAC;QACL,CAAC;QAED,IAAI,SAAS,IAAI,CAAC;YACd,OAAO,CAAC,SAAS,GAAG,CAAC,aAAa,GAAG,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;QAE5D,OAAO,EAAE,CAAC;IACd,CAAC;CACJ"}

package/dist/evaluator/LlamaContext/types.d.ts

@@ -1,3 +1,6 @@
+import { PickOptions } from "../../utils/utilTypes.js";
+import type { LlamaGrammarEvaluationState } from "../LlamaGrammarEvaluationState.js";
+import type { TokenBias } from "../TokenBias.js";
 import type { Token } from "../../types.js";
 import type { LlamaContextSequence } from "./LlamaContext.js";
 export type LlamaContextOptions = {
@@ -81,7 +84,11 @@ export type LlamaContextOptions = {
          */
         min?: number;
     };
-    /** control the parallel sequences processing behavior */
+    /**
+     * Control the parallel sequences processing behavior.
+     *
+     * See {@link BatchingOptions} for more information.
+     */
     batching?: BatchingOptions;
     /**
      * Load the provided LoRA adapters onto the context.
@@ -182,18 +189,18 @@ export type LlamaContextSequenceRepeatPenalty = {
 export type BatchingOptions = {
     /**
      * The strategy used to dispatch items to be processed when there are items pending to be processed.
-     * - **`"nextTick"`** - dispatch the items on the next even loop tick.
+     * - **`"nextCycle"`** - dispatch the items on the next event loop cycle.
      * You can provide a custom function to define a custom dispatch schedule.
      *
-     * Defaults to `"nextTick"`.
+     * Defaults to `"nextCycle"`.
      */
-    dispatchSchedule?: "nextTick" | CustomBatchingDispatchSchedule;
+    dispatchSchedule?: "nextCycle" | CustomBatchingDispatchSchedule;
     /**
      * The strategy used to prioritize pending items to be processed.
      * - **`"maximumParallelism"`** - process as many different sequences in parallel as possible.
      * - **`"firstInFirstOut"`** - process items in the order they were added.
      * - **Custom prioritization function** - a custom function that prioritizes the items to be processed.
-     * See the `CustomBatchingPrioritizationStrategy` type for more information.
+     * See the {@link CustomBatchingPrioritizationStrategy} type for more information.
      *
      * Defaults to `"maximumParallelism"`.
      */
@@ -227,6 +234,191 @@ export type ContextTokensDeleteRange = {
     start: number;
     end: number;
 };
+export type SequenceEvaluateOptions = {
+    temperature?: number;
+    minP?: number;
+    topK?: number;
+    topP?: number;
+    /**
+     * Used to control the randomness of the generated text.
+     *
+     * Change the seed to get different results.
+     *
+     * Defaults to the current epoch time.
+     *
+     * Only relevant when using `temperature`.
+     */
+    seed?: number;
+    grammarEvaluationState?: LlamaGrammarEvaluationState | (() => LlamaGrammarEvaluationState | undefined);
+    repeatPenalty?: LlamaContextSequenceRepeatPenalty;
+    /**
+     * Adjust the probability of tokens being generated.
+     * Can be used to bias the model to generate tokens that you want it to lean towards,
+     * or to avoid generating tokens that you want it to avoid.
+     */
+    tokenBias?: TokenBias | (() => TokenBias);
+    /**
+     * When a lot of tokens are queued for the next batch, more than the configured `batchSize`, the tokens for each sequence will be
+     * evaluated based on the strategy chosen for the context.
+     * By default, the `"maximumParallelism"` strategy is used, which will try to evaluate as many sequences in parallel as possible,
+     * but at some point, it'll have to choose which sequences to evaluate more tokens of, so it'll prioritize the sequences with the
+     * highest evaluation priority.
+     * Also, a custom strategy can be used to prioritize the sequences differently, but generally, the higher the evaluation priority
+     * is, the more likely and more tokens will be evaluated for that sequence in the next queued batch.
+     */
+    evaluationPriority?: EvaluationPriority;
+    /**
+     * Override the sequence context shift options for this evaluation
+     *
+     * See {@link ContextShiftOptions} for more information.
+     */
+    contextShift?: ContextShiftOptions;
+    /**
+     * Yield an EOG (End Of Generation) token (like EOS and EOT) when it's generated.
+     * When `false` the generation will stop when an EOG token is generated and the token won't be yielded.
+     * Defaults to `false`.
+     */
+    yieldEogToken?: boolean;
+};
+export type SequenceEvaluateMetadataOptions = {
+    /**
+     * Get the confidence (probability) of the selected token.
+     *
+     * Same as `probabilities.get(token)` from the output.
+     *
+     * If you need only this value, you can skip getting the full probabilities list to improve performance.
+     *
+     * This value might be slightly different when evaluated on different GPUs and configurations.
+     */
+    readonly confidence?: boolean;
+    /**
+     * Get the full probabilities list of tokens from the vocabulary to be the next token, after applying the given options.
+     *
+     * Only enable when needed, as it impacts the performance.
+     *
+     * Defaults to `false`.
+     */
+    readonly probabilities?: boolean;
+};
+export type SequenceEvaluateOutput<Options extends {
+    readonly confidence?: boolean;
+    readonly probabilities?: boolean;
+} = {
+    readonly confidence: true;
+    readonly probabilities: true;
+}> = PickOptions<{
+    /**
+     * The next token generated by the model and selected using the given options (such a temperature).
+     */
+    token: Token;
+    /**
+     * The confidence (probability) of the selected token.
+     *
+     * Same as `probabilities.get(token)`.
+     *
+     * If you need only this value, you can skip getting the full probabilities list to improve performance.
+     *
+     * This value might be slightly different when evaluated on different GPUs and configurations.
+     */
+    confidence: number;
+    /**
+     * The probabilities of the tokens from the vocabulary to be the next token.
+     *
+     * A probability is a number from `0` to `1`.
+     *
+     * The probabilities might be slightly different when evaluated on different GPUs and configurations.
+     *
+     * The map is sorted by the probability of the tokens from the highest to the lowest,
+     * and is reflected in the order of the entries when iterating over the map.
+     * Use `.entries().next().value` to get the top probability pair
+     * ([learn more](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Map/entries)).
+     */
+    probabilities: Map<Token, number>;
+}, Options & {
+    token: true;
+}>;
+export type ControlledEvaluateInputItem = Token | [
+    token: Token,
+    options: {
+        generateNext?: {
+            /**
+             * Get the full probabilities list of tokens from the vocabulary to be the next token, after applying the given options.
+             *
+             * Only enable when needed, as it impacts the performance.
+             *
+             * Defaults to `false`.
+             */
+            probabilities?: boolean;
+            /**
+             * Get the confidence (probability) of the selected token.
+             *
+             * Same as `next.probabilities.get(next.token)` from the output.
+             *
+             * If you need only this value, you can skip getting the full probabilities list to improve performance.
+             *
+             * This value might be slightly different when evaluated on different GPUs and configurations.
+             */
+            confidence?: boolean;
+            /**
+             * Generate the next token with the provided options using sampling.
+             *
+             * Setting this to `true` will generate probabilities for the next token and sample it.
+             */
+            token?: boolean;
+            options?: {
+                temperature?: number;
+                minP?: number;
+                topK?: number;
+                topP?: number;
+                /**
+                 * Used to control the randomness of the generated text.
+                 *
+                 * Change the seed to get different results.
+                 *
+                 * Defaults to the current epoch time.
+                 *
+                 * Only relevant when using `temperature`.
+                 */
+                seed?: number;
+                repeatPenalty?: LlamaContextSequenceRepeatPenalty;
+                /**
+                 * Adjust the probability of tokens being generated.
+                 * Can be used to bias the model to generate tokens that you want it to lean towards,
+                 * or to avoid generating tokens that you want it to avoid.
+                 */
+                tokenBias?: TokenBias | (() => TokenBias);
+            };
+        };
+    }
+];
+export type ControlledEvaluateIndexOutput = {
+    next: {
+        token?: Token | null;
+        /**
+         * The confidence (probability) of the selected token (the `token` field in this object).
+         *
+         * Same as `next.probabilities.get(next.token)`.
+         *
+         * If you need only this value, you can skip getting the full probabilities list to improve performance.
+         *
+         * This value might be slightly different when evaluated on different GPUs and configurations.
+         */
+        confidence?: number;
+        /**
+         * The probabilities of the tokens from the vocabulary to be the next token.
+         *
+         * A probability is a number from `0` to `1`.
+         *
+         * The probabilities might be slightly different when evaluated on different GPUs and configurations.
+         *
+         * The map is sorted by the probability of the tokens from the highest to the lowest,
+         * and is reflected in the order of the entries when iterating over the map.
+         * Use `.entries().next().value` to get the top probability pair
+         * ([learn more](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Map/entries)).
+         */
+        probabilities?: Map<Token, number>;
+    };
+};
 /**
  * 1 - low
  *
@@ -235,6 +427,7 @@ export type ContextTokensDeleteRange = {
 export type EvaluationPriority = 1 | 2 | 3 | 4 | 5;
 export type BatchItem = {
     readonly tokens: readonly Token[];
+    readonly logits: readonly (true | undefined)[];
     readonly evaluationPriority: EvaluationPriority;
 };
 export type PrioritizedBatchItem = {

package/dist/evaluator/LlamaEmbeddingContext.d.ts

@@ -37,6 +37,9 @@ export type LlamaEmbeddingContextOptions = {
      */
     ignoreMemorySafetyChecks?: boolean;
 };
+/**
+ * @see [Using Embedding](https://node-llama-cpp.withcat.ai/guide/embedding) tutorial
+ */
 export declare class LlamaEmbeddingContext {
     readonly onDispose: EventRelay<void>;
     private constructor();

package/dist/evaluator/LlamaEmbeddingContext.js

@@ -2,6 +2,9 @@ import { AsyncDisposeAggregator, EventRelay, withLock } from "lifecycle-utils";
 import { tokenizeInput } from "../utils/tokenizeInput.js";
 import { resolveBeginningTokenToPrepend, resolveEndTokenToAppend } from "../utils/tokenizerUtils.js";
 import { LlamaEmbedding } from "./LlamaEmbedding.js";
+/**
+ * @see [Using Embedding](https://node-llama-cpp.withcat.ai/guide/embedding) tutorial
+ */
 export class LlamaEmbeddingContext {
     /** @internal */ _llamaContext;
     /** @internal */ _sequence;

package/dist/evaluator/LlamaEmbeddingContext.js.map

@@ -1 +1 @@
-{"version":3,"file":"LlamaEmbeddingContext.js","sourceRoot":"","sources":["../../src/evaluator/LlamaEmbeddingContext.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,sBAAsB,EAAE,UAAU,EAAE,QAAQ,EAAC,MAAM,iBAAiB,CAAC;AAG7E,OAAO,EAAC,aAAa,EAAC,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAC,8BAA8B,EAAE,uBAAuB,EAAC,MAAM,4BAA4B,CAAC;AACnG,OAAO,EAAC,cAAc,EAAC,MAAM,qBAAqB,CAAC;AA2CnD,MAAM,OAAO,qBAAqB;IAC9B,gBAAgB,CAAkB,aAAa,CAAe;IAC9D,gBAAgB,CAAkB,SAAS,CAAuB;IAClE,gBAAgB,CAAkB,kBAAkB,GAAG,IAAI,sBAAsB,EAAE,CAAC;IAEpE,SAAS,GAAG,IAAI,UAAU,EAAQ,CAAC;IAEnD,YAAoB,EAChB,aAAa,EAGhB;QACG,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC;QAElD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CACvB,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,EAAE;YAC7C,KAAK,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,CAAC;QAC3C,CAAC,CAAC,CACL,CAAC;QACF,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;QAC1D,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;YACnC,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;QACvC,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,KAAmC;QAC5D,MAAM,aAAa,GAAG,aAAa,CAAC,KAAK,EAAE,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QAEhG,IAAI,aAAa,CAAC,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW;YACrD,MAAM,IAAI,KAAK,CACX,yCAAyC;gBACzC,sFAAsF,CACzF,CAAC;aACD,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;YAC/B,OAAO,IAAI,cAAc,CAAC;gBACtB,MAAM,EAAE,EAAE;aACb,CAAC,CAAC;QAEP,MAAM,cAAc,GAAG,8BAA8B,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACpG,IAAI,cAAc,IAAI,IAAI,IAAI,aAAa,CAAC,CAAC,CAAC,KAAK,cAAc;YAC7D,aAAa,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QAE1C,MAAM,QAAQ,GAAG,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACvF,IAAI,QAAQ,IAAI,IAAI,IAAI,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,QAAQ;YACrD,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEjC,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,UAAU,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC,CAAC;oBAC1C,KAAK,EAAE,CAAC;oBACR,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,cAAc;iBACrC,CAAC,CAAC,CAAC;YAEJ,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,aAAa,EAAE,EAAC,WAAW,EAAE,IAAI,EAAC,CAAC,CAAC;YAC7E,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;gBACjC,MAAM,CAAC,4CAA4C;YACvD,CAAC;YAED,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAC7E,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAE9C,OAAO,IAAI,cAAc,CAAC;gBACtB,MAAM,EAAE,eAAe;aAC1B,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,OAAO;QAChB,MAAM,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,CAAC;IAC5C,CAAC;IAED,cAAc;IACP,CAAC,MAAM,CAAC,YAAY,CAAC;QACxB,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;IAC1B,CAAC;IAED,IAAW,QAAQ;QACf,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC;IACvC,CAAC;IAED,IAAW,KAAK;QACZ,OAAO,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC;IACpC,CAAC;IAED,gBAAgB;IACT,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EACxB,MAAM,EAGT,EAAE,EACC,WAAW,EACX,SAAS,EACT,OAAO,GAAG,CAAC,EACX,YAAY,EACZ,wBAAwB,EACG;QAC3B,IAAI,MAAM,CAAC,YAAY,CAAC,UAAU,IAAI,MAAM,CAAC,YAAY,CAAC,UAAU;YAChE,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC;QAEzF,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC;YAC5C,WAAW;YACX,SAAS;YACT,OAAO;YACP,YAAY;YACZ,wBAAwB;YACxB,WAAW,EAAE,IAAI;SACpB,CAAC,CAAC;QAEH,OAAO,IAAI,qBAAqB,CAAC;YAC7B,aAAa,EAAE,YAAY;SAC9B,CAAC,CAAC;IACP,CAAC;CACJ"}
+{"version":3,"file":"LlamaEmbeddingContext.js","sourceRoot":"","sources":["../../src/evaluator/LlamaEmbeddingContext.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,sBAAsB,EAAE,UAAU,EAAE,QAAQ,EAAC,MAAM,iBAAiB,CAAC;AAG7E,OAAO,EAAC,aAAa,EAAC,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAC,8BAA8B,EAAE,uBAAuB,EAAC,MAAM,4BAA4B,CAAC;AACnG,OAAO,EAAC,cAAc,EAAC,MAAM,qBAAqB,CAAC;AA2CnD;;GAEG;AACH,MAAM,OAAO,qBAAqB;IAC9B,gBAAgB,CAAkB,aAAa,CAAe;IAC9D,gBAAgB,CAAkB,SAAS,CAAuB;IAClE,gBAAgB,CAAkB,kBAAkB,GAAG,IAAI,sBAAsB,EAAE,CAAC;IAEpE,SAAS,GAAG,IAAI,UAAU,EAAQ,CAAC;IAEnD,YAAoB,EAChB,aAAa,EAGhB;QACG,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC;QAElD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CACvB,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,EAAE;YAC7C,KAAK,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,CAAC;QAC3C,CAAC,CAAC,CACL,CAAC;QACF,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;QAC1D,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;YACnC,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;QACvC,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,KAAmC;QAC5D,MAAM,aAAa,GAAG,aAAa,CAAC,KAAK,EAAE,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QAEhG,IAAI,aAAa,CAAC,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW;YACrD,MAAM,IAAI,KAAK,CACX,yCAAyC;gBACzC,sFAAsF,CACzF,CAAC;aACD,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;YAC/B,OAAO,IAAI,cAAc,CAAC;gBACtB,MAAM,EAAE,EAAE;aACb,CAAC,CAAC;QAEP,MAAM,cAAc,GAAG,8BAA8B,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACpG,IAAI,cAAc,IAAI,IAAI,IAAI,aAAa,CAAC,CAAC,CAAC,KAAK,cAAc;YAC7D,aAAa,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QAE1C,MAAM,QAAQ,GAAG,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACvF,IAAI,QAAQ,IAAI,IAAI,IAAI,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,QAAQ;YACrD,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEjC,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,UAAU,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC,CAAC;oBAC1C,KAAK,EAAE,CAAC;oBACR,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,cAAc;iBACrC,CAAC,CAAC,CAAC;YAEJ,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,aAAa,EAAE,EAAC,WAAW,EAAE,IAAI,EAAC,CAAC,CAAC;YAC7E,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;gBACjC,MAAM,CAAC,4CAA4C;YACvD,CAAC;YAED,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAC7E,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAE9C,OAAO,IAAI,cAAc,CAAC;gBACtB,MAAM,EAAE,eAAe;aAC1B,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,OAAO;QAChB,MAAM,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,CAAC;IAC5C,CAAC;IAED,cAAc;IACP,CAAC,MAAM,CAAC,YAAY,CAAC;QACxB,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;IAC1B,CAAC;IAED,IAAW,QAAQ;QACf,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC;IACvC,CAAC;IAED,IAAW,KAAK;QACZ,OAAO,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC;IACpC,CAAC;IAED,gBAAgB;IACT,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EACxB,MAAM,EAGT,EAAE,EACC,WAAW,EACX,SAAS,EACT,OAAO,GAAG,CAAC,EACX,YAAY,EACZ,wBAAwB,EACG;QAC3B,IAAI,MAAM,CAAC,YAAY,CAAC,UAAU,IAAI,MAAM,CAAC,YAAY,CAAC,UAAU;YAChE,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC;QAEzF,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC;YAC5C,WAAW;YACX,SAAS;YACT,OAAO;YACP,YAAY;YACZ,wBAAwB;YACxB,WAAW,EAAE,IAAI;SACpB,CAAC,CAAC;QAEH,OAAO,IAAI,qBAAqB,CAAC;YAC7B,aAAa,EAAE,YAAY;SAC9B,CAAC,CAAC;IACP,CAAC;CACJ"}

package/dist/evaluator/LlamaGrammar.d.ts

@@ -15,12 +15,18 @@ export type LlamaGrammarOptions = {
      */
     rootRuleName?: string;
 };
+/**
+ * @see [Using Grammar](https://node-llama-cpp.withcat.ai/guide/grammar) tutorial
+ */
 export declare class LlamaGrammar {
     /**
      * > GBNF files are supported.
      * > More info here: [
      * github:ggerganov/llama.cpp:grammars/README.md
      * ](https://github.com/ggerganov/llama.cpp/blob/f5fe98d11bdf9e7797bcfb05c0c3601ffc4b9d26/grammars/README.md)
+     *
+     * Prefer to create a new instance of this class by using `llama.createGrammar(...)`.
+     * @deprecated Use `llama.createGrammar(...)` instead.
      * @param llama
      * @param options
      */
@@ -29,5 +35,5 @@ export declare class LlamaGrammar {
     get rootRuleName(): string;
     get stopGenerationTriggers(): readonly (string | import("../utils/LlamaText.js")._LlamaText | readonly (string | Token)[])[];
     get trimWhitespaceSuffix(): boolean;
-    static getFor(llama: Llama, type: "json" | "json_arr" | "list" | "c" | "arithmetic" | "japanese" | "chess"): Promise<LlamaGrammar>;
+    static getFor(llama: Llama, type: "json" | "json_arr" | "english" | "list" | "c" | "arithmetic" | "japanese" | "chess"): Promise<LlamaGrammar>;
 }

package/dist/evaluator/LlamaGrammar.js

@@ -2,6 +2,9 @@ import path from "path";
 import fs from "fs-extra";
 import { getGrammarsFolder } from "../utils/getGrammarsFolder.js";
 import { LlamaText } from "../utils/LlamaText.js";
+/**
+ * @see [Using Grammar](https://node-llama-cpp.withcat.ai/guide/grammar) tutorial
+ */
 export class LlamaGrammar {
     /** @internal */ _llama;
     /** @internal */ _grammar;
@@ -14,6 +17,9 @@ export class LlamaGrammar {
      * > More info here: [
      * github:ggerganov/llama.cpp:grammars/README.md
      * ](https://github.com/ggerganov/llama.cpp/blob/f5fe98d11bdf9e7797bcfb05c0c3601ffc4b9d26/grammars/README.md)
+     *
+     * Prefer to create a new instance of this class by using `llama.createGrammar(...)`.
+     * @deprecated Use `llama.createGrammar(...)` instead.
      * @param llama
      * @param options
      */

package/dist/evaluator/LlamaGrammar.js.map

@@ -1 +1 @@
-{"version":3,"file":"LlamaGrammar.js","sourceRoot":"","sources":["../../src/evaluator/LlamaGrammar.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,iBAAiB,EAAC,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAC,SAAS,EAAC,MAAM,uBAAuB,CAAC;AAwBhD,MAAM,OAAO,YAAY;IACrB,gBAAgB,CAAiB,MAAM,CAAQ;IAC/C,gBAAgB,CAAiB,QAAQ,CAAe;IACxD,gBAAgB,CAAkB,uBAAuB,CAAgE;IACzH,gBAAgB,CAAkB,qBAAqB,CAAU;IACjE,gBAAgB,CAAkB,YAAY,CAAS;IACvD,gBAAgB,CAAkB,aAAa,CAAS;IAExD;;;;;;;OAOG;IACH,YAAmB,KAAY,EAAE,EAC7B,OAAO,EAAE,sBAAsB,GAAG,EAAE,EAAE,oBAAoB,GAAG,KAAK,EAAE,YAAY,GAAG,MAAM,EACvE;QAClB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,QAAQ,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE;YAC5D,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;YACnC,YAAY;SACf,CAAC,CAAC;QACH,IAAI,CAAC,uBAAuB,GAAG,sBAAsB,IAAI,EAAE,CAAC;QAC5D,IAAI,CAAC,qBAAqB,GAAG,oBAAoB,CAAC;QAClD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC;QAC5B,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;IACtC,CAAC;IAED,IAAW,OAAO;QACd,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED,IAAW,YAAY;QACnB,OAAO,IAAI,CAAC,aAAa,CAAC;IAC9B,CAAC;IAED,IAAW,sBAAsB;QAC7B,OAAO,IAAI,CAAC,uBAAuB,CAAC;IACxC,CAAC;IAED,IAAW,oBAAoB;QAC3B,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACtC,CAAC;IAED;;;OAGG;IACI,SAAS,CAAC,IAAY;QACzB,OAAO,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IACxD,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAY,EAAE,IAA8E;QACnH,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAEhE,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,GAAG,OAAO,CAAC,CAAC;QAE9D,IAAI,MAAM,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YACnC,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;YACvD,OAAO,IAAI,YAAY,CAAC,KAAK,EAAE;gBAC3B,OAAO;gBACP,sBAAsB,EAAE,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,oEAAoE;gBAC5H,oBAAoB,EAAE,IAAI;aAC7B,CAAC,CAAC;QACP,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,uBAAuB,cAAc,GAAG,CAAC,CAAC;IAC5F,CAAC;CACJ"}
+{"version":3,"file":"LlamaGrammar.js","sourceRoot":"","sources":["../../src/evaluator/LlamaGrammar.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,iBAAiB,EAAC,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAC,SAAS,EAAC,MAAM,uBAAuB,CAAC;AAwBhD;;GAEG;AACH,MAAM,OAAO,YAAY;IACrB,gBAAgB,CAAiB,MAAM,CAAQ;IAC/C,gBAAgB,CAAiB,QAAQ,CAAe;IACxD,gBAAgB,CAAkB,uBAAuB,CAAgE;IACzH,gBAAgB,CAAkB,qBAAqB,CAAU;IACjE,gBAAgB,CAAkB,YAAY,CAAS;IACvD,gBAAgB,CAAkB,aAAa,CAAS;IAExD;;;;;;;;;;OAUG;IACH,YAAmB,KAAY,EAAE,EAC7B,OAAO,EAAE,sBAAsB,GAAG,EAAE,EAAE,oBAAoB,GAAG,KAAK,EAAE,YAAY,GAAG,MAAM,EACvE;QAClB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,QAAQ,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE;YAC5D,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;YACnC,YAAY;SACf,CAAC,CAAC;QACH,IAAI,CAAC,uBAAuB,GAAG,sBAAsB,IAAI,EAAE,CAAC;QAC5D,IAAI,CAAC,qBAAqB,GAAG,oBAAoB,CAAC;QAClD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC;QAC5B,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;IACtC,CAAC;IAED,IAAW,OAAO;QACd,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED,IAAW,YAAY;QACnB,OAAO,IAAI,CAAC,aAAa,CAAC;IAC9B,CAAC;IAED,IAAW,sBAAsB;QAC7B,OAAO,IAAI,CAAC,uBAAuB,CAAC;IACxC,CAAC;IAED,IAAW,oBAAoB;QAC3B,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACtC,CAAC;IAED;;;OAGG;IACI,SAAS,CAAC,IAAY;QACzB,OAAO,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IACxD,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAY,EAAE,IAA0F;QAC/H,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAEhE,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,GAAG,OAAO,CAAC,CAAC;QAE9D,IAAI,MAAM,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YACnC,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;YACvD,OAAO,IAAI,YAAY,CAAC,KAAK,EAAE;gBAC3B,OAAO;gBACP,sBAAsB,EAAE,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,oEAAoE;gBAC5H,oBAAoB,EAAE,IAAI;aAC7B,CAAC,CAAC;QACP,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,uBAAuB,cAAc,GAAG,CAAC,CAAC;IAC5F,CAAC;CACJ"}

package/dist/evaluator/LlamaGrammarEvaluationState.d.ts

@@ -12,8 +12,8 @@ export type LlamaGrammarEvaluationStateOptions = {
  * This is only needed when using the `LlamaContext` class directly, since `LlamaChatSession` already handles this for you.
  */
 export declare class LlamaGrammarEvaluationState {
-    /**
-     * @param options
-     */
-    constructor({ model, grammar }: LlamaGrammarEvaluationStateOptions);
+    constructor(options: LlamaGrammarEvaluationStateOptions);
+    constructor(existingState: LlamaGrammarEvaluationState);
+    /** Clone the grammar evaluation state */
+    clone(): LlamaGrammarEvaluationState;
 }

package/dist/evaluator/LlamaGrammarEvaluationState.js

@@ -8,14 +8,22 @@
 export class LlamaGrammarEvaluationState {
     /** @internal */ _llama;
     /** @internal */ _state;
-    /**
-     * @param options
-     */
-    constructor({ model, grammar }) {
-        this._llama = model._llama;
-        if (model._llama !== grammar._llama)
-            throw new Error("The given LlamaModel and LlamaGrammar must be from the same Llama instance");
-        this._state = new model._llama._bindings.AddonGrammarEvaluationState(model._model, grammar._grammar);
+    constructor(existingStateOrOptions) {
+        if (existingStateOrOptions instanceof LlamaGrammarEvaluationState) {
+            this._llama = existingStateOrOptions._llama;
+            this._state = new this._llama._bindings.AddonGrammarEvaluationState(existingStateOrOptions._state);
+        }
+        else {
+            const { model, grammar } = existingStateOrOptions;
+            this._llama = model._llama;
+            if (model._llama !== grammar._llama)
+                throw new Error("The given LlamaModel and LlamaGrammar must be from the same Llama instance");
+            this._state = new model._llama._bindings.AddonGrammarEvaluationState(model._model, grammar._grammar);
+        }
+    }
+    /** Clone the grammar evaluation state */
+    clone() {
+        return new LlamaGrammarEvaluationState(this);
     }
 }
 //# sourceMappingURL=LlamaGrammarEvaluationState.js.map

package/dist/evaluator/LlamaGrammarEvaluationState.js.map

@@ -1 +1 @@
-{"version":3,"file":"LlamaGrammarEvaluationState.js","sourceRoot":"","sources":["../../src/evaluator/LlamaGrammarEvaluationState.ts"],"names":[],"mappings":"AAWA;;;;;;GAMG;AACH,MAAM,OAAO,2BAA2B;IACpC,gBAAgB,CAAiB,MAAM,CAAQ;IAC/C,gBAAgB,CAAiB,MAAM,CAA8B;IAErE;;OAEG;IACH,YAAmB,EAAC,KAAK,EAAE,OAAO,EAAqC;QACnE,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;QAE3B,IAAI,KAAK,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM;YAC/B,MAAM,IAAI,KAAK,CAAC,4EAA4E,CAAC,CAAC;QAElG,IAAI,CAAC,MAAM,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,2BAA2B,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IACzG,CAAC;CACJ"}
+{"version":3,"file":"LlamaGrammarEvaluationState.js","sourceRoot":"","sources":["../../src/evaluator/LlamaGrammarEvaluationState.ts"],"names":[],"mappings":"AAWA;;;;;;GAMG;AACH,MAAM,OAAO,2BAA2B;IACpC,gBAAgB,CAAiB,MAAM,CAAQ;IAC/C,gBAAgB,CAAiB,MAAM,CAA8B;IAIrE,YAAmB,sBAAwF;QACvG,IAAI,sBAAsB,YAAY,2BAA2B,EAAE,CAAC;YAChE,IAAI,CAAC,MAAM,GAAG,sBAAsB,CAAC,MAAM,CAAC;YAC5C,IAAI,CAAC,MAAM,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,2BAA2B,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC;QACvG,CAAC;aAAM,CAAC;YACJ,MAAM,EAAC,KAAK,EAAE,OAAO,EAAC,GAAG,sBAAsB,CAAC;YAChD,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;YAE3B,IAAI,KAAK,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM;gBAC/B,MAAM,IAAI,KAAK,CAAC,4EAA4E,CAAC,CAAC;YAElG,IAAI,CAAC,MAAM,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,2BAA2B,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzG,CAAC;IACL,CAAC;IAED,yCAAyC;IAClC,KAAK;QACR,OAAO,IAAI,2BAA2B,CAAC,IAAI,CAAC,CAAC;IACjD,CAAC;CACJ"}

package/dist/evaluator/LlamaJsonSchemaGrammar.d.ts

@@ -1,10 +1,15 @@
 import { GbnfJsonSchema, GbnfJsonSchemaToType } from "../utils/gbnfJson/types.js";
 import { Llama } from "../bindings/Llama.js";
 import { LlamaGrammar } from "./LlamaGrammar.js";
+/**
+ * @see [Using a JSON Schema Grammar](https://node-llama-cpp.withcat.ai/guide/grammar#json-schema) tutorial
+ * @see [Reducing Hallucinations When Using JSON Schema Grammar](https://node-llama-cpp.withcat.ai/guide/grammar#reducing-json-schema-hallucinations) tutorial
+ */
 export declare class LlamaJsonSchemaGrammar<const T extends GbnfJsonSchema> extends LlamaGrammar {
     private readonly _schema;
     /**
      * Prefer to create a new instance of this class by using `llama.createGrammarForJsonSchema(...)`.
+     * @deprecated Use `llama.createGrammarForJsonSchema(...)` instead.
      */
     constructor(llama: Llama, schema: Readonly<T>);
     get schema(): Readonly<T>;

package/dist/evaluator/LlamaJsonSchemaGrammar.js

@@ -2,10 +2,16 @@ import { getGbnfGrammarForGbnfJsonSchema } from "../utils/gbnfJson/getGbnfGramma
 import { validateObjectAgainstGbnfSchema } from "../utils/gbnfJson/utils/validateObjectAgainstGbnfSchema.js";
 import { LlamaText } from "../utils/LlamaText.js";
 import { LlamaGrammar } from "./LlamaGrammar.js";
+/* eslint-disable @stylistic/max-len */
+/**
+ * @see [Using a JSON Schema Grammar](https://node-llama-cpp.withcat.ai/guide/grammar#json-schema) tutorial
+ * @see [Reducing Hallucinations When Using JSON Schema Grammar](https://node-llama-cpp.withcat.ai/guide/grammar#reducing-json-schema-hallucinations) tutorial
+ */
 export class LlamaJsonSchemaGrammar extends LlamaGrammar {
     _schema;
     /**
      * Prefer to create a new instance of this class by using `llama.createGrammarForJsonSchema(...)`.
+     * @deprecated Use `llama.createGrammarForJsonSchema(...)` instead.
      */
     constructor(llama, schema) {
         const grammar = getGbnfGrammarForGbnfJsonSchema(schema);
@@ -25,4 +31,5 @@ export class LlamaJsonSchemaGrammar extends LlamaGrammar {
         return parsedJson;
     }
 }
+/* eslint-enable @stylistic/max-len */
 //# sourceMappingURL=LlamaJsonSchemaGrammar.js.map

package/dist/evaluator/LlamaJsonSchemaGrammar.js.map

@@ -1 +1 @@
-{"version":3,"file":"LlamaJsonSchemaGrammar.js","sourceRoot":"","sources":["../../src/evaluator/LlamaJsonSchemaGrammar.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,+BAA+B,EAAC,MAAM,sDAAsD,CAAC;AACrG,OAAO,EAAC,+BAA+B,EAAC,MAAM,4DAA4D,CAAC;AAC3G,OAAO,EAAC,SAAS,EAAC,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAC,YAAY,EAAC,MAAM,mBAAmB,CAAC;AAE/C,MAAM,OAAO,sBAAuD,SAAQ,YAAY;IACnE,OAAO,CAAI;IAE5B;;OAEG;IACH,YAAmB,KAAY,EAAE,MAAmB;QAChD,MAAM,OAAO,GAAG,+BAA+B,CAAC,MAAM,CAAC,CAAC;QAExD,KAAK,CAAC,KAAK,EAAE;YACT,OAAO;YACP,sBAAsB,EAAE,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrD,oBAAoB,EAAE,IAAI;SAC7B,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IAC1B,CAAC;IAED,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAEM,KAAK,CAAC,IAAY;QACrB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEpC,+BAA+B,CAAC,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAE1D,OAAO,UAAU,CAAC;IACtB,CAAC;CACJ"}
+{"version":3,"file":"LlamaJsonSchemaGrammar.js","sourceRoot":"","sources":["../../src/evaluator/LlamaJsonSchemaGrammar.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,+BAA+B,EAAC,MAAM,sDAAsD,CAAC;AACrG,OAAO,EAAC,+BAA+B,EAAC,MAAM,4DAA4D,CAAC;AAC3G,OAAO,EAAC,SAAS,EAAC,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAC,YAAY,EAAC,MAAM,mBAAmB,CAAC;AAE/C,uCAAuC;AACvC;;;GAGG;AACH,MAAM,OAAO,sBAAuD,SAAQ,YAAY;IACnE,OAAO,CAAI;IAE5B;;;OAGG;IACH,YAAmB,KAAY,EAAE,MAAmB;QAChD,MAAM,OAAO,GAAG,+BAA+B,CAAC,MAAM,CAAC,CAAC;QAExD,KAAK,CAAC,KAAK,EAAE;YACT,OAAO;YACP,sBAAsB,EAAE,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrD,oBAAoB,EAAE,IAAI;SAC7B,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IAC1B,CAAC;IAED,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAEM,KAAK,CAAC,IAAY;QACrB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEpC,+BAA+B,CAAC,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAE1D,OAAO,UAAU,CAAC;IACtB,CAAC;CACJ;AACD,sCAAsC"}