node-llama-cpp 3.0.0-beta.8 → 3.0.0

package/dist/evaluator/LlamaEmbeddingContext.js

@@ -1,77 +1,70 @@
-import { DisposeAggregator, EventRelay, withLock } from "lifecycle-utils";
+import { AsyncDisposeAggregator, EventRelay, withLock } from "lifecycle-utils";
 import { tokenizeInput } from "../utils/tokenizeInput.js";
-import { LlamaContext } from "./LlamaContext/LlamaContext.js";
+import { LlamaEmbedding } from "./LlamaEmbedding.js";
 export class LlamaEmbeddingContext {
     /** @internal */ _llamaContext;
     /** @internal */ _sequence;
-    /** @internal */ _disposeAggregator = new DisposeAggregator();
+    /** @internal */ _disposeAggregator = new AsyncDisposeAggregator();
     onDispose = new EventRelay();
-    constructor({ model, contextSize = model.trainContextSize, batchSize = contextSize, threads = 6 }) {
-        const resolvedContextSize = Math.min(contextSize, model.trainContextSize);
-        const resolvedBatchSize = Math.min(batchSize, resolvedContextSize);
-        this._llamaContext = new LlamaContext({
-            model,
-            contextSize: resolvedContextSize,
-            batchSize: resolvedBatchSize,
-            threads,
-            _embedding: true,
-            _noSeed: true
-        });
+    constructor({ _llamaContext }) {
+        this._llamaContext = _llamaContext;
         this._sequence = this._llamaContext.getSequence();
         this._disposeAggregator.add(this._llamaContext.onDispose.createListener(() => {
-            this._disposeAggregator.dispose();
+            void this._disposeAggregator.dispose();
         }));
         this._disposeAggregator.add(this.onDispose.dispatchEvent);
-        this._disposeAggregator.add(() => {
-            this._llamaContext.dispose();
+        this._disposeAggregator.add(async () => {
+            await this._llamaContext.dispose();
         });
     }
     async getEmbeddingFor(input) {
-        const resolvedInput = tokenizeInput(input, this._llamaContext.model.tokenize);
+        const resolvedInput = tokenizeInput(input, this._llamaContext.model.tokenizer);
         if (resolvedInput.length > this._llamaContext.contextSize)
             throw new Error("Input is longer than the context size. " +
                 "Try to increase the context size or use another model that supports longer contexts.");
         else if (resolvedInput.length === 0)
-            return new LlamaEmbedding({ vector: [] });
+            return new LlamaEmbedding({
+                vector: []
+            });
         return await withLock(this, "evaluate", async () => {
             await this._sequence.eraseContextTokenRanges([{
                     start: 0,
                     end: this._sequence.nextTokenIndex
                 }]);
-            await this._sequence.evaluateWithoutGeneratingNewTokens(resolvedInput);
-            const embedding = this._llamaContext._ctx.getEmbedding();
+            const iterator = this._sequence.evaluate(resolvedInput, { _noSampling: true });
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            for await (const token of iterator) {
+                break; // only generate one token to get embeddings
+            }
+            const embedding = this._llamaContext._ctx.getEmbedding(resolvedInput.length);
             const embeddingVector = Array.from(embedding);
-            return new LlamaEmbedding({ vector: embeddingVector });
+            return new LlamaEmbedding({
+                vector: embeddingVector
+            });
         });
     }
-    dispose() {
-        this._disposeAggregator.dispose();
+    async dispose() {
+        await this._disposeAggregator.dispose();
     }
     /** @hidden */
-    [Symbol.dispose]() {
+    [Symbol.asyncDispose]() {
         return this.dispose();
     }
     get disposed() {
         return this._llamaContext.disposed;
     }
-}
-export class LlamaEmbedding {
-    vector;
-    constructor({ vector }) {
-        this.vector = vector;
-    }
-    toJSON() {
-        return {
-            type: "LlamaEmbedding",
-            vector: this.vector
-        };
-    }
-    static fromJSON(json) {
-        if (json == null || json.type !== "LlamaEmbedding" || !(json.vector instanceof Array) ||
-            json.vector.some(v => typeof v !== "number"))
-            throw new Error("Invalid LlamaEmbedding JSON");
-        return new LlamaEmbedding({
-            vector: json.vector
+    /** @internal */
+    static async _create({ _model }, { contextSize, batchSize, threads = 6, createSignal, ignoreMemorySafetyChecks }) {
+        const llamaContext = await _model.createContext({
+            contextSize,
+            batchSize,
+            threads,
+            createSignal,
+            ignoreMemorySafetyChecks,
+            _embeddings: true
+        });
+        return new LlamaEmbeddingContext({
+            _llamaContext: llamaContext
         });
     }
 }

package/dist/evaluator/LlamaEmbeddingContext.js.map

@@ -1 +1 @@
-{"version":3,"file":"LlamaEmbeddingContext.js","sourceRoot":"","sources":["../../src/evaluator/LlamaEmbeddingContext.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,iBAAiB,EAAE,UAAU,EAAE,QAAQ,EAAC,MAAM,iBAAiB,CAAC;AAGxE,OAAO,EAAC,aAAa,EAAC,MAAM,2BAA2B,CAAC;AAExD,OAAO,EAAC,YAAY,EAAuB,MAAM,gCAAgC,CAAC;AAkBlF,MAAM,OAAO,qBAAqB;IAC9B,gBAAgB,CAAkB,aAAa,CAAe;IAC9D,gBAAgB,CAAkB,SAAS,CAAuB;IAClE,gBAAgB,CAAkB,kBAAkB,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAE/D,SAAS,GAAG,IAAI,UAAU,EAAQ,CAAC;IAEnD,YAAmB,EACf,KAAK,EACL,WAAW,GAAG,KAAK,CAAC,gBAAgB,EACpC,SAAS,GAAG,WAAW,EACvB,OAAO,GAAG,CAAC,EACgB;QAC3B,MAAM,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,gBAAgB,CAAC,CAAC;QAC1E,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,mBAAmB,CAAC,CAAC;QAEnE,IAAI,CAAC,aAAa,GAAG,IAAI,YAAY,CAAC;YAClC,KAAK;YACL,WAAW,EAAE,mBAAmB;YAChC,SAAS,EAAE,iBAAiB;YAC5B,OAAO;YACP,UAAU,EAAE,IAAI;YAChB,OAAO,EAAE,IAAI;SAChB,CAAC,CAAC;QACH,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC;QAElD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CACvB,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,EAAE;YAC7C,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,CAAC;QACtC,CAAC,CAAC,CACL,CAAC;QACF,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;QAC1D,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,GAAG,EAAE;YAC7B,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;QACjC,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,KAAmC;QAC5D,MAAM,aAAa,GAAG,aAAa,CAAC,KAAK,EAAE,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAE9E,IAAI,aAAa,CAAC,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW;YACrD,MAAM,IAAI,KAAK,CACX,yCAAyC;gBACzC,sFAAsF,CACzF,CAAC;aACD,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;YAC/B,OAAO,IAAI,cAAc,CAAC,EAAC,MAAM,EAAE,EAAE,EAAC,CAAC,CAAC;QAE5C,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,UAAU,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC,CAAC;oBAC1C,KAAK,EAAE,CAAC;oBACR,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,cAAc;iBACrC,CAAC,CAAC,CAAC;YAEJ,MAAM,IAAI,CAAC,SAAS,CAAC,kCAAkC,CAAC,aAAa,CAAC,CAAC;YAEvE,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACzD,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAE9C,OAAO,IAAI,cAAc,CAAC,EAAC,MAAM,EAAE,eAAe,EAAC,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,OAAO;QACV,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,CAAC;IACtC,CAAC;IAED,cAAc;IACP,CAAC,MAAM,CAAC,OAAO,CAAC;QACnB,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;IAC1B,CAAC;IAED,IAAW,QAAQ;QACf,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC;IACvC,CAAC;CACJ;AAOD,MAAM,OAAO,cAAc;IACP,MAAM,CAAW;IAEjC,YAAmB,EAAC,MAAM,EAAqB;QAC3C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACzB,CAAC;IAEM,MAAM;QACT,OAAO;YACH,IAAI,EAAE,gBAAgB;YACtB,MAAM,EAAE,IAAI,CAAC,MAAM;SACtB,CAAC;IACN,CAAC;IAEM,MAAM,CAAC,QAAQ,CAAC,IAAwB;QAC3C,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,gBAAgB,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,YAAY,KAAK,CAAC;YACjF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC;YAE5C,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAEnD,OAAO,IAAI,cAAc,CAAC;YACtB,MAAM,EAAE,IAAI,CAAC,MAAM;SACtB,CAAC,CAAC;IACP,CAAC;CACJ"}
+{"version":3,"file":"LlamaEmbeddingContext.js","sourceRoot":"","sources":["../../src/evaluator/LlamaEmbeddingContext.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,sBAAsB,EAAE,UAAU,EAAE,QAAQ,EAAC,MAAM,iBAAiB,CAAC;AAG7E,OAAO,EAAC,aAAa,EAAC,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAC,cAAc,EAAC,MAAM,qBAAqB,CAAC;AA2CnD,MAAM,OAAO,qBAAqB;IAC9B,gBAAgB,CAAkB,aAAa,CAAe;IAC9D,gBAAgB,CAAkB,SAAS,CAAuB;IAClE,gBAAgB,CAAkB,kBAAkB,GAAG,IAAI,sBAAsB,EAAE,CAAC;IAEpE,SAAS,GAAG,IAAI,UAAU,EAAQ,CAAC;IAEnD,YAAoB,EAChB,aAAa,EAGhB;QACG,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC;QAElD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CACvB,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,EAAE;YAC7C,KAAK,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,CAAC;QAC3C,CAAC,CAAC,CACL,CAAC;QACF,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;QAC1D,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;YACnC,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;QACvC,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,KAAmC;QAC5D,MAAM,aAAa,GAAG,aAAa,CAAC,KAAK,EAAE,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAE/E,IAAI,aAAa,CAAC,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW;YACrD,MAAM,IAAI,KAAK,CACX,yCAAyC;gBACzC,sFAAsF,CACzF,CAAC;aACD,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;YAC/B,OAAO,IAAI,cAAc,CAAC;gBACtB,MAAM,EAAE,EAAE;aACb,CAAC,CAAC;QAEP,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,UAAU,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC,CAAC;oBAC1C,KAAK,EAAE,CAAC;oBACR,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,cAAc;iBACrC,CAAC,CAAC,CAAC;YAEJ,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,aAAa,EAAE,EAAC,WAAW,EAAE,IAAI,EAAC,CAAC,CAAC;YAC7E,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;gBACjC,MAAM,CAAC,4CAA4C;YACvD,CAAC;YAED,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAC7E,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAE9C,OAAO,IAAI,cAAc,CAAC;gBACtB,MAAM,EAAE,eAAe;aAC1B,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,OAAO;QAChB,MAAM,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,CAAC;IAC5C,CAAC;IAED,cAAc;IACP,CAAC,MAAM,CAAC,YAAY,CAAC;QACxB,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;IAC1B,CAAC;IAED,IAAW,QAAQ;QACf,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC;IACvC,CAAC;IAED,gBAAgB;IACT,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EACxB,MAAM,EAGT,EAAE,EACC,WAAW,EACX,SAAS,EACT,OAAO,GAAG,CAAC,EACX,YAAY,EACZ,wBAAwB,EACG;QAC3B,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC;YAC5C,WAAW;YACX,SAAS;YACT,OAAO;YACP,YAAY;YACZ,wBAAwB;YACxB,WAAW,EAAE,IAAI;SACpB,CAAC,CAAC;QAEH,OAAO,IAAI,qBAAqB,CAAC;YAC7B,aAAa,EAAE,YAAY;SAC9B,CAAC,CAAC;IACP,CAAC;CACJ"}

package/dist/evaluator/LlamaGrammar.d.ts

@@ -1,30 +1,33 @@
 import { LlamaText } from "../utils/LlamaText.js";
-import { StopGenerationTrigger } from "../utils/StopGenerationDetector.js";
 import { Llama } from "../bindings/Llama.js";
+import { Token } from "../types.js";
 export type LlamaGrammarOptions = {
-    llama: Llama;
     /** GBNF grammar */
     grammar: string;
-    /** print the grammar to stdout */
-    printGrammar?: boolean;
     /** Consider any of these as EOS for the generated text. Only supported by `LlamaChat` and `LlamaChatSession` */
-    stopGenerationTriggers?: readonly (StopGenerationTrigger | LlamaText)[];
+    stopGenerationTriggers?: readonly (LlamaText | string | readonly (string | Token)[])[];
     /** Trim whitespace from the end of the generated text. Only supported by `LlamaChat` and `LlamaChatSession` */
     trimWhitespaceSuffix?: boolean;
+    /**
+     * Root rule name.
+     *
+     * Defaults to `"root"`.
+     */
+    rootRuleName?: string;
 };
 export declare class LlamaGrammar {
-    private readonly _stopGenerationTriggers;
-    private readonly _trimWhitespaceSuffix;
-    private readonly _grammarText;
     /**
      * > GBNF files are supported.
-     * > More info here: [github:ggerganov/llama.cpp:grammars/README.md](
-     * > https://github.com/ggerganov/llama.cpp/blob/f5fe98d11bdf9e7797bcfb05c0c3601ffc4b9d26/grammars/README.md)
+     * > More info here: [
+     * github:ggerganov/llama.cpp:grammars/README.md
+     * ](https://github.com/ggerganov/llama.cpp/blob/f5fe98d11bdf9e7797bcfb05c0c3601ffc4b9d26/grammars/README.md)
+     * @param llama
      * @param options
      */
-    constructor({ llama, grammar, stopGenerationTriggers, trimWhitespaceSuffix, printGrammar }: LlamaGrammarOptions);
+    constructor(llama: Llama, { grammar, stopGenerationTriggers, trimWhitespaceSuffix, rootRuleName }: LlamaGrammarOptions);
     get grammar(): string;
-    get stopGenerationTriggers(): readonly (StopGenerationTrigger | LlamaText)[];
+    get rootRuleName(): string;
+    get stopGenerationTriggers(): readonly (string | import("../utils/LlamaText.js")._LlamaText | readonly (string | Token)[])[];
     get trimWhitespaceSuffix(): boolean;
-    static getFor(llama: Llama, type: "json" | "list" | "arithmetic" | "japanese" | "chess"): Promise<LlamaGrammar>;
+    static getFor(llama: Llama, type: "json" | "json_arr" | "list" | "c" | "arithmetic" | "japanese" | "chess"): Promise<LlamaGrammar>;
 }

package/dist/evaluator/LlamaGrammar.js

@@ -5,27 +5,35 @@ import { LlamaText } from "../utils/LlamaText.js";
 export class LlamaGrammar {
     /** @internal */ _llama;
     /** @internal */ _grammar;
-    _stopGenerationTriggers;
-    _trimWhitespaceSuffix;
-    _grammarText;
+    /** @internal */ _stopGenerationTriggers;
+    /** @internal */ _trimWhitespaceSuffix;
+    /** @internal */ _grammarText;
+    /** @internal */ _rootRuleName;
     /**
      * > GBNF files are supported.
-     * > More info here: [github:ggerganov/llama.cpp:grammars/README.md](
-     * > https://github.com/ggerganov/llama.cpp/blob/f5fe98d11bdf9e7797bcfb05c0c3601ffc4b9d26/grammars/README.md)
+     * > More info here: [
+     * github:ggerganov/llama.cpp:grammars/README.md
+     * ](https://github.com/ggerganov/llama.cpp/blob/f5fe98d11bdf9e7797bcfb05c0c3601ffc4b9d26/grammars/README.md)
+     * @param llama
      * @param options
      */
-    constructor({ llama, grammar, stopGenerationTriggers = [], trimWhitespaceSuffix = false, printGrammar = false }) {
+    constructor(llama, { grammar, stopGenerationTriggers = [], trimWhitespaceSuffix = false, rootRuleName = "root" }) {
         this._llama = llama;
         this._grammar = new this._llama._bindings.AddonGrammar(grammar, {
-            printGrammar
+            addonExports: this._llama._bindings,
+            rootRuleName
         });
         this._stopGenerationTriggers = stopGenerationTriggers ?? [];
         this._trimWhitespaceSuffix = trimWhitespaceSuffix;
         this._grammarText = grammar;
+        this._rootRuleName = rootRuleName;
     }
     get grammar() {
         return this._grammarText;
     }
+    get rootRuleName() {
+        return this._rootRuleName;
+    }
     get stopGenerationTriggers() {
         return this._stopGenerationTriggers;
     }
@@ -37,10 +45,9 @@ export class LlamaGrammar {
         const grammarFile = path.join(grammarsFolder, type + ".gbnf");
         if (await fs.pathExists(grammarFile)) {
             const grammar = await fs.readFile(grammarFile, "utf8");
-            return new LlamaGrammar({
-                llama,
+            return new LlamaGrammar(llama, {
                 grammar,
-                stopGenerationTriggers: [LlamaText(["\n".repeat(10)])],
+                stopGenerationTriggers: [LlamaText(["\n".repeat(10)])], // this is a workaround for the model not stopping to generate text,
                 trimWhitespaceSuffix: true
             });
         }

package/dist/evaluator/LlamaGrammar.js.map

@@ -1 +1 @@
-{"version":3,"file":"LlamaGrammar.js","sourceRoot":"","sources":["../../src/evaluator/LlamaGrammar.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,iBAAiB,EAAC,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAC,SAAS,EAAC,MAAM,uBAAuB,CAAC;AAsBhD,MAAM,OAAO,YAAY;IACrB,gBAAgB,CAAiB,MAAM,CAAQ;IAC/C,gBAAgB,CAAiB,QAAQ,CAAe;IACvC,uBAAuB,CAAiD;IACxE,qBAAqB,CAAU;IAC/B,YAAY,CAAS;IAEtC;;;;;OAKG;IACH,YAAmB,EACf,KAAK,EAAE,OAAO,EAAE,sBAAsB,GAAG,EAAE,EAAE,oBAAoB,GAAG,KAAK,EAAE,YAAY,GAAG,KAAK,EAC7E;QAClB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,QAAQ,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE;YAC5D,YAAY;SACf,CAAC,CAAC;QACH,IAAI,CAAC,uBAAuB,GAAG,sBAAsB,IAAI,EAAE,CAAC;QAC5D,IAAI,CAAC,qBAAqB,GAAG,oBAAoB,CAAC;QAClD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC;IAChC,CAAC;IAED,IAAW,OAAO;QACd,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED,IAAW,sBAAsB;QAC7B,OAAO,IAAI,CAAC,uBAAuB,CAAC;IACxC,CAAC;IAED,IAAW,oBAAoB;QAC3B,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACtC,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAY,EAAE,IAA2D;QAChG,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAEhE,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,GAAG,OAAO,CAAC,CAAC;QAE9D,IAAI,MAAM,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE;YAClC,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;YACvD,OAAO,IAAI,YAAY,CAAC;gBACpB,KAAK;gBACL,OAAO;gBACP,sBAAsB,EAAE,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBACtD,oBAAoB,EAAE,IAAI;aAC7B,CAAC,CAAC;SACN;QAED,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,uBAAuB,cAAc,GAAG,CAAC,CAAC;IAC5F,CAAC;CACJ"}
+{"version":3,"file":"LlamaGrammar.js","sourceRoot":"","sources":["../../src/evaluator/LlamaGrammar.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,iBAAiB,EAAC,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAC,SAAS,EAAC,MAAM,uBAAuB,CAAC;AAwBhD,MAAM,OAAO,YAAY;IACrB,gBAAgB,CAAiB,MAAM,CAAQ;IAC/C,gBAAgB,CAAiB,QAAQ,CAAe;IACxD,gBAAgB,CAAkB,uBAAuB,CAAgE;IACzH,gBAAgB,CAAkB,qBAAqB,CAAU;IACjE,gBAAgB,CAAkB,YAAY,CAAS;IACvD,gBAAgB,CAAkB,aAAa,CAAS;IAExD;;;;;;;OAOG;IACH,YAAmB,KAAY,EAAE,EAC7B,OAAO,EAAE,sBAAsB,GAAG,EAAE,EAAE,oBAAoB,GAAG,KAAK,EAAE,YAAY,GAAG,MAAM,EACvE;QAClB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,QAAQ,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE;YAC5D,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;YACnC,YAAY;SACf,CAAC,CAAC;QACH,IAAI,CAAC,uBAAuB,GAAG,sBAAsB,IAAI,EAAE,CAAC;QAC5D,IAAI,CAAC,qBAAqB,GAAG,oBAAoB,CAAC;QAClD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC;QAC5B,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;IACtC,CAAC;IAED,IAAW,OAAO;QACd,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED,IAAW,YAAY;QACnB,OAAO,IAAI,CAAC,aAAa,CAAC;IAC9B,CAAC;IAED,IAAW,sBAAsB;QAC7B,OAAO,IAAI,CAAC,uBAAuB,CAAC;IACxC,CAAC;IAED,IAAW,oBAAoB;QAC3B,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACtC,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAY,EAAE,IAA8E;QACnH,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAEhE,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,GAAG,OAAO,CAAC,CAAC;QAE9D,IAAI,MAAM,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YACnC,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;YACvD,OAAO,IAAI,YAAY,CAAC,KAAK,EAAE;gBAC3B,OAAO;gBACP,sBAAsB,EAAE,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,oEAAoE;gBAC5H,oBAAoB,EAAE,IAAI;aAC7B,CAAC,CAAC;QACP,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,uBAAuB,cAAc,GAAG,CAAC,CAAC;IAC5F,CAAC;CACJ"}

package/dist/evaluator/LlamaGrammarEvaluationState.d.ts

@@ -1,15 +1,19 @@
-import { LlamaGrammar } from "./LlamaGrammar.js";
+import type { LlamaGrammar } from "./LlamaGrammar.js";
+import type { LlamaModel } from "./LlamaModel/LlamaModel.js";
 export type LlamaGrammarEvaluationStateOptions = {
+    model: LlamaModel;
     grammar: LlamaGrammar;
 };
 /**
  * Grammar evaluation state is used to track the model response to determine the next allowed characters for the model to generate.
+ *
  * Create a new grammar evaluation state for every response you generate with the model.
- * This is only needed when using the `LlamaContext` class directly, as `LlamaChatSession` already handles this for you.
+ *
+ * This is only needed when using the `LlamaContext` class directly, since `LlamaChatSession` already handles this for you.
  */
 export declare class LlamaGrammarEvaluationState {
     /**
      * @param options
      */
-    constructor({ grammar }: LlamaGrammarEvaluationStateOptions);
+    constructor({ model, grammar }: LlamaGrammarEvaluationStateOptions);
 }

package/dist/evaluator/LlamaGrammarEvaluationState.js

@@ -1,7 +1,9 @@
 /**
  * Grammar evaluation state is used to track the model response to determine the next allowed characters for the model to generate.
+ *
  * Create a new grammar evaluation state for every response you generate with the model.
- * This is only needed when using the `LlamaContext` class directly, as `LlamaChatSession` already handles this for you.
+ *
+ * This is only needed when using the `LlamaContext` class directly, since `LlamaChatSession` already handles this for you.
  */
 export class LlamaGrammarEvaluationState {
     /** @internal */ _llama;
@@ -9,9 +11,11 @@ export class LlamaGrammarEvaluationState {
     /**
      * @param options
      */
-    constructor({ grammar }) {
-        this._llama = grammar._llama;
-        this._state = new grammar._llama._bindings.AddonGrammarEvaluationState(grammar._grammar);
+    constructor({ model, grammar }) {
+        this._llama = model._llama;
+        if (model._llama !== grammar._llama)
+            throw new Error("The given LlamaModel and LlamaGrammar must be from the same Llama instance");
+        this._state = new model._llama._bindings.AddonGrammarEvaluationState(model._model, grammar._grammar);
     }
 }
 //# sourceMappingURL=LlamaGrammarEvaluationState.js.map

package/dist/evaluator/LlamaGrammarEvaluationState.js.map

@@ -1 +1 @@
-{"version":3,"file":"LlamaGrammarEvaluationState.js","sourceRoot":"","sources":["../../src/evaluator/LlamaGrammarEvaluationState.ts"],"names":[],"mappings":"AASA;;;;GAIG;AACH,MAAM,OAAO,2BAA2B;IACpC,gBAAgB,CAAiB,MAAM,CAAQ;IAC/C,gBAAgB,CAAiB,MAAM,CAA8B;IAErE;;OAEG;IACH,YAAmB,EAAC,OAAO,EAAqC;QAC5D,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,MAAM,GAAG,IAAI,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,2BAA2B,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC7F,CAAC;CACJ"}
+{"version":3,"file":"LlamaGrammarEvaluationState.js","sourceRoot":"","sources":["../../src/evaluator/LlamaGrammarEvaluationState.ts"],"names":[],"mappings":"AAWA;;;;;;GAMG;AACH,MAAM,OAAO,2BAA2B;IACpC,gBAAgB,CAAiB,MAAM,CAAQ;IAC/C,gBAAgB,CAAiB,MAAM,CAA8B;IAErE;;OAEG;IACH,YAAmB,EAAC,KAAK,EAAE,OAAO,EAAqC;QACnE,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;QAE3B,IAAI,KAAK,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM;YAC/B,MAAM,IAAI,KAAK,CAAC,4EAA4E,CAAC,CAAC;QAElG,IAAI,CAAC,MAAM,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,2BAA2B,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IACzG,CAAC;CACJ"}

package/dist/evaluator/LlamaJsonSchemaGrammar.d.ts

@@ -3,6 +3,9 @@ import { Llama } from "../bindings/Llama.js";
 import { LlamaGrammar } from "./LlamaGrammar.js";
 export declare class LlamaJsonSchemaGrammar<const T extends Readonly<GbnfJsonSchema>> extends LlamaGrammar {
     private readonly _schema;
+    /**
+     * Prefer to create a new instance of this class by using `llama.createGrammarForJsonSchema(...)`.
+     */
     constructor(llama: Llama, schema: T);
     parse(json: string): GbnfJsonSchemaToType<T>;
 }

package/dist/evaluator/LlamaJsonSchemaGrammar.js

@@ -1,13 +1,15 @@
-import { getGbnfGrammarForGbnfJsonSchema } from "../utils/getGbnfGrammarForGbnfJsonSchema.js";
+import { getGbnfGrammarForGbnfJsonSchema } from "../utils/gbnfJson/getGbnfGrammarForGbnfJsonSchema.js";
 import { validateObjectAgainstGbnfSchema } from "../utils/gbnfJson/utils/validateObjectAgainstGbnfSchema.js";
 import { LlamaText } from "../utils/LlamaText.js";
 import { LlamaGrammar } from "./LlamaGrammar.js";
 export class LlamaJsonSchemaGrammar extends LlamaGrammar {
     _schema;
+    /**
+     * Prefer to create a new instance of this class by using `llama.createGrammarForJsonSchema(...)`.
+     */
     constructor(llama, schema) {
         const grammar = getGbnfGrammarForGbnfJsonSchema(schema);
-        super({
-            llama,
+        super(llama, {
             grammar,
             stopGenerationTriggers: [LlamaText(["\n".repeat(4)])],
             trimWhitespaceSuffix: true

package/dist/evaluator/LlamaJsonSchemaGrammar.js.map

@@ -1 +1 @@
-{"version":3,"file":"LlamaJsonSchemaGrammar.js","sourceRoot":"","sources":["../../src/evaluator/LlamaJsonSchemaGrammar.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,+BAA+B,EAAC,MAAM,6CAA6C,CAAC;AAC5F,OAAO,EAAC,+BAA+B,EAAC,MAAM,4DAA4D,CAAC;AAC3G,OAAO,EAAC,SAAS,EAAC,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAC,YAAY,EAAC,MAAM,mBAAmB,CAAC;AAE/C,MAAM,OAAO,sBAAiE,SAAQ,YAAY;IAC7E,OAAO,CAAI;IAE5B,YAAmB,KAAY,EAAE,MAAS;QACtC,MAAM,OAAO,GAAG,+BAA+B,CAAC,MAAM,CAAC,CAAC;QAExD,KAAK,CAAC;YACF,KAAK;YACL,OAAO;YACP,sBAAsB,EAAE,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrD,oBAAoB,EAAE,IAAI;SAC7B,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IAC1B,CAAC;IAEM,KAAK,CAAC,IAAY;QACrB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEpC,+BAA+B,CAAC,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAE1D,OAAO,UAAU,CAAC;IACtB,CAAC;CACJ"}
+{"version":3,"file":"LlamaJsonSchemaGrammar.js","sourceRoot":"","sources":["../../src/evaluator/LlamaJsonSchemaGrammar.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,+BAA+B,EAAC,MAAM,sDAAsD,CAAC;AACrG,OAAO,EAAC,+BAA+B,EAAC,MAAM,4DAA4D,CAAC;AAC3G,OAAO,EAAC,SAAS,EAAC,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAC,YAAY,EAAC,MAAM,mBAAmB,CAAC;AAE/C,MAAM,OAAO,sBAAiE,SAAQ,YAAY;IAC7E,OAAO,CAAI;IAE5B;;OAEG;IACH,YAAmB,KAAY,EAAE,MAAS;QACtC,MAAM,OAAO,GAAG,+BAA+B,CAAC,MAAM,CAAC,CAAC;QAExD,KAAK,CAAC,KAAK,EAAE;YACT,OAAO;YACP,sBAAsB,EAAE,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrD,oBAAoB,EAAE,IAAI;SAC7B,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IAC1B,CAAC;IAEM,KAAK,CAAC,IAAY;QACrB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEpC,+BAA+B,CAAC,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAE1D,OAAO,UAAU,CAAC;IACtB,CAAC;CACJ"}

package/dist/evaluator/LlamaModel/LlamaModel.d.ts

@@ -0,0 +1,255 @@
+import { EventRelay } from "lifecycle-utils";
+import { Token, Tokenizer } from "../../types.js";
+import { ModelTypeDescription } from "../../bindings/AddonTypes.js";
+import { LlamaVocabularyType } from "../../bindings/types.js";
+import { GgufFileInfo } from "../../gguf/types/GgufFileInfoTypes.js";
+import { GgufInsights } from "../../gguf/insights/GgufInsights.js";
+import { LlamaContextOptions } from "../LlamaContext/types.js";
+import { LlamaContext } from "../LlamaContext/LlamaContext.js";
+import { LlamaEmbeddingContext, LlamaEmbeddingContextOptions } from "../LlamaEmbeddingContext.js";
+import { GgufMetadata } from "../../gguf/types/GgufMetadataTypes.js";
+import { OverridesObject } from "../../utils/OverridesObject.js";
+import { TokenAttributes } from "./utils/TokenAttributes.js";
+import type { BuiltinSpecialTokenValue } from "../../utils/LlamaText.js";
+export type LlamaModelOptions = {
+    /** path to the model on the filesystem */
+    modelPath: string;
+    /**
+     * Number of layers to store in VRAM.
+     * - **`"auto"`** - adapt to the current VRAM state and try to fit as many layers as possible in it.
+     * Takes into account the VRAM required to create a context with a `contextSize` set to `"auto"`.
+     * - **`"max"`** - store all layers in VRAM. If there's not enough VRAM, an error will be thrown. Use with caution.
+     * - **`number`** - store the specified number of layers in VRAM. If there's not enough VRAM, an error will be thrown. Use with caution.
+     * - **`{min?: number, max?: number, fitContext?: {contextSize: number}}`** - adapt to the current VRAM state and try to fit as
+     * many layers as possible in it, but at least `min` and at most `max` layers. Set `fitContext` to the parameters of a context you
+     * intend to create with the model, so it'll take it into account in the calculations and leave enough memory for such a context.
+     *
+     * If GPU support is disabled, will be set to `0` automatically.
+     *
+     * Defaults to `"auto"`.
+     */
+    gpuLayers?: "auto" | "max" | number | {
+        min?: number;
+        max?: number;
+        fitContext?: {
+            contextSize?: number;
+            /**
+             * Defaults to `false`.
+             */
+            embeddingContext?: boolean;
+        };
+    };
+    /**
+     * Only load the vocabulary, not weight tensors.
+     *
+     * Useful when you only want to use the model to use its tokenizer but not for evaluation.
+     *
+     * Defaults to `false`.
+     */
+    vocabOnly?: boolean;
+    /**
+     * Use mmap if possible.
+     *
+     * Defaults to `true`.
+     */
+    useMmap?: boolean;
+    /**
+     * Force the system to keep the model in the RAM/VRAM.
+     * Use with caution as this can crash your system if the available resources are insufficient.
+     */
+    useMlock?: boolean;
+    /**
+     * Check for tensor validity before actually loading the model.
+     * Using it increases the time it takes to load the model.
+     *
+     * Defaults to `false`.
+     */
+    checkTensors?: boolean;
+    /**
+     * Enable flash attention by default for contexts created with this model.
+     * Only works with models that support flash attention.
+     *
+     * Flash attention is an optimization in the attention mechanism that makes inference faster, more efficient and uses less memory.
+     *
+     * The support for flash attention is currently experimental and may not always work as expected.
+     * Use with caution.
+     *
+     * This option will be ignored if flash attention is not supported by the model.
+     *
+     * Enabling this affects the calculations of default values for the model and contexts created with it
+     * as flash attention reduces the amount of memory required,
+     * which allows for more layers to be offloaded to the GPU and for context sizes to be bigger.
+     *
+     * Defaults to `false`.
+     *
+     * Upon flash attention exiting the experimental status, the default value will become `true`.
+     */
+    defaultContextFlashAttention?: boolean;
+    /**
+     * Called with the load percentage when the model is being loaded.
+     * @param loadProgress - a number between 0 (exclusive) and 1 (inclusive).
+     */
+    onLoadProgress?(loadProgress: number): void;
+    /** An abort signal to abort the model load */
+    loadSignal?: AbortSignal;
+    /**
+     * Ignore insufficient memory errors and continue with the model load.
+     * Can cause the process to crash if there's not enough VRAM to fit the model.
+     *
+     * Defaults to `false`.
+     */
+    ignoreMemorySafetyChecks?: boolean;
+    /**
+     * Metadata overrides to load the model with.
+     *
+     * > **Note:** Most metadata value overrides aren't supported and overriding them will have no effect on `llama.cpp`.
+     * > Only use this for metadata values that are explicitly documented to be supported by `llama.cpp` to be overridden,
+     * > and only in cases when this is crucial, as this is not guaranteed to always work as expected.
+     */
+    metadataOverrides?: OverridesObject<GgufMetadata, number | bigint | boolean | string>;
+};
+export declare class LlamaModel {
+    readonly tokenizer: Tokenizer;
+    readonly onDispose: EventRelay<void>;
+    private constructor();
+    dispose(): Promise<void>;
+    /** @hidden */
+    [Symbol.asyncDispose](): Promise<void>;
+    get disposed(): boolean;
+    get tokens(): LlamaModelTokens;
+    get filename(): string | undefined;
+    get fileInfo(): GgufFileInfo;
+    get fileInsights(): GgufInsights;
+    /**
+     * Number of layers offloaded to the GPU.
+     * If GPU support is disabled, this will always be `0`.
+     */
+    get gpuLayers(): number;
+    /**
+     * Total model size in memory in bytes
+     */
+    get size(): number;
+    get flashAttentionSupported(): boolean;
+    get defaultContextFlashAttention(): boolean;
+    /**
+     * Transform text into tokens that can be fed to the model
+     * @param text - the text to tokenize
+     * @param [specialTokens] - if set to true, text that correspond to special tokens will be tokenized to those tokens.
+     * For example, `<s>` will be tokenized to the BOS token if `specialTokens` is set to `true`,
+     * otherwise it will be tokenized to tokens that corresponds to the plaintext `<s>` string.
+     * @param [options] - additional options for tokenization.
+     * If set to `"trimLeadingSpace"`, a leading space will be trimmed from the tokenized output if the output has an
+     * additional space at the beginning.
+     */
+    tokenize(text: string, specialTokens?: boolean, options?: "trimLeadingSpace"): Token[];
+    tokenize(text: BuiltinSpecialTokenValue, specialTokens: "builtin"): Token[];
+    /**
+     * Transform tokens into text
+     * @param tokens - the tokens to detokenize.
+     * @param [specialTokens] - if set to `true`, special tokens will be detokenized to their corresponding token text representation.
+     *
+     * Recommended for debugging purposes only.
+     *
+     * > **Note:** there may be additional spaces around special tokens that were not present in the original text - this is not a bug,
+     * this is [how the tokenizer is supposed to work](https://github.com/ggerganov/llama.cpp/pull/7697#issuecomment-2144003246).
+     *
+     * Defaults to `false`.
+     * @param [lastTokens] - the last few tokens that preceded the tokens to detokenize.
+     * If provided, the last few tokens will be used to determine whether a space has to be added before the current tokens or not,
+     * and apply other detokenizer-specific heuristics to provide the correct text continuation to the existing tokens.
+     *
+     * Using it may have no effect with some models, but it is still recommended.
+     */
+    detokenize(tokens: readonly Token[], specialTokens?: boolean, lastTokens?: readonly Token[]): string;
+    getTokenAttributes(token: Token): TokenAttributes;
+    /** Check whether the given token is a special token (a control-type token or a token with no normal text representation) */
+    isSpecialToken(token: Token | undefined): boolean;
+    iterateAllTokens(): Generator<Token, void, unknown>;
+    /** Check whether the given token is an EOG (End Of Generation) token, like EOS or EOT. */
+    isEogToken(token: Token | undefined): boolean;
+    createContext(options?: LlamaContextOptions): Promise<LlamaContext>;
+    createEmbeddingContext(options?: LlamaEmbeddingContextOptions): Promise<LlamaEmbeddingContext>;
+    /**
+     * Get warnings about the model file that would affect its usage.
+     *
+     * These warnings include all the warnings generated by `GgufInsights`, but are more comprehensive.
+     */
+    getWarnings(): string[];
+    /** @hidden `ModelTypeDescription` type alias is too long in the documentation */
+    get typeDescription(): ModelTypeDescription;
+    /** The context size the model was trained on */
+    get trainContextSize(): number;
+    /** The size of an embedding vector the model can produce */
+    get embeddingVectorSize(): number;
+    get vocabularyType(): LlamaVocabularyType;
+}
+export declare class LlamaModelTokens {
+    private constructor();
+    /**
+     * @returns infill tokens
+     */
+    get infill(): LlamaModelInfillTokens;
+    /**
+     * @returns The BOS (Beginning Of Sequence) token.
+     */
+    get bos(): Token | null;
+    /**
+     * @returns The EOS (End Of Sequence) token.
+     */
+    get eos(): Token | null;
+    /**
+     * @returns The EOT (End Of Turn) token.
+     */
+    get eot(): Token | null;
+    /**
+     * @returns The NL (New Line) token.
+     */
+    get nl(): Token | null;
+    /**
+     * @returns The BOS (Beginning Of Sequence) token text representation.
+     */
+    get bosString(): string | null;
+    /**
+     * @returns The EOS (End Of Sequence) token text representation.
+     */
+    get eosString(): string | null;
+    /**
+     * @returns The EOT (End Of Turn) token text representation.
+     */
+    get eotString(): string | null;
+    /**
+     * @returns The NL (New Line) token text representation.
+     */
+    get nlString(): string | null;
+    /**
+     * @returns Whether we should prepend a BOS (Beginning Of Sequence) token for evaluations with this model.
+     */
+    get shouldPrependBosToken(): boolean;
+}
+export declare class LlamaModelInfillTokens {
+    private constructor();
+    /**
+     * @returns The beginning of infill prefix token.
+     */
+    get prefix(): Token | null;
+    /**
+     * @returns The beginning of infill middle token.
+     */
+    get middle(): Token | null;
+    /**
+     * @returns The beginning of infill suffix token.
+     */
+    get suffix(): Token | null;
+    /**
+     * @returns The beginning of infill prefix token as a string.
+     */
+    get prefixString(): string | null;
+    /**
+     * @returns The beginning of infill middle token as a string.
+     */
+    get middleString(): string | null;
+    /**
+     * @returns The beginning of infill suffix token as a string.
+     */
+    get suffixString(): string | null;
+}