node-llama-cpp 1.3.1 → 2.1.0

package/dist/llamaEvaluator/LlamaBins.d.ts

@@ -1,4 +1,4 @@
-import { type LLAMAModel, type LLAMAContext } from "../utils/getBin.js";
+import { type LLAMAModel, type LLAMAContext, type LLAMAGrammar } from "../utils/getBin.js";
 export declare const llamaCppNode: import("../utils/getBin.js").LlamaCppNodeModule;
-declare const LLAMAModel: LLAMAModel, LLAMAContext: LLAMAContext;
-export { LLAMAModel, LLAMAContext };
+declare const LLAMAModel: LLAMAModel, LLAMAContext: LLAMAContext, LLAMAGrammar: LLAMAGrammar;
+export { LLAMAModel, LLAMAContext, LLAMAGrammar };

package/dist/llamaEvaluator/LlamaBins.js

@@ -1,5 +1,5 @@
 import { loadBin } from "../utils/getBin.js";
 export const llamaCppNode = await loadBin();
-const { LLAMAModel, LLAMAContext } = llamaCppNode;
-export { LLAMAModel, LLAMAContext };
+const { LLAMAModel, LLAMAContext, LLAMAGrammar } = llamaCppNode;
+export { LLAMAModel, LLAMAContext, LLAMAGrammar };
 //# sourceMappingURL=LlamaBins.js.map

package/dist/llamaEvaluator/LlamaBins.js.map

@@ -1 +1 @@
-{"version":3,"file":"LlamaBins.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaBins.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,OAAO,EAAqC,MAAM,oBAAoB,CAAC;AAE/E,MAAM,CAAC,MAAM,YAAY,GAAG,MAAM,OAAO,EAAE,CAAC;AAC5C,MAAM,EAAC,UAAU,EAAE,YAAY,EAAC,GAAG,YAAY,CAAC;AAEhD,OAAO,EAAC,UAAU,EAAE,YAAY,EAAC,CAAC"}
+{"version":3,"file":"LlamaBins.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaBins.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,OAAO,EAAwD,MAAM,oBAAoB,CAAC;AAElG,MAAM,CAAC,MAAM,YAAY,GAAG,MAAM,OAAO,EAAE,CAAC;AAC5C,MAAM,EAAC,UAAU,EAAE,YAAY,EAAE,YAAY,EAAC,GAAG,YAAY,CAAC;AAE9D,OAAO,EAAC,UAAU,EAAE,YAAY,EAAE,YAAY,EAAC,CAAC"}

package/dist/llamaEvaluator/LlamaChatSession.d.ts

@@ -1,5 +1,6 @@
 /// <reference types="node" />
 import { ChatPromptWrapper } from "../ChatPromptWrapper.js";
+import { Token } from "../types.js";
 import { LlamaContext } from "./LlamaContext.js";
 export declare class LlamaChatSession {
     private readonly _systemPrompt;
@@ -7,18 +8,22 @@ export declare class LlamaChatSession {
     private readonly _promptWrapper;
     private _promptIndex;
     private _initialized;
+    private _lastStopString;
+    private _lastStopStringSuffix;
     private readonly _ctx;
     constructor({ context, printLLamaSystemInfo, promptWrapper, systemPrompt }: {
         context: LlamaContext;
         printLLamaSystemInfo?: boolean;
-        promptWrapper?: ChatPromptWrapper;
+        promptWrapper?: ChatPromptWrapper | "auto";
         systemPrompt?: string;
     });
     get initialized(): boolean;
     get context(): LlamaContext;
     init(): Promise<void>;
-    prompt(prompt: string, onToken?: (tokens: number[]) => void, { signal }?: {
+    prompt(prompt: string, { onToken, signal, maxTokens }?: {
+        onToken?(tokens: Token[]): void;
         signal?: AbortSignal;
+        maxTokens?: number;
     }): Promise<string>;
     private _evalTokens;
     private _checkStopString;

package/dist/llamaEvaluator/LlamaChatSession.js

@@ -2,6 +2,7 @@ import { defaultChatSystemPrompt } from "../config.js";
 import { withLock } from "../utils/withLock.js";
 import { AbortError } from "../AbortError.js";
 import { GeneralChatPromptWrapper } from "../chatWrappers/GeneralChatPromptWrapper.js";
+import { getChatWrapperByBos } from "../chatWrappers/createChatWrapperByBos.js";
 import { LlamaModel } from "./LlamaModel.js";
 const UNKNOWN_UNICODE_CHAR = "\ufffd";
 export class LlamaChatSession {
@@ -10,12 +11,22 @@ export class LlamaChatSession {
     _promptWrapper;
     _promptIndex = 0;
     _initialized = false;
+    _lastStopString = null;
+    _lastStopStringSuffix = null;
     _ctx;
     constructor({ context, printLLamaSystemInfo = false, promptWrapper = new GeneralChatPromptWrapper(), systemPrompt = defaultChatSystemPrompt }) {
         this._ctx = context;
         this._printLLamaSystemInfo = printLLamaSystemInfo;
-        this._promptWrapper = promptWrapper;
         this._systemPrompt = systemPrompt;
+        if (promptWrapper === "auto") {
+            const chatWrapper = getChatWrapperByBos(context.getBosString());
+            if (chatWrapper != null)
+                this._promptWrapper = new chatWrapper();
+            else
+                this._promptWrapper = new GeneralChatPromptWrapper();
+        }
+        else
+            this._promptWrapper = promptWrapper;
     }
     get initialized() {
         return this._initialized;
@@ -32,17 +43,30 @@ export class LlamaChatSession {
             this._initialized = true;
         });
     }
-    async prompt(prompt, onToken, { signal } = {}) {
+    async prompt(prompt, { onToken, signal, maxTokens } = {}) {
         if (!this.initialized)
             await this.init();
         return await withLock(this, "prompt", async () => {
-            const promptText = this._promptWrapper.wrapPrompt(prompt, { systemPrompt: this._systemPrompt, promptIndex: this._promptIndex });
+            const promptText = this._promptWrapper.wrapPrompt(prompt, {
+                systemPrompt: this._systemPrompt,
+                promptIndex: this._promptIndex,
+                lastStopString: this._lastStopString,
+                lastStopStringSuffix: this._promptIndex == 0
+                    ? (this._ctx.prependBos
+                        ? this._ctx.getBosString()
+                        : null)
+                    : this._lastStopStringSuffix
+            });
             this._promptIndex++;
-            return await this._evalTokens(this._ctx.encode(promptText), onToken, { signal });
+            this._lastStopString = null;
+            this._lastStopStringSuffix = null;
+            const { text, stopString, stopStringSuffix } = await this._evalTokens(this._ctx.encode(promptText), { onToken, signal, maxTokens });
+            this._lastStopString = stopString;
+            this._lastStopStringSuffix = stopStringSuffix;
+            return text;
         });
     }
-    async _evalTokens(tokens, onToken, { signal } = {}) {
-        const decodeTokens = (tokens) => this._ctx.decode(Uint32Array.from(tokens));
+    async _evalTokens(tokens, { onToken, signal, maxTokens } = {}) {
         const stopStrings = this._promptWrapper.getStopStrings();
         const stopStringIndexes = Array(stopStrings.length).fill(0);
         const skippedChunksQueue = [];
@@ -50,10 +74,14 @@ export class LlamaChatSession {
         for await (const chunk of this._ctx.evaluate(tokens)) {
             if (signal?.aborted)
                 throw new AbortError();
-            const tokenStr = decodeTokens([chunk]);
-            const { shouldReturn, skipTokenEvent } = this._checkStopString(tokenStr, stopStringIndexes);
+            const tokenStr = this._ctx.decode(Uint32Array.from([chunk]));
+            const { shouldReturn, skipTokenEvent, stopString, stopStringSuffix } = this._checkStopString(tokenStr, stopStringIndexes);
             if (shouldReturn)
-                return decodeTokens(res);
+                return {
+                    text: this._ctx.decode(Uint32Array.from(res)),
+                    stopString,
+                    stopStringSuffix
+                };
             // if the token is unknown, it means it's not complete character
             if (tokenStr === UNKNOWN_UNICODE_CHAR || skipTokenEvent) {
                 skippedChunksQueue.push(chunk);
@@ -66,8 +94,14 @@ export class LlamaChatSession {
             }
             res.push(chunk);
             onToken?.([chunk]);
+            if (maxTokens != null && maxTokens > 0 && res.length >= maxTokens)
+                break;
         }
-        return decodeTokens(res);
+        return {
+            text: this._ctx.decode(Uint32Array.from(res)),
+            stopString: null,
+            stopStringSuffix: null
+        };
     }
     _checkStopString(tokenStr, stopStringIndexes) {
         const stopStrings = this._promptWrapper.getStopStrings();
@@ -87,7 +121,13 @@ export class LlamaChatSession {
                 }
             }
             if (stopStringIndexes[stopStringIndex] === stopString.length) {
-                return { shouldReturn: true };
+                return {
+                    shouldReturn: true,
+                    stopString,
+                    stopStringSuffix: tokenStr.length === stopString.length
+                        ? null
+                        : tokenStr.slice(stopString.length)
+                };
             }
             skipTokenEvent ||= localShouldSkipTokenEvent;
         }

package/dist/llamaEvaluator/LlamaChatSession.js.map

@@ -1 +1 @@
-{"version":3,"file":"LlamaChatSession.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaChatSession.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,uBAAuB,EAAC,MAAM,cAAc,CAAC;AACrD,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAC;AAE9C,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAC,wBAAwB,EAAC,MAAM,6CAA6C,CAAC;AACrF,OAAO,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAG3C,MAAM,oBAAoB,GAAG,QAAQ,CAAC;AAEtC,MAAM,OAAO,gBAAgB;IACR,aAAa,CAAS;IACtB,qBAAqB,CAAU;IAC/B,cAAc,CAAoB;IAC3C,YAAY,GAAW,CAAC,CAAC;IACzB,YAAY,GAAY,KAAK,CAAC;IACrB,IAAI,CAAe;IAEpC,YAAmB,EACf,OAAO,EACP,oBAAoB,GAAG,KAAK,EAC5B,aAAa,GAAG,IAAI,wBAAwB,EAAE,EAC9C,YAAY,GAAG,uBAAuB,EAMzC;QACG,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC;QACpB,IAAI,CAAC,qBAAqB,GAAG,oBAAoB,CAAC;QAClD,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QAEpC,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;IACtC,CAAC;IAED,IAAW,WAAW;QAClB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED,IAAW,OAAO;QACd,OAAO,IAAI,CAAC,IAAI,CAAC;IACrB,CAAC;IAEM,KAAK,CAAC,IAAI;QACb,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE;YACpC,IAAI,IAAI,CAAC,YAAY;gBACjB,OAAO;YAEX,IAAI,IAAI,CAAC,qBAAqB;gBAC1B,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;YAE5D,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC7B,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,MAAM,CAAC,MAAc,EAAE,OAAoC,EAAE,EAAC,MAAM,KAA8B,EAAE;QAC7G,IAAI,CAAC,IAAI,CAAC,WAAW;YACjB,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAEtB,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,IAAI,EAAE;YAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,MAAM,EAAE,EAAC,YAAY,EAAE,IAAI,CAAC,aAAa,EAAE,WAAW,EAAE,IAAI,CAAC,YAAY,EAAC,CAAC,CAAC;YAC9H,IAAI,CAAC,YAAY,EAAE,CAAC;YAEpB,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,OAAO,EAAE,EAAC,MAAM,EAAC,CAAC,CAAC;QACnF,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,MAAmB,EAAE,OAAoC,EAAE,EAAC,MAAM,KAA8B,EAAE;QACxH,MAAM,YAAY,GAAG,CAAC,MAAgB,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QAEtF,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,CAAC;QACzD,MAAM,iBAAiB,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5D,MAAM,kBAAkB,GAAa,EAAE,CAAC;QACxC,MAAM,GAAG,GAAa,EAAE,CAAC;QAGzB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;YAClD,IAAI,MAAM,EAAE,OAAO;gBACf,MAAM,IAAI,UAAU,EAAE,CAAC;YAE3B,MAAM,QAAQ,GAAG,YAAY,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;YACvC,MAAM,EAAC,YAAY,EAAE,cAAc,EAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC;YAE1F,IAAI,YAAY;gBACZ,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC;YAE7B,gEAAgE;YAChE,IAAI,QAAQ,KAAK,oBAAoB,IAAI,cAAc,EAAE;gBACrD,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC/B,SAAS;aACZ;YAED,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE;gBAC/B,GAAG,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,CAAC;gBAChC,OAAO,EAAE,CAAC,kBAAkB,CAAC,CAAC;gBAC9B,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC;aACjC;YAED,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChB,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;SACtB;QAED,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAEO,gBAAgB,CAAC,QAAgB,EAAE,iBAA2B;QAClE,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,CAAC;QACzD,IAAI,cAAc,GAAG,KAAK,CAAC;QAE3B,KAAK,IAAI,eAAe,GAAG,CAAC,EAAE,eAAe,GAAG,WAAW,CAAC,MAAM,EAAE,eAAe,EAAE,EAAE;YACnF,MAAM,UAAU,GAAG,WAAW,CAAC,eAAe,CAAC,CAAC;YAEhD,IAAI,yBAAyB,GAAG,KAAK,CAAC;YACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,IAAI,iBAAiB,CAAC,eAAe,CAAC,KAAK,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBAClG,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC,EAAE;oBAChE,iBAAiB,CAAC,eAAe,CAAC,EAAE,CAAC;oBACrC,yBAAyB,GAAG,IAAI,CAAC;iBACpC;qBAAM;oBACH,iBAAiB,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;oBACvC,yBAAyB,GAAG,KAAK,CAAC;oBAClC,MAAM;iBACT;aACJ;YAED,IAAI,iBAAiB,CAAC,eAAe,CAAC,KAAK,UAAU,CAAC,MAAM,EAAE;gBAC1D,OAAO,EAAC,YAAY,EAAE,IAAI,EAAC,CAAC;aAC/B;YAED,cAAc,KAAK,yBAAyB,CAAC;SAChD;QAED,OAAO,EAAC,cAAc,EAAC,CAAC;IAC5B,CAAC;CACJ"}
+{"version":3,"file":"LlamaChatSession.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaChatSession.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,uBAAuB,EAAC,MAAM,cAAc,CAAC;AACrD,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAC;AAE9C,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAC,wBAAwB,EAAC,MAAM,6CAA6C,CAAC;AACrF,OAAO,EAAC,mBAAmB,EAAC,MAAM,2CAA2C,CAAC;AAE9E,OAAO,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAG3C,MAAM,oBAAoB,GAAG,QAAQ,CAAC;AAEtC,MAAM,OAAO,gBAAgB;IACR,aAAa,CAAS;IACtB,qBAAqB,CAAU;IAC/B,cAAc,CAAoB;IAC3C,YAAY,GAAW,CAAC,CAAC;IACzB,YAAY,GAAY,KAAK,CAAC;IAC9B,eAAe,GAAkB,IAAI,CAAC;IACtC,qBAAqB,GAAkB,IAAI,CAAC;IACnC,IAAI,CAAe;IAEpC,YAAmB,EACf,OAAO,EACP,oBAAoB,GAAG,KAAK,EAC5B,aAAa,GAAG,IAAI,wBAAwB,EAAE,EAC9C,YAAY,GAAG,uBAAuB,EAMzC;QACG,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC;QACpB,IAAI,CAAC,qBAAqB,GAAG,oBAAoB,CAAC;QAClD,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;QAElC,IAAI,aAAa,KAAK,MAAM,EAAE;YAC1B,MAAM,WAAW,GAAG,mBAAmB,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;YAEhE,IAAI,WAAW,IAAI,IAAI;gBACnB,IAAI,CAAC,cAAc,GAAG,IAAI,WAAW,EAAE,CAAC;;gBAExC,IAAI,CAAC,cAAc,GAAG,IAAI,wBAAwB,EAAE,CAAC;SAC5D;;YACG,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;IAC5C,CAAC;IAED,IAAW,WAAW;QAClB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED,IAAW,OAAO;QACd,OAAO,IAAI,CAAC,IAAI,CAAC;IACrB,CAAC;IAEM,KAAK,CAAC,IAAI;QACb,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE;YACpC,IAAI,IAAI,CAAC,YAAY;gBACjB,OAAO;YAEX,IAAI,IAAI,CAAC,qBAAqB;gBAC1B,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;YAE5D,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC7B,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,MAAM,CAAC,MAAc,EAAE,EAChC,OAAO,EAAE,MAAM,EAAE,SAAS,KACqD,EAAE;QACjF,IAAI,CAAC,IAAI,CAAC,WAAW;YACjB,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAEtB,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,IAAI,EAAE;YAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,MAAM,EAAE;gBACtD,YAAY,EAAE,IAAI,CAAC,aAAa;gBAChC,WAAW,EAAE,IAAI,CAAC,YAAY;gBAC9B,cAAc,EAAE,IAAI,CAAC,eAAe;gBACpC,oBAAoB,EAAE,IAAI,CAAC,YAAY,IAAI,CAAC;oBACxC,CAAC,CAAC,CACE,IAAI,CAAC,IAAI,CAAC,UAAU;wBAChB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;wBAC1B,CAAC,CAAC,IAAI,CACb;oBACD,CAAC,CAAC,IAAI,CAAC,qBAAqB;aACnC,CAAC,CAAC;YACH,IAAI,CAAC,YAAY,EAAE,CAAC;YACpB,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;YAC5B,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;YAElC,MAAM,EAAC,IAAI,EAAE,UAAU,EAAE,gBAAgB,EAAC,GACtC,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,EAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAC,CAAC,CAAC;YACvF,IAAI,CAAC,eAAe,GAAG,UAAU,CAAC;YAClC,IAAI,CAAC,qBAAqB,GAAG,gBAAgB,CAAC;YAE9C,OAAO,IAAI,CAAC;QAChB,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,MAAmB,EAAE,EAC3C,OAAO,EAAE,MAAM,EAAE,SAAS,KACqD,EAAE;QACjF,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,CAAC;QACzD,MAAM,iBAAiB,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5D,MAAM,kBAAkB,GAAY,EAAE,CAAC;QACvC,MAAM,GAAG,GAAY,EAAE,CAAC;QAExB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;YAClD,IAAI,MAAM,EAAE,OAAO;gBACf,MAAM,IAAI,UAAU,EAAE,CAAC;YAE3B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC7D,MAAM,EAAC,YAAY,EAAE,cAAc,EAAE,UAAU,EAAE,gBAAgB,EAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC;YAExH,IAAI,YAAY;gBACZ,OAAO;oBACH,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAC7C,UAAU;oBACV,gBAAgB;iBACnB,CAAC;YAEN,gEAAgE;YAChE,IAAI,QAAQ,KAAK,oBAAoB,IAAI,cAAc,EAAE;gBACrD,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC/B,SAAS;aACZ;YAED,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE;gBAC/B,GAAG,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,CAAC;gBAChC,OAAO,EAAE,CAAC,kBAAkB,CAAC,CAAC;gBAC9B,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC;aACjC;YAED,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChB,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;YAEnB,IAAI,SAAS,IAAI,IAAI,IAAI,SAAS,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,IAAI,SAAS;gBAC7D,MAAM;SACb;QAED,OAAO;YACH,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC7C,UAAU,EAAE,IAAI;YAChB,gBAAgB,EAAE,IAAI;SACzB,CAAC;IACN,CAAC;IAEO,gBAAgB,CAAC,QAAgB,EAAE,iBAA2B;QAClE,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,CAAC;QACzD,IAAI,cAAc,GAAG,KAAK,CAAC;QAE3B,KAAK,IAAI,eAAe,GAAG,CAAC,EAAE,eAAe,GAAG,WAAW,CAAC,MAAM,EAAE,eAAe,EAAE,EAAE;YACnF,MAAM,UAAU,GAAG,WAAW,CAAC,eAAe,CAAC,CAAC;YAEhD,IAAI,yBAAyB,GAAG,KAAK,CAAC;YACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,IAAI,iBAAiB,CAAC,eAAe,CAAC,KAAK,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBAClG,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC,EAAE;oBAChE,iBAAiB,CAAC,eAAe,CAAC,EAAE,CAAC;oBACrC,yBAAyB,GAAG,IAAI,CAAC;iBACpC;qBAAM;oBACH,iBAAiB,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;oBACvC,yBAAyB,GAAG,KAAK,CAAC;oBAClC,MAAM;iBACT;aACJ;YAED,IAAI,iBAAiB,CAAC,eAAe,CAAC,KAAK,UAAU,CAAC,MAAM,EAAE;gBAC1D,OAAO;oBACH,YAAY,EAAE,IAAI;oBAClB,UAAU;oBACV,gBAAgB,EAAE,QAAQ,CAAC,MAAM,KAAK,UAAU,CAAC,MAAM;wBACnD,CAAC,CAAC,IAAI;wBACN,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC;iBAC1C,CAAC;aACL;YAED,cAAc,KAAK,yBAAyB,CAAC;SAChD;QAED,OAAO,EAAC,cAAc,EAAC,CAAC;IAC5B,CAAC;CACJ"}

package/dist/llamaEvaluator/LlamaContext.d.ts

@@ -1,12 +1,41 @@
+import { Token } from "../types.js";
 import { LlamaModel } from "./LlamaModel.js";
+import { LlamaGrammar } from "./LlamaGrammar.js";
 export declare class LlamaContext {
     private readonly _ctx;
     private _prependBos;
-    constructor({ model, prependBos }: {
+    constructor({ model, grammar, prependBos }: {
         model: LlamaModel;
+        grammar?: LlamaGrammar;
         prependBos?: boolean;
     });
     encode(text: string): Uint32Array;
     decode(tokens: Uint32Array): string;
-    evaluate(tokens: Uint32Array, getRestrictions?: () => Uint32Array): AsyncGenerator<number, void, unknown>;
+    get prependBos(): boolean;
+    /**
+     * @returns {Token | null} The BOS (Beginning Of Sequence) token.
+     */
+    getBosToken(): Token | null;
+    /**
+     * @returns {Token | null} The EOS (End Of Sequence) token.
+     */
+    getEosToken(): Token | null;
+    /**
+     * @returns {Token | null} The NL (New Line) token.
+     */
+    getNlToken(): Token | null;
+    /**
+     * @returns {string | null} The BOS (Beginning Of Sequence) token as a string.
+     */
+    getBosString(): string | null;
+    /**
+     * @returns {string | null} The EOS (End Of Sequence) token as a string.
+     */
+    getEosString(): string | null;
+    /**
+     * @returns {string | null} The NL (New Line) token as a string.
+     */
+    getNlString(): string | null;
+    getContextSize(): number;
+    evaluate(tokens: Uint32Array): AsyncGenerator<Token, void>;
 }

package/dist/llamaEvaluator/LlamaContext.js

@@ -1,31 +1,98 @@
-import { LLAMAContext, llamaCppNode } from "./LlamaBins.js";
+import { removeNullFields } from "../utils/removeNullFields.js";
+import { LLAMAContext } from "./LlamaBins.js";
 export class LlamaContext {
     _ctx;
     _prependBos;
-    constructor({ model, prependBos = true }) {
-        this._ctx = new LLAMAContext(model._model);
+    constructor({ model, grammar, prependBos = true }) {
+        this._ctx = new LLAMAContext(model._model, removeNullFields({
+            grammar: grammar?._grammar
+        }));
         this._prependBos = prependBos;
     }
     encode(text) {
+        if (text === "")
+            return new Uint32Array();
         return this._ctx.encode(text);
     }
     decode(tokens) {
+        if (tokens.length === 0)
+            return "";
         return this._ctx.decode(tokens);
     }
-    async *evaluate(tokens, getRestrictions) {
+    get prependBos() {
+        return this._prependBos;
+    }
+    /**
+     * @returns {Token | null} The BOS (Beginning Of Sequence) token.
+     */
+    getBosToken() {
+        const bosToken = this._ctx.tokenBos();
+        if (bosToken === -1)
+            return null;
+        return bosToken;
+    }
+    /**
+     * @returns {Token | null} The EOS (End Of Sequence) token.
+     */
+    getEosToken() {
+        const eosToken = this._ctx.tokenEos();
+        if (eosToken === -1)
+            return null;
+        return eosToken;
+    }
+    /**
+     * @returns {Token | null} The NL (New Line) token.
+     */
+    getNlToken() {
+        const nlToken = this._ctx.tokenNl();
+        if (nlToken === -1)
+            return null;
+        return nlToken;
+    }
+    /**
+     * @returns {string | null} The BOS (Beginning Of Sequence) token as a string.
+     */
+    getBosString() {
+        const bosToken = this.getBosToken();
+        if (bosToken == null)
+            return null;
+        return this._ctx.getTokenString(bosToken);
+    }
+    /**
+     * @returns {string | null} The EOS (End Of Sequence) token as a string.
+     */
+    getEosString() {
+        const eosToken = this.getEosToken();
+        if (eosToken == null)
+            return null;
+        return this._ctx.getTokenString(eosToken);
+    }
+    /**
+     * @returns {string | null} The NL (New Line) token as a string.
+     */
+    getNlString() {
+        const nlToken = this.getNlToken();
+        if (nlToken == null)
+            return null;
+        return this._ctx.getTokenString(nlToken);
+    }
+    getContextSize() {
+        return this._ctx.getContextSize();
+    }
+    async *evaluate(tokens) {
         let evalTokens = tokens;
         if (this._prependBos) {
             const tokenArray = Array.from(tokens);
-            tokenArray.unshift(llamaCppNode.tokenBos());
+            tokenArray.unshift(this._ctx.tokenBos());
             evalTokens = Uint32Array.from(tokenArray);
             this._prependBos = false;
         }
         // eslint-disable-next-line no-constant-condition
         while (true) {
             // Evaluate to get the next token.
-            const nextToken = await this._ctx.eval(evalTokens, getRestrictions?.());
+            const nextToken = await this._ctx.eval(evalTokens);
             // the assistant finished answering
-            if (nextToken === llamaCppNode.tokenEos())
+            if (nextToken === this._ctx.tokenEos())
                 break;
             yield nextToken;
             // Create tokens for the next eval.

package/dist/llamaEvaluator/LlamaContext.js.map

@@ -1 +1 @@
-{"version":3,"file":"LlamaContext.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaContext.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAE,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAG1D,MAAM,OAAO,YAAY;IACJ,IAAI,CAAe;IAC5B,WAAW,CAAU;IAE7B,YAAmB,EAAC,KAAK,EAAE,UAAU,GAAG,IAAI,EAA4C;QACpF,IAAI,CAAC,IAAI,GAAG,IAAI,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC3C,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,IAAY;QACtB,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,MAAmB;QAC7B,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAEM,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAmB,EAAE,eAAmC;QAC3E,IAAI,UAAU,GAAG,MAAM,CAAC;QAExB,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtC,UAAU,CAAC,OAAO,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAC,CAAC;YAE5C,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC1C,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;SAC5B;QAED,iDAAiD;QACjD,OAAO,IAAI,EAAE;YACT,kCAAkC;YAClC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;YAExE,mCAAmC;YACnC,IAAI,SAAS,KAAK,YAAY,CAAC,QAAQ,EAAE;gBACrC,MAAM;YAEV,MAAM,SAAS,CAAC;YAEhB,mCAAmC;YACnC,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;SAC9C;IACL,CAAC;CAEJ"}
+{"version":3,"file":"LlamaContext.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaContext.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,gBAAgB,EAAC,MAAM,8BAA8B,CAAC;AAE9D,OAAO,EAAC,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAI5C,MAAM,OAAO,YAAY;IACJ,IAAI,CAAe;IAC5B,WAAW,CAAU;IAE7B,YAAmB,EAAC,KAAK,EAAE,OAAO,EAAE,UAAU,GAAG,IAAI,EAAoE;QACrH,IAAI,CAAC,IAAI,GAAG,IAAI,YAAY,CAAC,KAAK,CAAC,MAAM,EAAE,gBAAgB,CAAC;YACxD,OAAO,EAAE,OAAO,EAAE,QAAQ;SAC7B,CAAC,CAAC,CAAC;QACJ,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,IAAY;QACtB,IAAI,IAAI,KAAK,EAAE;YACX,OAAO,IAAI,WAAW,EAAE,CAAC;QAE7B,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,MAAmB;QAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YACnB,OAAO,EAAE,CAAC;QAEd,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAED,IAAW,UAAU;QACjB,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED;;OAEG;IACI,WAAW;QACd,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;QAEtC,IAAI,QAAQ,KAAK,CAAC,CAAC;YACf,OAAO,IAAI,CAAC;QAEhB,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED;;OAEG;IACI,WAAW;QACd,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;QAEtC,IAAI,QAAQ,KAAK,CAAC,CAAC;YACf,OAAO,IAAI,CAAC;QAEhB,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED;;OAEG;IACI,UAAU;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QAEpC,IAAI,OAAO,KAAK,CAAC,CAAC;YACd,OAAO,IAAI,CAAC;QAEhB,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;OAEG;IACI,YAAY;QACf,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAEpC,IAAI,QAAQ,IAAI,IAAI;YAChB,OAAO,IAAI,CAAC;QAEhB,OAAO,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACI,YAAY;QACf,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAEpC,IAAI,QAAQ,IAAI,IAAI;YAChB,OAAO,IAAI,CAAC;QAEhB,OAAO,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACI,WAAW;QACd,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QAElC,IAAI,OAAO,IAAI,IAAI;YACf,OAAO,IAAI,CAAC;QAEhB,OAAO,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;IAC7C,CAAC;IAEM,cAAc;QACjB,OAAO,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;IACtC,CAAC;IAEM,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAmB;QACtC,IAAI,UAAU,GAAG,MAAM,CAAC;QAExB,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,MAAM,UAAU,GAAY,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC/C,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEzC,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC1C,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;SAC5B;QAED,iDAAiD;QACjD,OAAO,IAAI,EAAE;YACT,kCAAkC;YAClC,MAAM,SAAS,GAAU,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAE1D,mCAAmC;YACnC,IAAI,SAAS,KAAK,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE;gBAClC,MAAM;YAEV,MAAM,SAAS,CAAC;YAEhB,mCAAmC;YACnC,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;SAC9C;IACL,CAAC;CAEJ"}

package/dist/llamaEvaluator/LlamaGrammar.d.ts

@@ -0,0 +1,14 @@
+export declare class LlamaGrammar {
+    /**
+     * GBNF files are supported.
+     * More info here: https://github.com/ggerganov/llama.cpp/blob/f5fe98d11bdf9e7797bcfb05c0c3601ffc4b9d26/grammars/README.md
+     * @param {object} options
+     * @param {string} options.grammar - GBNF grammar
+     * @param {boolean} [options.printGrammar] - print the grammar to stdout
+     */
+    constructor({ grammar, printGrammar }: {
+        grammar: string;
+        printGrammar?: boolean;
+    });
+    static getFor(type: "json" | "list" | "arithmetic" | "japanese" | "chess"): Promise<LlamaGrammar>;
+}

package/dist/llamaEvaluator/LlamaGrammar.js

@@ -0,0 +1,30 @@
+import path from "path";
+import fs from "fs-extra";
+import { getGrammarsFolder } from "../utils/getGrammarsFolder.js";
+import { LLAMAGrammar } from "./LlamaBins.js";
+export class LlamaGrammar {
+    /** @internal */
+    _grammar;
+    /**
+     * GBNF files are supported.
+     * More info here: https://github.com/ggerganov/llama.cpp/blob/f5fe98d11bdf9e7797bcfb05c0c3601ffc4b9d26/grammars/README.md
+     * @param {object} options
+     * @param {string} options.grammar - GBNF grammar
+     * @param {boolean} [options.printGrammar] - print the grammar to stdout
+     */
+    constructor({ grammar, printGrammar = false }) {
+        this._grammar = new LLAMAGrammar(grammar, {
+            printGrammar
+        });
+    }
+    static async getFor(type) {
+        const grammarsFolder = await getGrammarsFolder();
+        const grammarFile = path.join(grammarsFolder, type + ".gbnf");
+        if (await fs.exists(grammarFile)) {
+            const grammar = await fs.readFile(grammarFile, "utf8");
+            return new LlamaGrammar({ grammar });
+        }
+        throw new Error(`Grammar file for type "${type}" was not found in "${grammarsFolder}"`);
+    }
+}
+//# sourceMappingURL=LlamaGrammar.js.map

package/dist/llamaEvaluator/LlamaGrammar.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LlamaGrammar.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaGrammar.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,iBAAiB,EAAC,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAC,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAG5C,MAAM,OAAO,YAAY;IACrB,gBAAgB;IACA,QAAQ,CAAe;IAEvC;;;;;;OAMG;IACH,YAAmB,EAAC,OAAO,EAAE,YAAY,GAAG,KAAK,EAA4C;QACzF,IAAI,CAAC,QAAQ,GAAG,IAAI,YAAY,CAAC,OAAO,EAAE;YACtC,YAAY;SACf,CAAC,CAAC;IACP,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,IAA2D;QAClF,MAAM,cAAc,GAAG,MAAM,iBAAiB,EAAE,CAAC;QAEjD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,GAAG,OAAO,CAAC,CAAC;QAE9D,IAAI,MAAM,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE;YAC9B,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;YACvD,OAAO,IAAI,YAAY,CAAC,EAAC,OAAO,EAAC,CAAC,CAAC;SACtC;QAED,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,uBAAuB,cAAc,GAAG,CAAC,CAAC;IAC5F,CAAC;CACJ"}

package/dist/llamaEvaluator/LlamaModel.d.ts

@@ -9,6 +9,26 @@ export declare class LlamaModel {
      * @param {number} [options.batchSize] - prompt processing batch size
      * @param {number} [options.gpuLayers] - number of layers to store in VRAM
      * @param {boolean} [options.lowVram] - if true, reduce VRAM usage at the cost of performance
+     * @param {number} [options.temperature] - Temperature is a hyperparameter that controls the randomness of the generated text.
+     * It affects the probability distribution of the model's output tokens.
+     * A higher temperature (e.g., 1.5) makes the output more random and creative,
+     * while a lower temperature (e.g., 0.5) makes the output more focused, deterministic, and conservative.
+     * The suggested temperature is 0.8, which provides a balance between randomness and determinism.
+     * At the extreme, a temperature of 0 will always pick the most likely next token, leading to identical outputs in each run.
+     *
+     * Set to `0` to disable.
+     * @param {number} [options.topK] - Limits the model to consider only the K most likely next tokens for sampling at each step of
+     * sequence generation.
+     * An integer number between `1` and the size of the vocabulary.
+     * Set to `0` to disable (which uses the full vocabulary).
+     *
+     * Only relevant when `temperature` is set to a value greater than 0.
+     * @param {number} [options.topP] - Dynamically selects the smallest set of tokens whose cumulative probability exceeds the threshold P,
+     * and samples the next token only from this set.
+     * A float number between `0` and `1`.
+     * Set to `1` to disable.
+     *
+     * Only relevant when `temperature` is set to a value greater than `0`.
      * @param {boolean} [options.f16Kv] - use fp16 for KV cache
      * @param {boolean} [options.logitsAll] - the llama_eval() call computes all logits, not just the last one
      * @param {boolean} [options.vocabOnly] - only load the vocabulary, no weights
@@ -16,7 +36,7 @@ export declare class LlamaModel {
      * @param {boolean} [options.useMlock] - force system to keep model in RAM
      * @param {boolean} [options.embedding] - embedding mode only
      */
-    constructor({ modelPath, seed, contextSize, batchSize, gpuLayers, lowVram, f16Kv, logitsAll, vocabOnly, useMmap, useMlock, embedding }: {
+    constructor({ modelPath, seed, contextSize, batchSize, gpuLayers, lowVram, temperature, topK, topP, f16Kv, logitsAll, vocabOnly, useMmap, useMlock, embedding }: {
         /** path to the model on the filesystem */
         modelPath: string;
         /** If null, a random seed will be used */
@@ -29,6 +49,34 @@ export declare class LlamaModel {
         gpuLayers?: number;
         /** if true, reduce VRAM usage at the cost of performance */
         lowVram?: boolean;
+        /**
+         * Temperature is a hyperparameter that controls the randomness of the generated text.
+         * It affects the probability distribution of the model's output tokens.
+         * A higher temperature (e.g., 1.5) makes the output more random and creative,
+         * while a lower temperature (e.g., 0.5) makes the output more focused, deterministic, and conservative.
+         * The suggested temperature is 0.8, which provides a balance between randomness and determinism.
+         * At the extreme, a temperature of 0 will always pick the most likely next token, leading to identical outputs in each run.
+         *
+         * Set to `0` to disable.
+         */
+        temperature?: number;
+        /**
+         * Limits the model to consider only the K most likely next tokens for sampling at each step of sequence generation.
+         * An integer number between `1` and the size of the vocabulary.
+         * Set to `0` to disable (which uses the full vocabulary).
+         *
+         * Only relevant when `temperature` is set to a value greater than 0.
+         * */
+        topK?: number;
+        /**
+         * Dynamically selects the smallest set of tokens whose cumulative probability exceeds the threshold P,
+         * and samples the next token only from this set.
+         * A float number between `0` and `1`.
+         * Set to `1` to disable.
+         *
+         * Only relevant when `temperature` is set to a value greater than `0`.
+         * */
+        topP?: number;
         /** use fp16 for KV cache */
         f16Kv?: boolean;
         /** the llama_eval() call computes all logits, not just the last one */

package/dist/llamaEvaluator/LlamaModel.js

@@ -1,3 +1,4 @@
+import { removeNullFields } from "../utils/removeNullFields.js";
 import { llamaCppNode, LLAMAModel } from "./LlamaBins.js";
 export class LlamaModel {
     /** @internal */
@@ -12,6 +13,26 @@ export class LlamaModel {
      * @param {number} [options.batchSize] - prompt processing batch size
      * @param {number} [options.gpuLayers] - number of layers to store in VRAM
      * @param {boolean} [options.lowVram] - if true, reduce VRAM usage at the cost of performance
+     * @param {number} [options.temperature] - Temperature is a hyperparameter that controls the randomness of the generated text.
+     * It affects the probability distribution of the model's output tokens.
+     * A higher temperature (e.g., 1.5) makes the output more random and creative,
+     * while a lower temperature (e.g., 0.5) makes the output more focused, deterministic, and conservative.
+     * The suggested temperature is 0.8, which provides a balance between randomness and determinism.
+     * At the extreme, a temperature of 0 will always pick the most likely next token, leading to identical outputs in each run.
+     *
+     * Set to `0` to disable.
+     * @param {number} [options.topK] - Limits the model to consider only the K most likely next tokens for sampling at each step of
+     * sequence generation.
+     * An integer number between `1` and the size of the vocabulary.
+     * Set to `0` to disable (which uses the full vocabulary).
+     *
+     * Only relevant when `temperature` is set to a value greater than 0.
+     * @param {number} [options.topP] - Dynamically selects the smallest set of tokens whose cumulative probability exceeds the threshold P,
+     * and samples the next token only from this set.
+     * A float number between `0` and `1`.
+     * Set to `1` to disable.
+     *
+     * Only relevant when `temperature` is set to a value greater than `0`.
      * @param {boolean} [options.f16Kv] - use fp16 for KV cache
      * @param {boolean} [options.logitsAll] - the llama_eval() call computes all logits, not just the last one
      * @param {boolean} [options.vocabOnly] - only load the vocabulary, no weights
@@ -19,13 +40,16 @@ export class LlamaModel {
      * @param {boolean} [options.useMlock] - force system to keep model in RAM
      * @param {boolean} [options.embedding] - embedding mode only
      */
-    constructor({ modelPath, seed = null, contextSize = 1024 * 4, batchSize, gpuLayers, lowVram, f16Kv, logitsAll, vocabOnly, useMmap, useMlock, embedding }) {
+    constructor({ modelPath, seed = null, contextSize = 1024 * 4, batchSize, gpuLayers, lowVram, temperature = 0, topK = 40, topP = 0.95, f16Kv, logitsAll, vocabOnly, useMmap, useMlock, embedding }) {
         this._model = new LLAMAModel(modelPath, removeNullFields({
             seed: seed != null ? Math.max(-1, seed) : undefined,
             contextSize,
             batchSize,
             gpuLayers,
             lowVram,
+            temperature,
+            topK,
+            topP,
             f16Kv,
             logitsAll,
             vocabOnly,
@@ -38,12 +62,4 @@ export class LlamaModel {
         return llamaCppNode.systemInfo();
     }
 }
-function removeNullFields(obj) {
-    const newObj = Object.assign({}, obj);
-    for (const key in obj) {
-        if (newObj[key] == null)
-            delete newObj[key];
-    }
-    return newObj;
-}
 //# sourceMappingURL=LlamaModel.js.map

package/dist/llamaEvaluator/LlamaModel.js.map

@@ -1 +1 @@
-{"version":3,"file":"LlamaModel.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaModel.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAE,UAAU,EAAC,MAAM,gBAAgB,CAAC;AAGxD,MAAM,OAAO,UAAU;IACnB,gBAAgB;IACA,MAAM,CAAa;IAEnC;;;;;;;;;;;;;;;;OAgBG;IACH,YAAmB,EACf,SAAS,EAAE,IAAI,GAAG,IAAI,EAAE,WAAW,GAAG,IAAI,GAAG,CAAC,EAAE,SAAS,EAAE,SAAS,EACpE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAqCrE;QACG,IAAI,CAAC,MAAM,GAAG,IAAI,UAAU,CAAC,SAAS,EAAE,gBAAgB,CAAC;YACrD,IAAI,EAAE,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS;YACnD,WAAW;YACX,SAAS;YACT,SAAS;YACT,OAAO;YACP,KAAK;YACL,SAAS;YACT,SAAS;YACT,OAAO;YACP,QAAQ;YACR,SAAS;SACZ,CAAC,CAAC,CAAC;IACR,CAAC;IAEM,MAAM,KAAK,UAAU;QACxB,OAAO,YAAY,CAAC,UAAU,EAAE,CAAC;IACrC,CAAC;CACJ;AAED,SAAS,gBAAgB,CAAmB,GAAM;IAC9C,MAAM,MAAM,GAAM,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IAEzC,KAAK,MAAM,GAAG,IAAI,GAAG,EAAE;QACnB,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI;YACnB,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;KAC1B;IAED,OAAO,MAAM,CAAC;AAClB,CAAC"}
+{"version":3,"file":"LlamaModel.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaModel.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,gBAAgB,EAAC,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAC,YAAY,EAAE,UAAU,EAAC,MAAM,gBAAgB,CAAC;AAGxD,MAAM,OAAO,UAAU;IACnB,gBAAgB;IACA,MAAM,CAAa;IAEnC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoCG;IACH,YAAmB,EACf,SAAS,EAAE,IAAI,GAAG,IAAI,EAAE,WAAW,GAAG,IAAI,GAAG,CAAC,EAAE,SAAS,EAAE,SAAS,EACpE,OAAO,EAAE,WAAW,GAAG,CAAC,EAAE,IAAI,GAAG,EAAE,EAAE,IAAI,GAAG,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAoE9G;QACG,IAAI,CAAC,MAAM,GAAG,IAAI,UAAU,CAAC,SAAS,EAAE,gBAAgB,CAAC;YACrD,IAAI,EAAE,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS;YACnD,WAAW;YACX,SAAS;YACT,SAAS;YACT,OAAO;YACP,WAAW;YACX,IAAI;YACJ,IAAI;YACJ,KAAK;YACL,SAAS;YACT,SAAS;YACT,OAAO;YACP,QAAQ;YACR,SAAS;SACZ,CAAC,CAAC,CAAC;IACR,CAAC;IAEM,MAAM,KAAK,UAAU;QACxB,OAAO,YAAY,CAAC,UAAU,EAAE,CAAC;IACrC,CAAC;CACJ"}

package/dist/types.d.ts

@@ -0,0 +1 @@
+export type Token = number;

package/dist/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}

package/dist/utils/binariesGithubRelease.d.ts

@@ -0,0 +1,6 @@
+type BinariesGithubReleaseFile = {
+    release: "latest" | string;
+};
+export declare function getBinariesGithubRelease(): Promise<string>;
+export declare function setBinariesGithubRelease(release: BinariesGithubReleaseFile["release"]): Promise<void>;
+export {};

package/dist/utils/binariesGithubRelease.js

@@ -0,0 +1,15 @@
+import fs from "fs-extra";
+import { binariesGithubReleasePath } from "../config.js";
+export async function getBinariesGithubRelease() {
+    const binariesGithubRelease = await fs.readJson(binariesGithubReleasePath);
+    return binariesGithubRelease.release;
+}
+export async function setBinariesGithubRelease(release) {
+    const binariesGithubReleaseJson = {
+        release: release
+    };
+    await fs.writeJson(binariesGithubReleasePath, binariesGithubReleaseJson, {
+        spaces: 4
+    });
+}
+//# sourceMappingURL=binariesGithubRelease.js.map

package/dist/utils/binariesGithubRelease.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"binariesGithubRelease.js","sourceRoot":"","sources":["../../src/utils/binariesGithubRelease.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,yBAAyB,EAAC,MAAM,cAAc,CAAC;AAMvD,MAAM,CAAC,KAAK,UAAU,wBAAwB;IAC1C,MAAM,qBAAqB,GAA8B,MAAM,EAAE,CAAC,QAAQ,CAAC,yBAAyB,CAAC,CAAC;IAEtG,OAAO,qBAAqB,CAAC,OAAO,CAAC;AACzC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,OAA6C;IACxF,MAAM,yBAAyB,GAA8B;QACzD,OAAO,EAAE,OAAO;KACnB,CAAC;IAEF,MAAM,EAAE,CAAC,SAAS,CAAC,yBAAyB,EAAE,yBAAyB,EAAE;QACrE,MAAM,EAAE,CAAC;KACZ,CAAC,CAAC;AACP,CAAC"}

package/dist/utils/compileLLamaCpp.d.ts

@@ -1,6 +1,8 @@
-export declare function compileLlamaCpp({ arch, nodeTarget, setUsedBingFlag }: {
+export declare function compileLlamaCpp({ arch, nodeTarget, setUsedBingFlag, metal, cuda }: {
     arch?: string;
     nodeTarget?: string;
     setUsedBingFlag?: boolean;
+    metal?: boolean;
+    cuda?: boolean;
 }): Promise<void>;
 export declare function getCompiledLlamaCppBinaryPath(): Promise<string | null>;