node-llama-cpp 3.0.0-beta.1 → 3.0.0-beta.10

package/dist/evaluator/LlamaChatSession/LlamaChatSession.js

@@ -0,0 +1,219 @@
+import { DisposeAggregator, DisposedError, EventRelay, withLock } from "lifecycle-utils";
+import { defaultChatSystemPrompt } from "../../config.js";
+import { appendUserMessageToChatHistory } from "../../utils/appendUserMessageToChatHistory.js";
+import { LlamaChat } from "../LlamaChat/LlamaChat.js";
+export class LlamaChatSession {
+    /** @internal */ _disposeAggregator = new DisposeAggregator();
+    /** @internal */ _autoDisposeSequence;
+    /** @internal */ _contextShift;
+    /** @internal */ _chatHistory;
+    /** @internal */ _lastEvaluation;
+    /** @internal */ _chat;
+    onDispose = new EventRelay();
+    /**
+     * @param options
+     */
+    constructor({ contextSequence, chatWrapper = "auto", systemPrompt = defaultChatSystemPrompt, autoDisposeSequence = true, contextShift }) {
+        if (contextSequence == null)
+            throw new Error("contextSequence cannot be null");
+        if (contextSequence.disposed)
+            throw new DisposedError();
+        this._contextShift = contextShift;
+        this._chatHistory = [{
+                type: "system",
+                text: systemPrompt
+            }];
+        this._chat = new LlamaChat({
+            autoDisposeSequence,
+            chatWrapper,
+            contextSequence
+        });
+        this._autoDisposeSequence = autoDisposeSequence;
+        this._disposeAggregator.add(this._chat.onDispose.createListener(() => {
+            this.dispose();
+        }));
+        this._disposeAggregator.add(this.onDispose.dispatchEvent);
+    }
+    dispose({ disposeSequence = this._autoDisposeSequence } = {}) {
+        if (this._chat == null)
+            return;
+        this._chat.dispose({ disposeSequence });
+        this._chat = null;
+        this._disposeAggregator.dispose();
+    }
+    /** @hidden */
+    [Symbol.dispose]() {
+        return this.dispose();
+    }
+    get disposed() {
+        return this._chat == null || this._chat.disposed;
+    }
+    get chatWrapper() {
+        if (this._chat == null)
+            throw new DisposedError();
+        return this._chat.chatWrapper;
+    }
+    get sequence() {
+        if (this._chat == null)
+            throw new DisposedError();
+        return this._chat.sequence;
+    }
+    get context() {
+        return this.sequence.context;
+    }
+    get model() {
+        return this.sequence.model;
+    }
+    /**
+     * @param prompt
+     * @param [options]
+     */
+    async prompt(prompt, { functions, documentFunctionParams, onToken, signal, maxTokens, temperature, minP, topK, topP, grammar, trimWhitespaceSuffix = false, repeatPenalty } = {}) {
+        const { responseText } = await this.promptWithMeta(prompt, {
+            // this is a workaround to allow passing both `functions` and `grammar`
+            functions: functions,
+            documentFunctionParams: documentFunctionParams,
+            onToken, signal, maxTokens, temperature, minP, topK, topP, grammar, trimWhitespaceSuffix, repeatPenalty
+        });
+        return responseText;
+    }
+    /**
+     * @param prompt
+     * @param [options]
+     */
+    async promptWithMeta(prompt, { functions, documentFunctionParams, onToken, signal, maxTokens, temperature, minP, topK, topP, grammar, trimWhitespaceSuffix = false, repeatPenalty, evaluationPriority } = {}) {
+        this._ensureNotDisposed();
+        if (grammar != null && grammar._llama !== this.model._llama)
+            throw new Error("The LlamaGrammar used by passed to this function was created with a different Llama instance than the one used by this sequence's model. Make sure you use the same Llama instance for both the model and the grammar.");
+        return await withLock(this, "prompt", async () => {
+            this._ensureNotDisposed();
+            if (this._chat == null)
+                throw new DisposedError();
+            let lastEvaluation = this._lastEvaluation;
+            let newChatHistory = appendUserMessageToChatHistory(this._chatHistory, prompt);
+            let newContextWindowChatHistory = lastEvaluation?.contextWindow == null
+                ? undefined
+                : appendUserMessageToChatHistory(lastEvaluation?.contextWindow, prompt);
+            newChatHistory.push({
+                type: "model",
+                response: []
+            });
+            if (newContextWindowChatHistory != null)
+                newContextWindowChatHistory.push({
+                    type: "model",
+                    response: []
+                });
+            // eslint-disable-next-line no-constant-condition
+            while (true) {
+                const { functionCall, lastEvaluation: currentLastEvaluation, metadata } = await this._chat.generateResponse(newChatHistory, {
+                    functions,
+                    documentFunctionParams,
+                    grammar: grammar,
+                    onToken,
+                    signal,
+                    repeatPenalty,
+                    minP,
+                    topK,
+                    topP,
+                    maxTokens,
+                    temperature,
+                    trimWhitespaceSuffix,
+                    contextShift: {
+                        ...this._contextShift,
+                        lastEvaluationMetadata: lastEvaluation?.contextShiftMetadata
+                    },
+                    evaluationPriority,
+                    lastEvaluationContextWindow: {
+                        history: newContextWindowChatHistory,
+                        minimumOverlapPercentageToPreventContextShift: 0.01
+                    }
+                });
+                this._ensureNotDisposed();
+                lastEvaluation = currentLastEvaluation;
+                newChatHistory = lastEvaluation.cleanHistory;
+                if (functionCall != null) {
+                    const functionDefinition = functions?.[functionCall.functionName];
+                    if (functionDefinition == null)
+                        throw new Error(`The model tried to call function "${functionCall.functionName}" which is not defined`);
+                    const functionCallResult = await functionDefinition.handler(functionCall.params);
+                    this._ensureNotDisposed();
+                    newChatHistory = addFunctionCallToChatHistory({
+                        chatHistory: newChatHistory,
+                        functionName: functionCall.functionName,
+                        functionDescription: functionDefinition.description,
+                        callParams: functionCall.params,
+                        callResult: functionCallResult,
+                        raw: functionCall.raw + this._chat.chatWrapper.generateFunctionCallResult(functionCall.functionName, functionCall.params, functionCallResult)
+                    });
+                    newContextWindowChatHistory = addFunctionCallToChatHistory({
+                        chatHistory: lastEvaluation.contextWindow,
+                        functionName: functionCall.functionName,
+                        functionDescription: functionDefinition.description,
+                        callParams: functionCall.params,
+                        callResult: functionCallResult,
+                        raw: functionCall.raw + this._chat.chatWrapper.generateFunctionCallResult(functionCall.functionName, functionCall.params, functionCallResult)
+                    });
+                    lastEvaluation.cleanHistory = newChatHistory;
+                    lastEvaluation.contextWindow = newContextWindowChatHistory;
+                    continue;
+                }
+                this._lastEvaluation = lastEvaluation;
+                this._chatHistory = newChatHistory;
+                const lastModelResponseItem = getLastModelResponseItem(newChatHistory);
+                return {
+                    response: lastModelResponseItem.response,
+                    responseText: lastModelResponseItem.response
+                        .filter((item) => typeof item === "string")
+                        .join(""),
+                    stopReason: metadata.stopReason,
+                    remainingGenerationAfterStop: metadata.remainingGenerationAfterStop
+                };
+            }
+        });
+    }
+    getChatHistory() {
+        return structuredClone(this._chatHistory);
+    }
+    getLastEvaluationContextWindow() {
+        if (this._lastEvaluation == null)
+            return null;
+        return structuredClone(this._lastEvaluation?.contextWindow);
+    }
+    setChatHistory(chatHistory) {
+        this._chatHistory = structuredClone(chatHistory);
+        this._lastEvaluation = undefined;
+    }
+    /** @internal */
+    _ensureNotDisposed() {
+        if (this.disposed)
+            throw new DisposedError();
+    }
+}
+function addFunctionCallToChatHistory({ chatHistory, functionName, functionDescription, callParams, callResult, raw }) {
+    const newChatHistory = chatHistory.slice();
+    if (newChatHistory.length === 0 || newChatHistory[newChatHistory.length - 1].type !== "model")
+        newChatHistory.push({
+            type: "model",
+            response: []
+        });
+    const lastModelResponseItem = newChatHistory[newChatHistory.length - 1];
+    const newLastModelResponseItem = { ...lastModelResponseItem };
+    newChatHistory[newChatHistory.length - 1] = newLastModelResponseItem;
+    const modelResponse = newLastModelResponseItem.response.slice();
+    newLastModelResponseItem.response = modelResponse;
+    modelResponse.push({
+        type: "functionCall",
+        name: functionName,
+        description: functionDescription,
+        params: callParams,
+        result: callResult,
+        raw
+    });
+    return newChatHistory;
+}
+function getLastModelResponseItem(chatHistory) {
+    if (chatHistory.length === 0 || chatHistory[chatHistory.length - 1].type !== "model")
+        throw new Error("Expected chat history to end with a model response");
+    return chatHistory[chatHistory.length - 1];
+}
+//# sourceMappingURL=LlamaChatSession.js.map

package/dist/evaluator/LlamaChatSession/LlamaChatSession.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LlamaChatSession.js","sourceRoot":"","sources":["../../../src/evaluator/LlamaChatSession/LlamaChatSession.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,iBAAiB,EAAE,aAAa,EAAE,UAAU,EAAE,QAAQ,EAAC,MAAM,iBAAiB,CAAC;AACvF,OAAO,EAAC,uBAAuB,EAAC,MAAM,iBAAiB,CAAC;AAGxD,OAAO,EAAC,8BAA8B,EAAC,MAAM,+CAA+C,CAAC;AAG7F,OAAO,EAAC,SAAS,EAAkD,MAAM,2BAA2B,CAAC;AA2IrG,MAAM,OAAO,gBAAgB;IACzB,gBAAgB,CAAkB,kBAAkB,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC/E,gBAAgB,CAAkB,oBAAoB,CAAU;IAChE,gBAAgB,CAAkB,aAAa,CAAuC;IACtF,gBAAgB,CAAS,YAAY,CAAoB;IACzD,gBAAgB,CAAS,eAAe,CAAuC;IAC/E,gBAAgB,CAAS,KAAK,CAAmB;IAEjC,SAAS,GAAG,IAAI,UAAU,EAAQ,CAAC;IAEnD;;OAEG;IACH,YAAmB,EACf,eAAe,EACf,WAAW,GAAG,MAAM,EACpB,YAAY,GAAG,uBAAuB,EACtC,mBAAmB,GAAG,IAAI,EAC1B,YAAY,EACU;QACtB,IAAI,eAAe,IAAI,IAAI;YACvB,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAEtD,IAAI,eAAe,CAAC,QAAQ;YACxB,MAAM,IAAI,aAAa,EAAE,CAAC;QAE9B,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;QAClC,IAAI,CAAC,YAAY,GAAG,CAAC;gBACjB,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,YAAY;aACrB,CAAC,CAAC;QAEH,IAAI,CAAC,KAAK,GAAG,IAAI,SAAS,CAAC;YACvB,mBAAmB;YACnB,WAAW;YACX,eAAe;SAClB,CAAC,CAAC;QAEH,IAAI,CAAC,oBAAoB,GAAG,mBAAmB,CAAC;QAEhD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CACvB,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,EAAE;YACrC,IAAI,CAAC,OAAO,EAAE,CAAC;QACnB,CAAC,CAAC,CACL,CAAC;QACF,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;IAC9D,CAAC;IAEM,OAAO,CAAC,EAAC,eAAe,GAAG,IAAI,CAAC,oBAAoB,KAAiC,EAAE;QAC1F,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI;YAClB,OAAO;QAEX,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAC,eAAe,EAAC,CAAC,CAAC;QACtC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAElB,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,CAAC;IACtC,CAAC;IAED,cAAc;IACP,CAAC,MAAM,CAAC,OAAO,CAAC;QACnB,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;IAC1B,CAAC;IAED,IAAW,QAAQ;QACf,OAAO,IAAI,CAAC,KAAK,IAAI,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;IACrD,CAAC;IAED,IAAW,WAAW;QAClB,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI;YAClB,MAAM,IAAI,aAAa,EAAE,CAAC;QAE9B,OAAO,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;IAClC,CAAC;IAED,IAAW,QAAQ;QACf,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI;YAClB,MAAM,IAAI,aAAa,EAAE,CAAC;QAE9B,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;IAC/B,CAAC;IAED,IAAW,OAAO;QACd,OAAO,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;IACjC,CAAC;IAED,IAAW,KAAK;QACZ,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IAC/B,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,MAAM,CAA4E,MAAc,EAAE,EAC3G,SAAS,EACT,sBAAsB,EACtB,OAAO,EACP,MAAM,EACN,SAAS,EACT,WAAW,EACX,IAAI,EACJ,IAAI,EACJ,IAAI,EACJ,OAAO,EACP,oBAAoB,GAAG,KAAK,EAC5B,aAAa,KACsB,EAAE;QACrC,MAAM,EAAC,YAAY,EAAC,GAAG,MAAM,IAAI,CAAC,cAAc,CAAY,MAAM,EAAE;YAChE,uEAAuE;YACvE,SAAS,EAAE,SAAsB;YACjC,sBAAsB,EAAE,sBAAmC;YAE3D,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,oBAAoB,EAAE,aAAa;SAC1G,CAAC,CAAC;QAEH,OAAO,YAAY,CAAC;IACxB,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,cAAc,CAA4E,MAAc,EAAE,EACnH,SAAS,EACT,sBAAsB,EACtB,OAAO,EACP,MAAM,EACN,SAAS,EACT,WAAW,EACX,IAAI,EACJ,IAAI,EACJ,IAAI,EACJ,OAAO,EACP,oBAAoB,GAAG,KAAK,EAC5B,aAAa,EACb,kBAAkB,KACiB,EAAE;QACrC,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE1B,IAAI,OAAO,IAAI,IAAI,IAAI,OAAO,CAAC,MAAM,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM;YACvD,MAAM,IAAI,KAAK,CAAC,wNAAwN,CAAC,CAAC;QAE9O,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,IAAI,EAAE;YAC7C,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAE1B,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI;gBAClB,MAAM,IAAI,aAAa,EAAE,CAAC;YAE9B,IAAI,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC;YAC1C,IAAI,cAAc,GAAG,8BAA8B,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;YAC/E,IAAI,2BAA2B,GAAG,cAAc,EAAE,aAAa,IAAI,IAAI;gBACnE,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,8BAA8B,CAAC,cAAc,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC;YAE5E,cAAc,CAAC,IAAI,CAAC;gBAChB,IAAI,EAAE,OAAO;gBACb,QAAQ,EAAE,EAAE;aACf,CAAC,CAAC;YAEH,IAAI,2BAA2B,IAAI,IAAI;gBACnC,2BAA2B,CAAC,IAAI,CAAC;oBAC7B,IAAI,EAAE,OAAO;oBACb,QAAQ,EAAE,EAAE;iBACf,CAAC,CAAC;YAEP,iDAAiD;YACjD,OAAO,IAAI,EAAE;gBACT,MAAM,EACF,YAAY,EACZ,cAAc,EAAE,qBAAqB,EACrC,QAAQ,EACX,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAY,cAAc,EAAE;oBAC7D,SAAS;oBACT,sBAAsB;oBACtB,OAAO,EAAE,OAAoB;oBAC7B,OAAO;oBACP,MAAM;oBACN,aAAa;oBACb,IAAI;oBACJ,IAAI;oBACJ,IAAI;oBACJ,SAAS;oBACT,WAAW;oBACX,oBAAoB;oBACpB,YAAY,EAAE;wBACV,GAAG,IAAI,CAAC,aAAa;wBACrB,sBAAsB,EAAE,cAAc,EAAE,oBAAoB;qBAC/D;oBACD,kBAAkB;oBAClB,2BAA2B,EAAE;wBACzB,OAAO,EAAE,2BAA2B;wBACpC,6CAA6C,EAAE,IAAI;qBACtD;iBACJ,CAAC,CAAC;gBACH,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBAE1B,cAAc,GAAG,qBAAqB,CAAC;gBACvC,cAAc,GAAG,cAAc,CAAC,YAAY,CAAC;gBAE7C,IAAI,YAAY,IAAI,IAAI,EAAE;oBACtB,MAAM,kBAAkB,GAAG,SAAS,EAAE,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;oBAElE,IAAI,kBAAkB,IAAI,IAAI;wBAC1B,MAAM,IAAI,KAAK,CAAC,qCAAqC,YAAY,CAAC,YAAY,wBAAwB,CAAC,CAAC;oBAE5G,MAAM,kBAAkB,GAAG,MAAM,kBAAkB,CAAC,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;oBACjF,IAAI,CAAC,kBAAkB,EAAE,CAAC;oBAE1B,cAAc,GAAG,4BAA4B,CAAC;wBAC1C,WAAW,EAAE,cAAc;wBAC3B,YAAY,EAAE,YAAY,CAAC,YAAY;wBACvC,mBAAmB,EAAE,kBAAkB,CAAC,WAAW;wBACnD,UAAU,EAAE,YAAY,CAAC,MAAM;wBAC/B,UAAU,EAAE,kBAAkB;wBAC9B,GAAG,EAAE,YAAY,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,0BAA0B,CACrE,YAAY,CAAC,YAAY,EACzB,YAAY,CAAC,MAAM,EACnB,kBAAkB,CACrB;qBACJ,CAAC,CAAC;oBAEH,2BAA2B,GAAG,4BAA4B,CAAC;wBACvD,WAAW,EAAE,cAAc,CAAC,aAAa;wBACzC,YAAY,EAAE,YAAY,CAAC,YAAY;wBACvC,mBAAmB,EAAE,kBAAkB,CAAC,WAAW;wBACnD,UAAU,EAAE,YAAY,CAAC,MAAM;wBAC/B,UAAU,EAAE,kBAAkB;wBAC9B,GAAG,EAAE,YAAY,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,0BAA0B,CACrE,YAAY,CAAC,YAAY,EACzB,YAAY,CAAC,MAAM,EACnB,kBAAkB,CACrB;qBACJ,CAAC,CAAC;oBACH,cAAc,CAAC,YAAY,GAAG,cAAc,CAAC;oBAC7C,cAAc,CAAC,aAAa,GAAG,2BAA2B,CAAC;oBAE3D,SAAS;iBACZ;gBAED,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;gBACtC,IAAI,CAAC,YAAY,GAAG,cAAc,CAAC;gBAEnC,MAAM,qBAAqB,GAAG,wBAAwB,CAAC,cAAc,CAAC,CAAC;gBAEvE,OAAO;oBACH,QAAQ,EAAE,qBAAqB,CAAC,QAAQ;oBACxC,YAAY,EAAE,qBAAqB,CAAC,QAAQ;yBACvC,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC;yBAC1D,IAAI,CAAC,EAAE,CAAC;oBACb,UAAU,EAAE,QAAQ,CAAC,UAAU;oBAC/B,4BAA4B,EAAE,QAAQ,CAAC,4BAA4B;iBACtE,CAAC;aACL;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,cAAc;QACjB,OAAO,eAAe,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC9C,CAAC;IAEM,8BAA8B;QACjC,IAAI,IAAI,CAAC,eAAe,IAAI,IAAI;YAC5B,OAAO,IAAI,CAAC;QAEhB,OAAO,eAAe,CAAC,IAAI,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;IAChE,CAAC;IAEM,cAAc,CAAC,WAA8B;QAChD,IAAI,CAAC,YAAY,GAAG,eAAe,CAAC,WAAW,CAAC,CAAC;QACjD,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;IACrC,CAAC;IAED,gBAAgB;IACR,kBAAkB;QACtB,IAAI,IAAI,CAAC,QAAQ;YACb,MAAM,IAAI,aAAa,EAAE,CAAC;IAClC,CAAC;CACJ;AAED,SAAS,4BAA4B,CAAC,EAClC,WAAW,EACX,YAAY,EACZ,mBAAmB,EACnB,UAAU,EACV,UAAU,EACV,GAAG,EAQN;IACG,MAAM,cAAc,GAAG,WAAW,CAAC,KAAK,EAAE,CAAC;IAC3C,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,IAAI,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO;QACzF,cAAc,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,OAAO;YACb,QAAQ,EAAE,EAAE;SACf,CAAC,CAAC;IAEP,MAAM,qBAAqB,GAAG,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAsB,CAAC;IAC7F,MAAM,wBAAwB,GAAG,EAAC,GAAG,qBAAqB,EAAC,CAAC;IAC5D,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,wBAAwB,CAAC;IAErE,MAAM,aAAa,GAAG,wBAAwB,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IAChE,wBAAwB,CAAC,QAAQ,GAAG,aAAa,CAAC;IAElD,aAAa,CAAC,IAAI,CAAC;QACf,IAAI,EAAE,cAAc;QACpB,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,mBAAmB;QAChC,MAAM,EAAE,UAAU;QAClB,MAAM,EAAE,UAAU;QAClB,GAAG;KACN,CAAC,CAAC;IAEH,OAAO,cAAc,CAAC;AAC1B,CAAC;AAED,SAAS,wBAAwB,CAAC,WAA8B;IAC5D,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,IAAI,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO;QAChF,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IAE1E,OAAO,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAsB,CAAC;AACpE,CAAC"}

package/dist/evaluator/LlamaChatSession/utils/defineChatSessionFunction.d.ts

@@ -0,0 +1,7 @@
+import { GbnfJsonSchema, GbnfJsonSchemaToType } from "../../../utils/gbnfJson/types.js";
+import { ChatSessionModelFunction } from "../../../types.js";
+export declare function defineChatSessionFunction<const Params extends GbnfJsonSchema | undefined>({ description, params, handler }: {
+    description?: string;
+    params?: Params;
+    handler: (params: GbnfJsonSchemaToType<Params>) => any;
+}): ChatSessionModelFunction<Params>;

package/dist/evaluator/LlamaChatSession/utils/defineChatSessionFunction.js

@@ -0,0 +1,8 @@
+export function defineChatSessionFunction({ description, params, handler }) {
+    return {
+        description,
+        params,
+        handler
+    };
+}
+//# sourceMappingURL=defineChatSessionFunction.js.map

package/dist/evaluator/LlamaChatSession/utils/defineChatSessionFunction.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"defineChatSessionFunction.js","sourceRoot":"","sources":["../../../../src/evaluator/LlamaChatSession/utils/defineChatSessionFunction.ts"],"names":[],"mappings":"AAGA,MAAM,UAAU,yBAAyB,CAAkD,EACvF,WAAW,EACX,MAAM,EACN,OAAO,EAKV;IACG,OAAO;QACH,WAAW;QACX,MAAM;QACN,OAAO;KACV,CAAC;AACN,CAAC"}

package/dist/{llamaEvaluator → evaluator}/LlamaContext/LlamaContext.d.ts

@@ -2,13 +2,13 @@ import { EventRelay } from "lifecycle-utils";
 import { Token } from "../../types.js";
 import { LlamaModel } from "../LlamaModel.js";
 import { LlamaGrammarEvaluationState } from "../LlamaGrammarEvaluationState.js";
-import { ContextShiftOptions, ContextTokensDeleteRange, EvaluationPriority, LlamaContextOptions, LlamaContextSequenceRepeatPenalty, TokenPriority } from "./types.js";
+import { ContextShiftOptions, ContextTokensDeleteRange, EvaluationPriority, LlamaContextOptions, LlamaContextSequenceRepeatPenalty } from "./types.js";
 export declare class LlamaContext {
     readonly onDispose: EventRelay<void>;
     /**
      * @param options
      */
-    constructor({ model, sequences, seed, contextSize, batchSize, f16Kv, logitsAll, embedding, threads, batching: { dispatchSchedule: batchingDispatchSchedule, itemsPrioritizingStrategy: batchingItemsPrioritizingStrategy } }: LlamaContextOptions);
+    constructor({ model, sequences, seed, contextSize, batchSize, threads, batching: { dispatchSchedule: batchingDispatchSchedule, itemsPrioritizingStrategy: batchingItemsPrioritizingStrategy }, _embedding, _noSeed }: LlamaContextOptions);
     dispose(): void;
     /** @hidden */
     [Symbol.dispose](): void;
@@ -24,11 +24,11 @@ export declare class LlamaContext {
      * When there are no sequences left, this method will throw an error.
      * @param [options]
      */
-    getSequence({ prependBos, contextShift: { size: contextShiftSize, strategy: contextShiftStrategy } }?: {
-        prependBos?: boolean;
+    getSequence({ contextShift: { size: contextShiftSize, strategy: contextShiftStrategy } }?: {
         contextShift?: ContextShiftOptions;
     }): LlamaContextSequence;
     dispatchPendingBatch(): void;
+    printTimings(): Promise<void>;
 }
 export declare class LlamaContextSequence {
     readonly onDispose: EventRelay<void>;
@@ -39,10 +39,12 @@ export declare class LlamaContextSequence {
     get disposed(): boolean;
     get context(): LlamaContext;
     get model(): LlamaModel;
-    get prependBos(): boolean;
     get nextTokenIndex(): number;
     get contextTokens(): Token[];
-    get contextTokenPriorities(): number[];
+    get isLoadedToMemory(): boolean;
+    compareContextTokens(tokens: Token[]): {
+        firstDifferentIndex: number;
+    };
     /**
      * Clear the history of the sequence.
      * If `prependBos` was enabled, the BOS token will be prepended to the sequence again.
@@ -58,11 +60,12 @@ export declare class LlamaContextSequence {
      * @param tokens
      * @param [options]
      */
-    evaluate(tokens: Token[], { temperature, topK, topP, grammarEvaluationState, repeatPenalty, evaluationPriority, tokenPriority }?: {
+    evaluate(tokens: Token[], { temperature, minP, topK, topP, grammarEvaluationState, repeatPenalty, evaluationPriority, contextShift: { size: contextShiftSize, strategy: contextShiftStrategy }, yieldEosToken }?: {
         temperature?: number;
+        minP?: number;
         topK?: number;
         topP?: number;
-        grammarEvaluationState?: LlamaGrammarEvaluationState;
+        grammarEvaluationState?: LlamaGrammarEvaluationState | (() => LlamaGrammarEvaluationState | undefined);
         repeatPenalty?: LlamaContextSequenceRepeatPenalty;
         /**
          * When a lot of tokens are queued for the next batch, more than the configured `batchSize`, the tokens for each sequence will be
@@ -74,21 +77,21 @@ export declare class LlamaContextSequence {
          * is, the more likely and more tokens will be evaluated for that sequence in the next queued batch.
          */
         evaluationPriority?: EvaluationPriority;
+        /** Override the sequence context shift options for this evaluation */
+        contextShift?: ContextShiftOptions;
         /**
-         * When the context is full, tokens will be erased based on the context shift strategy chosen.
-         * By default, the lowest priority tokens at the beginning of the context will be erased.
-         * To mark the priority of the evaluated tokens, use this option.
-         * The higher the priority is, the less likely it will be erased.
-         * The default priority is `1`.
+         * Yield the EOS token when it's generated.
+         * When `false` the generation will stop when the EOS token is generated and the EOS token won't be yielded.
+         * Defaults to `false`.
          */
-        tokenPriority?: TokenPriority | TokenPriority[];
+        yieldEosToken?: boolean;
     }): AsyncGenerator<Token, void>;
     /**
      * Evaluate the provided tokens into the context sequence without generating new tokens.
      * @param tokens
      * @param [options]
      */
-    evaluateWithoutGeneratingNewTokens(tokens: Token[], { evaluationPriority, tokenPriority }?: {
+    evaluateWithoutGeneratingNewTokens(tokens: Token[], { evaluationPriority, contextShift: { size: contextShiftSize, strategy: contextShiftStrategy } }?: {
         grammarEvaluationState?: LlamaGrammarEvaluationState;
         /**
          * When a lot of tokens are queued for the next batch, more than the configured `batchSize`, the tokens for each sequence will be
@@ -100,13 +103,7 @@ export declare class LlamaContextSequence {
          * is, the more likely and more tokens will be evaluated for that sequence in the next queued batch.
          */
         evaluationPriority?: EvaluationPriority;
-        /**
-         * When the context is full, tokens will be erased based on the context shift strategy chosen.
-         * By default, the lowest priority tokens at the beginning of the context will be erased.
-         * To mark the priority of the evaluated tokens, use this option.
-         * The higher the priority is, the less likely it will be erased.
-         * The default priority is `1`.
-         */
-        tokenPriority?: TokenPriority | TokenPriority[];
+        /** Override the sequence context shift options for this evaluation */
+        contextShift?: ContextShiftOptions;
     }): Promise<void>;
 }