node-llama-cpp 3.0.0-beta.1 → 3.0.0-beta.11

package/dist/evaluator/LlamaChatSession/LlamaChatSession.js

@@ -0,0 +1,219 @@
+import { DisposeAggregator, DisposedError, EventRelay, withLock } from "lifecycle-utils";
+import { defaultChatSystemPrompt } from "../../config.js";
+import { appendUserMessageToChatHistory } from "../../utils/appendUserMessageToChatHistory.js";
+import { LlamaChat } from "../LlamaChat/LlamaChat.js";
+export class LlamaChatSession {
+    /** @internal */ _disposeAggregator = new DisposeAggregator();
+    /** @internal */ _autoDisposeSequence;
+    /** @internal */ _contextShift;
+    /** @internal */ _chatHistory;
+    /** @internal */ _lastEvaluation;
+    /** @internal */ _chat;
+    onDispose = new EventRelay();
+    /**
+     * @param options
+     */
+    constructor({ contextSequence, chatWrapper = "auto", systemPrompt = defaultChatSystemPrompt, autoDisposeSequence = true, contextShift }) {
+        if (contextSequence == null)
+            throw new Error("contextSequence cannot be null");
+        if (contextSequence.disposed)
+            throw new DisposedError();
+        this._contextShift = contextShift;
+        this._chatHistory = [{
+                type: "system",
+                text: systemPrompt
+            }];
+        this._chat = new LlamaChat({
+            autoDisposeSequence,
+            chatWrapper,
+            contextSequence
+        });
+        this._autoDisposeSequence = autoDisposeSequence;
+        this._disposeAggregator.add(this._chat.onDispose.createListener(() => {
+            this.dispose();
+        }));
+        this._disposeAggregator.add(this.onDispose.dispatchEvent);
+    }
+    dispose({ disposeSequence = this._autoDisposeSequence } = {}) {
+        if (this._chat == null)
+            return;
+        this._chat.dispose({ disposeSequence });
+        this._chat = null;
+        this._disposeAggregator.dispose();
+    }
+    /** @hidden */
+    [Symbol.dispose]() {
+        return this.dispose();
+    }
+    get disposed() {
+        return this._chat == null || this._chat.disposed;
+    }
+    get chatWrapper() {
+        if (this._chat == null)
+            throw new DisposedError();
+        return this._chat.chatWrapper;
+    }
+    get sequence() {
+        if (this._chat == null)
+            throw new DisposedError();
+        return this._chat.sequence;
+    }
+    get context() {
+        return this.sequence.context;
+    }
+    get model() {
+        return this.sequence.model;
+    }
+    /**
+     * @param prompt
+     * @param [options]
+     */
+    async prompt(prompt, { functions, documentFunctionParams, onToken, signal, maxTokens, temperature, minP, topK, topP, grammar, trimWhitespaceSuffix = false, repeatPenalty } = {}) {
+        const { responseText } = await this.promptWithMeta(prompt, {
+            // this is a workaround to allow passing both `functions` and `grammar`
+            functions: functions,
+            documentFunctionParams: documentFunctionParams,
+            onToken, signal, maxTokens, temperature, minP, topK, topP, grammar, trimWhitespaceSuffix, repeatPenalty
+        });
+        return responseText;
+    }
+    /**
+     * @param prompt
+     * @param [options]
+     */
+    async promptWithMeta(prompt, { functions, documentFunctionParams, onToken, signal, maxTokens, temperature, minP, topK, topP, grammar, trimWhitespaceSuffix = false, repeatPenalty, evaluationPriority } = {}) {
+        this._ensureNotDisposed();
+        if (grammar != null && grammar._llama !== this.model._llama)
+            throw new Error("The LlamaGrammar used by passed to this function was created with a different Llama instance than the one used by this sequence's model. Make sure you use the same Llama instance for both the model and the grammar.");
+        return await withLock(this, "prompt", signal, async () => {
+            this._ensureNotDisposed();
+            if (this._chat == null)
+                throw new DisposedError();
+            let lastEvaluation = this._lastEvaluation;
+            let newChatHistory = appendUserMessageToChatHistory(this._chatHistory, prompt);
+            let newContextWindowChatHistory = lastEvaluation?.contextWindow == null
+                ? undefined
+                : appendUserMessageToChatHistory(lastEvaluation?.contextWindow, prompt);
+            newChatHistory.push({
+                type: "model",
+                response: []
+            });
+            if (newContextWindowChatHistory != null)
+                newContextWindowChatHistory.push({
+                    type: "model",
+                    response: []
+                });
+            // eslint-disable-next-line no-constant-condition
+            while (true) {
+                const { functionCall, lastEvaluation: currentLastEvaluation, metadata } = await this._chat.generateResponse(newChatHistory, {
+                    functions,
+                    documentFunctionParams,
+                    grammar: grammar,
+                    onToken,
+                    signal,
+                    repeatPenalty,
+                    minP,
+                    topK,
+                    topP,
+                    maxTokens,
+                    temperature,
+                    trimWhitespaceSuffix,
+                    contextShift: {
+                        ...this._contextShift,
+                        lastEvaluationMetadata: lastEvaluation?.contextShiftMetadata
+                    },
+                    evaluationPriority,
+                    lastEvaluationContextWindow: {
+                        history: newContextWindowChatHistory,
+                        minimumOverlapPercentageToPreventContextShift: 0.01
+                    }
+                });
+                this._ensureNotDisposed();
+                lastEvaluation = currentLastEvaluation;
+                newChatHistory = lastEvaluation.cleanHistory;
+                if (functionCall != null) {
+                    const functionDefinition = functions?.[functionCall.functionName];
+                    if (functionDefinition == null)
+                        throw new Error(`The model tried to call function "${functionCall.functionName}" which is not defined`);
+                    const functionCallResult = await functionDefinition.handler(functionCall.params);
+                    this._ensureNotDisposed();
+                    newChatHistory = addFunctionCallToChatHistory({
+                        chatHistory: newChatHistory,
+                        functionName: functionCall.functionName,
+                        functionDescription: functionDefinition.description,
+                        callParams: functionCall.params,
+                        callResult: functionCallResult,
+                        raw: functionCall.raw + this._chat.chatWrapper.generateFunctionCallResult(functionCall.functionName, functionCall.params, functionCallResult)
+                    });
+                    newContextWindowChatHistory = addFunctionCallToChatHistory({
+                        chatHistory: lastEvaluation.contextWindow,
+                        functionName: functionCall.functionName,
+                        functionDescription: functionDefinition.description,
+                        callParams: functionCall.params,
+                        callResult: functionCallResult,
+                        raw: functionCall.raw + this._chat.chatWrapper.generateFunctionCallResult(functionCall.functionName, functionCall.params, functionCallResult)
+                    });
+                    lastEvaluation.cleanHistory = newChatHistory;
+                    lastEvaluation.contextWindow = newContextWindowChatHistory;
+                    continue;
+                }
+                this._lastEvaluation = lastEvaluation;
+                this._chatHistory = newChatHistory;
+                const lastModelResponseItem = getLastModelResponseItem(newChatHistory);
+                return {
+                    response: lastModelResponseItem.response,
+                    responseText: lastModelResponseItem.response
+                        .filter((item) => typeof item === "string")
+                        .join(""),
+                    stopReason: metadata.stopReason,
+                    remainingGenerationAfterStop: metadata.remainingGenerationAfterStop
+                };
+            }
+        });
+    }
+    getChatHistory() {
+        return structuredClone(this._chatHistory);
+    }
+    getLastEvaluationContextWindow() {
+        if (this._lastEvaluation == null)
+            return null;
+        return structuredClone(this._lastEvaluation?.contextWindow);
+    }
+    setChatHistory(chatHistory) {
+        this._chatHistory = structuredClone(chatHistory);
+        this._lastEvaluation = undefined;
+    }
+    /** @internal */
+    _ensureNotDisposed() {
+        if (this.disposed)
+            throw new DisposedError();
+    }
+}
+function addFunctionCallToChatHistory({ chatHistory, functionName, functionDescription, callParams, callResult, raw }) {
+    const newChatHistory = chatHistory.slice();
+    if (newChatHistory.length === 0 || newChatHistory[newChatHistory.length - 1].type !== "model")
+        newChatHistory.push({
+            type: "model",
+            response: []
+        });
+    const lastModelResponseItem = newChatHistory[newChatHistory.length - 1];
+    const newLastModelResponseItem = { ...lastModelResponseItem };
+    newChatHistory[newChatHistory.length - 1] = newLastModelResponseItem;
+    const modelResponse = newLastModelResponseItem.response.slice();
+    newLastModelResponseItem.response = modelResponse;
+    modelResponse.push({
+        type: "functionCall",
+        name: functionName,
+        description: functionDescription,
+        params: callParams,
+        result: callResult,
+        raw
+    });
+    return newChatHistory;
+}
+function getLastModelResponseItem(chatHistory) {
+    if (chatHistory.length === 0 || chatHistory[chatHistory.length - 1].type !== "model")
+        throw new Error("Expected chat history to end with a model response");
+    return chatHistory[chatHistory.length - 1];
+}
+//# sourceMappingURL=LlamaChatSession.js.map

package/dist/evaluator/LlamaChatSession/LlamaChatSession.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LlamaChatSession.js","sourceRoot":"","sources":["../../../src/evaluator/LlamaChatSession/LlamaChatSession.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,iBAAiB,EAAE,aAAa,EAAE,UAAU,EAAE,QAAQ,EAAC,MAAM,iBAAiB,CAAC;AACvF,OAAO,EAAC,uBAAuB,EAAC,MAAM,iBAAiB,CAAC;AAGxD,OAAO,EAAC,8BAA8B,EAAC,MAAM,+CAA+C,CAAC;AAG7F,OAAO,EAAC,SAAS,EAAkD,MAAM,2BAA2B,CAAC;AA2IrG,MAAM,OAAO,gBAAgB;IACzB,gBAAgB,CAAkB,kBAAkB,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC/E,gBAAgB,CAAkB,oBAAoB,CAAU;IAChE,gBAAgB,CAAkB,aAAa,CAAuC;IACtF,gBAAgB,CAAS,YAAY,CAAoB;IACzD,gBAAgB,CAAS,eAAe,CAAuC;IAC/E,gBAAgB,CAAS,KAAK,CAAmB;IAEjC,SAAS,GAAG,IAAI,UAAU,EAAQ,CAAC;IAEnD;;OAEG;IACH,YAAmB,EACf,eAAe,EACf,WAAW,GAAG,MAAM,EACpB,YAAY,GAAG,uBAAuB,EACtC,mBAAmB,GAAG,IAAI,EAC1B,YAAY,EACU;QACtB,IAAI,eAAe,IAAI,IAAI;YACvB,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAEtD,IAAI,eAAe,CAAC,QAAQ;YACxB,MAAM,IAAI,aAAa,EAAE,CAAC;QAE9B,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;QAClC,IAAI,CAAC,YAAY,GAAG,CAAC;gBACjB,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,YAAY;aACrB,CAAC,CAAC;QAEH,IAAI,CAAC,KAAK,GAAG,IAAI,SAAS,CAAC;YACvB,mBAAmB;YACnB,WAAW;YACX,eAAe;SAClB,CAAC,CAAC;QAEH,IAAI,CAAC,oBAAoB,GAAG,mBAAmB,CAAC;QAEhD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CACvB,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,EAAE;YACrC,IAAI,CAAC,OAAO,EAAE,CAAC;QACnB,CAAC,CAAC,CACL,CAAC;QACF,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;IAC9D,CAAC;IAEM,OAAO,CAAC,EAAC,eAAe,GAAG,IAAI,CAAC,oBAAoB,KAAiC,EAAE;QAC1F,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI;YAClB,OAAO;QAEX,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAC,eAAe,EAAC,CAAC,CAAC;QACtC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAElB,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,CAAC;IACtC,CAAC;IAED,cAAc;IACP,CAAC,MAAM,CAAC,OAAO,CAAC;QACnB,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;IAC1B,CAAC;IAED,IAAW,QAAQ;QACf,OAAO,IAAI,CAAC,KAAK,IAAI,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;IACrD,CAAC;IAED,IAAW,WAAW;QAClB,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI;YAClB,MAAM,IAAI,aAAa,EAAE,CAAC;QAE9B,OAAO,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;IAClC,CAAC;IAED,IAAW,QAAQ;QACf,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI;YAClB,MAAM,IAAI,aAAa,EAAE,CAAC;QAE9B,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;IAC/B,CAAC;IAED,IAAW,OAAO;QACd,OAAO,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;IACjC,CAAC;IAED,IAAW,KAAK;QACZ,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IAC/B,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,MAAM,CAA4E,MAAc,EAAE,EAC3G,SAAS,EACT,sBAAsB,EACtB,OAAO,EACP,MAAM,EACN,SAAS,EACT,WAAW,EACX,IAAI,EACJ,IAAI,EACJ,IAAI,EACJ,OAAO,EACP,oBAAoB,GAAG,KAAK,EAC5B,aAAa,KACsB,EAAE;QACrC,MAAM,EAAC,YAAY,EAAC,GAAG,MAAM,IAAI,CAAC,cAAc,CAAY,MAAM,EAAE;YAChE,uEAAuE;YACvE,SAAS,EAAE,SAAsB;YACjC,sBAAsB,EAAE,sBAAmC;YAE3D,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,oBAAoB,EAAE,aAAa;SAC1G,CAAC,CAAC;QAEH,OAAO,YAAY,CAAC;IACxB,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,cAAc,CAA4E,MAAc,EAAE,EACnH,SAAS,EACT,sBAAsB,EACtB,OAAO,EACP,MAAM,EACN,SAAS,EACT,WAAW,EACX,IAAI,EACJ,IAAI,EACJ,IAAI,EACJ,OAAO,EACP,oBAAoB,GAAG,KAAK,EAC5B,aAAa,EACb,kBAAkB,KACiB,EAAE;QACrC,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE1B,IAAI,OAAO,IAAI,IAAI,IAAI,OAAO,CAAC,MAAM,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM;YACvD,MAAM,IAAI,KAAK,CAAC,wNAAwN,CAAC,CAAC;QAE9O,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE;YACrD,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAE1B,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI;gBAClB,MAAM,IAAI,aAAa,EAAE,CAAC;YAE9B,IAAI,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC;YAC1C,IAAI,cAAc,GAAG,8BAA8B,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;YAC/E,IAAI,2BAA2B,GAAG,cAAc,EAAE,aAAa,IAAI,IAAI;gBACnE,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,8BAA8B,CAAC,cAAc,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC;YAE5E,cAAc,CAAC,IAAI,CAAC;gBAChB,IAAI,EAAE,OAAO;gBACb,QAAQ,EAAE,EAAE;aACf,CAAC,CAAC;YAEH,IAAI,2BAA2B,IAAI,IAAI;gBACnC,2BAA2B,CAAC,IAAI,CAAC;oBAC7B,IAAI,EAAE,OAAO;oBACb,QAAQ,EAAE,EAAE;iBACf,CAAC,CAAC;YAEP,iDAAiD;YACjD,OAAO,IAAI,EAAE;gBACT,MAAM,EACF,YAAY,EACZ,cAAc,EAAE,qBAAqB,EACrC,QAAQ,EACX,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAY,cAAc,EAAE;oBAC7D,SAAS;oBACT,sBAAsB;oBACtB,OAAO,EAAE,OAAoB;oBAC7B,OAAO;oBACP,MAAM;oBACN,aAAa;oBACb,IAAI;oBACJ,IAAI;oBACJ,IAAI;oBACJ,SAAS;oBACT,WAAW;oBACX,oBAAoB;oBACpB,YAAY,EAAE;wBACV,GAAG,IAAI,CAAC,aAAa;wBACrB,sBAAsB,EAAE,cAAc,EAAE,oBAAoB;qBAC/D;oBACD,kBAAkB;oBAClB,2BAA2B,EAAE;wBACzB,OAAO,EAAE,2BAA2B;wBACpC,6CAA6C,EAAE,IAAI;qBACtD;iBACJ,CAAC,CAAC;gBACH,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBAE1B,cAAc,GAAG,qBAAqB,CAAC;gBACvC,cAAc,GAAG,cAAc,CAAC,YAAY,CAAC;gBAE7C,IAAI,YAAY,IAAI,IAAI,EAAE;oBACtB,MAAM,kBAAkB,GAAG,SAAS,EAAE,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;oBAElE,IAAI,kBAAkB,IAAI,IAAI;wBAC1B,MAAM,IAAI,KAAK,CAAC,qCAAqC,YAAY,CAAC,YAAY,wBAAwB,CAAC,CAAC;oBAE5G,MAAM,kBAAkB,GAAG,MAAM,kBAAkB,CAAC,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;oBACjF,IAAI,CAAC,kBAAkB,EAAE,CAAC;oBAE1B,cAAc,GAAG,4BAA4B,CAAC;wBAC1C,WAAW,EAAE,cAAc;wBAC3B,YAAY,EAAE,YAAY,CAAC,YAAY;wBACvC,mBAAmB,EAAE,kBAAkB,CAAC,WAAW;wBACnD,UAAU,EAAE,YAAY,CAAC,MAAM;wBAC/B,UAAU,EAAE,kBAAkB;wBAC9B,GAAG,EAAE,YAAY,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,0BAA0B,CACrE,YAAY,CAAC,YAAY,EACzB,YAAY,CAAC,MAAM,EACnB,kBAAkB,CACrB;qBACJ,CAAC,CAAC;oBAEH,2BAA2B,GAAG,4BAA4B,CAAC;wBACvD,WAAW,EAAE,cAAc,CAAC,aAAa;wBACzC,YAAY,EAAE,YAAY,CAAC,YAAY;wBACvC,mBAAmB,EAAE,kBAAkB,CAAC,WAAW;wBACnD,UAAU,EAAE,YAAY,CAAC,MAAM;wBAC/B,UAAU,EAAE,kBAAkB;wBAC9B,GAAG,EAAE,YAAY,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,0BAA0B,CACrE,YAAY,CAAC,YAAY,EACzB,YAAY,CAAC,MAAM,EACnB,kBAAkB,CACrB;qBACJ,CAAC,CAAC;oBACH,cAAc,CAAC,YAAY,GAAG,cAAc,CAAC;oBAC7C,cAAc,CAAC,aAAa,GAAG,2BAA2B,CAAC;oBAE3D,SAAS;iBACZ;gBAED,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;gBACtC,IAAI,CAAC,YAAY,GAAG,cAAc,CAAC;gBAEnC,MAAM,qBAAqB,GAAG,wBAAwB,CAAC,cAAc,CAAC,CAAC;gBAEvE,OAAO;oBACH,QAAQ,EAAE,qBAAqB,CAAC,QAAQ;oBACxC,YAAY,EAAE,qBAAqB,CAAC,QAAQ;yBACvC,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC;yBAC1D,IAAI,CAAC,EAAE,CAAC;oBACb,UAAU,EAAE,QAAQ,CAAC,UAAU;oBAC/B,4BAA4B,EAAE,QAAQ,CAAC,4BAA4B;iBACtE,CAAC;aACL;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,cAAc;QACjB,OAAO,eAAe,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC9C,CAAC;IAEM,8BAA8B;QACjC,IAAI,IAAI,CAAC,eAAe,IAAI,IAAI;YAC5B,OAAO,IAAI,CAAC;QAEhB,OAAO,eAAe,CAAC,IAAI,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;IAChE,CAAC;IAEM,cAAc,CAAC,WAA8B;QAChD,IAAI,CAAC,YAAY,GAAG,eAAe,CAAC,WAAW,CAAC,CAAC;QACjD,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;IACrC,CAAC;IAED,gBAAgB;IACR,kBAAkB;QACtB,IAAI,IAAI,CAAC,QAAQ;YACb,MAAM,IAAI,aAAa,EAAE,CAAC;IAClC,CAAC;CACJ;AAED,SAAS,4BAA4B,CAAC,EAClC,WAAW,EACX,YAAY,EACZ,mBAAmB,EACnB,UAAU,EACV,UAAU,EACV,GAAG,EAQN;IACG,MAAM,cAAc,GAAG,WAAW,CAAC,KAAK,EAAE,CAAC;IAC3C,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,IAAI,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO;QACzF,cAAc,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,OAAO;YACb,QAAQ,EAAE,EAAE;SACf,CAAC,CAAC;IAEP,MAAM,qBAAqB,GAAG,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAsB,CAAC;IAC7F,MAAM,wBAAwB,GAAG,EAAC,GAAG,qBAAqB,EAAC,CAAC;IAC5D,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,wBAAwB,CAAC;IAErE,MAAM,aAAa,GAAG,wBAAwB,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IAChE,wBAAwB,CAAC,QAAQ,GAAG,aAAa,CAAC;IAElD,aAAa,CAAC,IAAI,CAAC;QACf,IAAI,EAAE,cAAc;QACpB,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,mBAAmB;QAChC,MAAM,EAAE,UAAU;QAClB,MAAM,EAAE,UAAU;QAClB,GAAG;KACN,CAAC,CAAC;IAEH,OAAO,cAAc,CAAC;AAC1B,CAAC;AAED,SAAS,wBAAwB,CAAC,WAA8B;IAC5D,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,IAAI,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO;QAChF,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IAE1E,OAAO,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAsB,CAAC;AACpE,CAAC"}

package/dist/evaluator/LlamaChatSession/utils/defineChatSessionFunction.d.ts

@@ -0,0 +1,7 @@
+import { GbnfJsonSchema, GbnfJsonSchemaToType } from "../../../utils/gbnfJson/types.js";
+import { ChatSessionModelFunction } from "../../../types.js";
+export declare function defineChatSessionFunction<const Params extends GbnfJsonSchema | undefined>({ description, params, handler }: {
+    description?: string;
+    params?: Params;
+    handler: (params: GbnfJsonSchemaToType<Params>) => any;
+}): ChatSessionModelFunction<Params>;

package/dist/evaluator/LlamaChatSession/utils/defineChatSessionFunction.js

@@ -0,0 +1,8 @@
+export function defineChatSessionFunction({ description, params, handler }) {
+    return {
+        description,
+        params,
+        handler
+    };
+}
+//# sourceMappingURL=defineChatSessionFunction.js.map

package/dist/evaluator/LlamaChatSession/utils/defineChatSessionFunction.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"defineChatSessionFunction.js","sourceRoot":"","sources":["../../../../src/evaluator/LlamaChatSession/utils/defineChatSessionFunction.ts"],"names":[],"mappings":"AAGA,MAAM,UAAU,yBAAyB,CAAkD,EACvF,WAAW,EACX,MAAM,EACN,OAAO,EAKV;IACG,OAAO;QACH,WAAW;QACX,MAAM;QACN,OAAO;KACV,CAAC;AACN,CAAC"}

package/dist/evaluator/LlamaCompletion.d.ts

@@ -0,0 +1,148 @@
+/// <reference types="node" />
+import { EventRelay } from "lifecycle-utils";
+import { LLamaContextualRepeatPenalty, Token } from "../types.js";
+import { LlamaText } from "../utils/LlamaText.js";
+import { StopGenerationTrigger } from "../utils/StopGenerationDetector.js";
+import { LlamaGrammar } from "./LlamaGrammar.js";
+import { EvaluationPriority } from "./LlamaContext/types.js";
+import { LlamaContextSequence } from "./LlamaContext/LlamaContext.js";
+export type LlamaCompletionOptions = {
+    contextSequence: LlamaContextSequence;
+    /** Automatically dispose the sequence when the object is disposed */
+    autoDisposeSequence?: boolean;
+};
+export type LlamaCompletionGenerationOptions = {
+    onToken?: (tokens: Token[]) => void;
+    signal?: AbortSignal;
+    maxTokens?: number;
+    /**
+     * Temperature is a hyperparameter that controls the randomness of the generated text.
+     * It affects the probability distribution of the model's output tokens.
+     * A higher temperature (e.g., 1.5) makes the output more random and creative,
+     * while a lower temperature (e.g., 0.5) makes the output more focused, deterministic, and conservative.
+     * The suggested temperature is 0.8, which provides a balance between randomness and determinism.
+     * At the extreme, a temperature of 0 will always pick the most likely next token, leading to identical outputs in each run.
+     *
+     * Set to `0` to disable.
+     * Disabled by default (set to `0`).
+     */
+    temperature?: number;
+    /**
+     * From the next token candidates, discard the percentage of tokens with the lowest probability.
+     * For example, if set to `0.05`, 5% of the lowest probability tokens will be discarded.
+     * This is useful for generating more high-quality results when using a high temperature.
+     * Set to a value between `0` and `1` to enable.
+     *
+     * Only relevant when `temperature` is set to a value greater than `0`.
+     * Disabled by default.
+     */
+    minP?: number;
+    /**
+     * Limits the model to consider only the K most likely next tokens for sampling at each step of sequence generation.
+     * An integer number between `1` and the size of the vocabulary.
+     * Set to `0` to disable (which uses the full vocabulary).
+     *
+     * Only relevant when `temperature` is set to a value greater than 0.
+     */
+    topK?: number;
+    /**
+     * Dynamically selects the smallest set of tokens whose cumulative probability exceeds the threshold P,
+     * and samples the next token only from this set.
+     * A float number between `0` and `1`.
+     * Set to `1` to disable.
+     *
+     * Only relevant when `temperature` is set to a value greater than `0`.
+     */
+    topP?: number;
+    /**
+     * Trim whitespace from the end of the generated text
+     * Disabled by default.
+     */
+    trimWhitespaceSuffix?: boolean;
+    repeatPenalty?: false | LLamaContextualRepeatPenalty;
+    /**
+     * See the parameter `evaluationPriority` on the `LlamaContextSequence.evaluate()` function for more information.
+     */
+    evaluationPriority?: EvaluationPriority;
+    grammar?: LlamaGrammar;
+    /** Consider any of these as EOS for the generated text */
+    stopGenerationTriggers?: readonly (StopGenerationTrigger | LlamaText)[];
+    /**
+     * The number of tokens to delete from the context window to make space for new ones.
+     * Defaults to 10% of the context size.
+     */
+    contextShiftSize?: number | ((sequence: LlamaContextSequence) => number | Promise<number>);
+    /**
+     * Context shift reconstructs the context with partial relevant data to continue generation when the context fills up.
+     * This flag disables this behavior.
+     * This flag will cause the generation to stop when the context fills up
+     * by setting an appropriate `maxTokens` value or lowering the given `maxTokens` value when needed.
+     * This flag will cause the generation to fail if there's no space for generating new tokens at all with the given inputs.
+     *
+     * Disabled by default. Not recommended unless you know what you're doing.
+     */
+    disableContextShift?: boolean;
+};
+export type LlamaInfillGenerationOptions = LlamaCompletionGenerationOptions & {
+    /**
+     * The minimum number of tokens to keep from the prefix input when making a context shift.
+     * Defaults to 10% of the context size.
+     */
+    minPrefixKeepTokens?: number | ((sequence: LlamaContextSequence) => number | Promise<number>);
+};
+export declare class LlamaCompletion {
+    readonly onDispose: EventRelay<void>;
+    constructor({ contextSequence, autoDisposeSequence }: LlamaCompletionOptions);
+    dispose({ disposeSequence }?: {
+        disposeSequence?: boolean;
+    }): void;
+    /** @hidden */
+    [Symbol.dispose](): void;
+    get disposed(): boolean;
+    get infillSupported(): boolean;
+    /**
+     * Generate a completion for an input.
+     */
+    generateCompletion(input: Token[] | string | LlamaText, options?: LlamaCompletionGenerationOptions): Promise<string>;
+    /**
+     * Same as `generateCompletion`, but returns additional metadata about the generation.
+     * See `generateCompletion` for more information.
+     */
+    generateCompletionWithMeta(input: Token[] | string | LlamaText, { onToken, signal, maxTokens, temperature, minP, topK, topP, trimWhitespaceSuffix, repeatPenalty, evaluationPriority, grammar, stopGenerationTriggers, contextShiftSize, disableContextShift }?: LlamaCompletionGenerationOptions): Promise<{
+        response: string;
+        metadata: {
+            remainingGenerationAfterStop: string | Token[];
+            stopReason: "eosToken" | "stopGenerationTrigger";
+        };
+    } | {
+        response: string;
+        metadata: {
+            stopReason: "maxTokens";
+            remainingGenerationAfterStop?: undefined;
+        };
+    }>;
+    /**
+     * Infill (also known as Fill-In-Middle), generates a completion for an input (`prefixInput`) that
+     * should connect to a given continuation (`suffixInput`).
+     * For example, for `prefixInput: "123"` and `suffixInput: "789"`, the model is expected to generate `456`
+     * to make the final text be `123456789`.
+     */
+    generateInfillCompletion(prefixInput: Token[] | string | LlamaText, suffixInput: Token[] | string | LlamaText, options?: LlamaInfillGenerationOptions): Promise<string>;
+    /**
+     * Same as `generateInfillCompletion`, but returns additional metadata about the generation.
+     * See `generateInfillCompletion` for more information.
+     */
+    generateInfillCompletionWithMeta(prefixInput: Token[] | string | LlamaText, suffixInput: Token[] | string | LlamaText, { onToken, signal, maxTokens, temperature, minP, topK, topP, trimWhitespaceSuffix, repeatPenalty, evaluationPriority, grammar, contextShiftSize, stopGenerationTriggers, minPrefixKeepTokens, disableContextShift }?: LlamaInfillGenerationOptions): Promise<{
+        response: string;
+        metadata: {
+            remainingGenerationAfterStop: string | Token[];
+            stopReason: "eosToken" | "stopGenerationTrigger";
+        };
+    } | {
+        response: string;
+        metadata: {
+            stopReason: "maxTokens";
+            remainingGenerationAfterStop?: undefined;
+        };
+    }>;
+}