node-llama-cpp 3.0.0-beta.3 → 3.0.0-beta.30

package/dist/evaluator/LlamaChatSession/LlamaChatSession.js

@@ -0,0 +1,419 @@
+import { DisposeAggregator, DisposedError, EventRelay, withLock } from "lifecycle-utils";
+import { defaultChatSystemPrompt } from "../../config.js";
+import { appendUserMessageToChatHistory } from "../../utils/appendUserMessageToChatHistory.js";
+import { LlamaChat } from "../LlamaChat/LlamaChat.js";
+import { wrapAbortSignal } from "../../utils/wrapAbortSignal.js";
+import { safeEventCallback } from "../../utils/safeEventCallback.js";
+import { LlamaChatSessionPromptCompletionEngine } from "./utils/LlamaChatSessionPromptCompletionEngine.js";
+export class LlamaChatSession {
+    /** @internal */ _disposeAggregator = new DisposeAggregator();
+    /** @internal */ _autoDisposeSequence;
+    /** @internal */ _contextShift;
+    /** @internal */ _chatLock = {};
+    /** @internal */ _chatHistory;
+    /** @internal */ _lastEvaluation;
+    /** @internal */ _chat;
+    /** @internal */ _chatHistoryStateRef = {};
+    /** @internal */ _preloadAndCompleteAbortControllers = new Set();
+    onDispose = new EventRelay();
+    /**
+     * @param options
+     */
+    constructor({ contextSequence, chatWrapper = "auto", systemPrompt = defaultChatSystemPrompt, forceAddSystemPrompt = false, autoDisposeSequence = true, contextShift }) {
+        if (contextSequence == null)
+            throw new Error("contextSequence cannot be null");
+        if (contextSequence.disposed)
+            throw new DisposedError();
+        this._contextShift = contextShift;
+        this._chat = new LlamaChat({
+            autoDisposeSequence,
+            chatWrapper,
+            contextSequence
+        });
+        const chatWrapperSupportsSystemMessages = this._chat.chatWrapper.settings.supportsSystemMessages;
+        if (chatWrapperSupportsSystemMessages == null || chatWrapperSupportsSystemMessages || forceAddSystemPrompt)
+            this._chatHistory = [{
+                    type: "system",
+                    text: systemPrompt
+                }];
+        else
+            this._chatHistory = [];
+        this._autoDisposeSequence = autoDisposeSequence;
+        this._disposeAggregator.add(this._chat.onDispose.createListener(() => {
+            this.dispose();
+        }));
+        this._disposeAggregator.add(this.onDispose.dispatchEvent);
+    }
+    dispose({ disposeSequence = this._autoDisposeSequence } = {}) {
+        if (this._chat == null)
+            return;
+        this._chat.dispose({ disposeSequence });
+        this._chat = null;
+        this._disposeAggregator.dispose();
+    }
+    /** @hidden */
+    [Symbol.dispose]() {
+        return this.dispose();
+    }
+    get disposed() {
+        return this._chat == null || this._chat.disposed;
+    }
+    get chatWrapper() {
+        if (this._chat == null)
+            throw new DisposedError();
+        return this._chat.chatWrapper;
+    }
+    get sequence() {
+        if (this._chat == null)
+            throw new DisposedError();
+        return this._chat.sequence;
+    }
+    get context() {
+        return this.sequence.context;
+    }
+    get model() {
+        return this.sequence.model;
+    }
+    /**
+     * @param prompt
+     * @param [options]
+     */
+    async prompt(prompt, { functions, documentFunctionParams, maxParallelFunctionCalls, onToken, signal, stopOnAbortSignal = false, maxTokens, temperature, minP, topK, topP, grammar, trimWhitespaceSuffix = false, repeatPenalty, tokenBias, customStopTriggers } = {}) {
+        const { responseText } = await this.promptWithMeta(prompt, {
+            // this is a workaround to allow passing both `functions` and `grammar`
+            functions: functions,
+            documentFunctionParams: documentFunctionParams,
+            maxParallelFunctionCalls: maxParallelFunctionCalls,
+            onToken, signal, stopOnAbortSignal, maxTokens, temperature, minP, topK, topP, grammar, trimWhitespaceSuffix, repeatPenalty,
+            tokenBias, customStopTriggers
+        });
+        return responseText;
+    }
+    /**
+     * @param prompt
+     * @param [options]
+     */
+    async promptWithMeta(prompt, { functions, documentFunctionParams, maxParallelFunctionCalls, onToken, signal, stopOnAbortSignal = false, maxTokens, temperature, minP, topK, topP, grammar, trimWhitespaceSuffix = false, repeatPenalty, tokenBias, customStopTriggers, evaluationPriority } = {}) {
+        this._ensureNotDisposed();
+        if (grammar != null && grammar._llama !== this.model._llama)
+            throw new Error("The LlamaGrammar used by passed to this function was created with a different Llama instance than the one used by this sequence's model. Make sure you use the same Llama instance for both the model and the grammar.");
+        this._stopAllPreloadAndPromptCompletions();
+        return await withLock(this._chatLock, "evaluation", signal, async () => {
+            this._ensureNotDisposed();
+            this._stopAllPreloadAndPromptCompletions();
+            if (this._chat == null)
+                throw new DisposedError();
+            const abortController = wrapAbortSignal(signal);
+            let lastEvaluation = this._lastEvaluation;
+            let newChatHistory = appendUserMessageToChatHistory(this._chatHistory, prompt);
+            let newContextWindowChatHistory = lastEvaluation?.contextWindow == null
+                ? undefined
+                : appendUserMessageToChatHistory(lastEvaluation?.contextWindow, prompt);
+            newChatHistory.push({
+                type: "model",
+                response: []
+            });
+            if (newContextWindowChatHistory != null)
+                newContextWindowChatHistory.push({
+                    type: "model",
+                    response: []
+                });
+            // eslint-disable-next-line no-constant-condition
+            while (true) {
+                const functionCallsAndResults = [];
+                let canThrowFunctionCallingErrors = false;
+                let abortedOnFunctionCallError = false;
+                const initialOutputTokens = this._chat.sequence.tokenMeter.usedOutputTokens;
+                const { lastEvaluation: currentLastEvaluation, metadata } = await this._chat.generateResponse(newChatHistory, {
+                    functions,
+                    documentFunctionParams,
+                    maxParallelFunctionCalls,
+                    grammar: grammar, // this is a workaround to allow passing both `functions` and `grammar`
+                    onToken: safeEventCallback(onToken),
+                    signal: abortController.signal,
+                    stopOnAbortSignal,
+                    repeatPenalty,
+                    minP,
+                    topK,
+                    topP,
+                    tokenBias,
+                    customStopTriggers,
+                    maxTokens,
+                    temperature,
+                    trimWhitespaceSuffix,
+                    contextShift: {
+                        ...this._contextShift,
+                        lastEvaluationMetadata: lastEvaluation?.contextShiftMetadata
+                    },
+                    evaluationPriority,
+                    lastEvaluationContextWindow: {
+                        history: newContextWindowChatHistory,
+                        minimumOverlapPercentageToPreventContextShift: 0.5
+                    },
+                    onFunctionCall: async (functionCall) => {
+                        functionCallsAndResults.push((async () => {
+                            try {
+                                const functionDefinition = functions?.[functionCall.functionName];
+                                if (functionDefinition == null)
+                                    throw new Error(`The model tried to call function "${functionCall.functionName}" which is not defined`);
+                                const functionCallResult = await functionDefinition.handler(functionCall.params);
+                                return {
+                                    functionCall,
+                                    functionDefinition,
+                                    functionCallResult
+                                };
+                            }
+                            catch (err) {
+                                if (!abortController.signal.aborted) {
+                                    abortedOnFunctionCallError = true;
+                                    abortController.abort(err);
+                                }
+                                if (canThrowFunctionCallingErrors)
+                                    throw err;
+                                return null;
+                            }
+                        })());
+                    }
+                });
+                this._ensureNotDisposed();
+                if (abortController.signal.aborted && (abortedOnFunctionCallError || !stopOnAbortSignal))
+                    throw abortController.signal.reason;
+                if (maxTokens != null)
+                    maxTokens = Math.max(0, maxTokens - (this._chat.sequence.tokenMeter.usedOutputTokens - initialOutputTokens));
+                lastEvaluation = currentLastEvaluation;
+                newChatHistory = lastEvaluation.cleanHistory;
+                if (functionCallsAndResults.length > 0) {
+                    canThrowFunctionCallingErrors = true;
+                    const functionCallResultsPromise = Promise.all(functionCallsAndResults);
+                    await Promise.race([
+                        functionCallResultsPromise,
+                        new Promise((accept, reject) => {
+                            abortController.signal.addEventListener("abort", () => {
+                                if (abortedOnFunctionCallError || !stopOnAbortSignal)
+                                    reject(abortController.signal.reason);
+                                else
+                                    accept();
+                            });
+                            if (abortController.signal.aborted) {
+                                if (abortedOnFunctionCallError || !stopOnAbortSignal)
+                                    reject(abortController.signal.reason);
+                                else
+                                    accept();
+                            }
+                        })
+                    ]);
+                    this._ensureNotDisposed();
+                    if (!abortController.signal.aborted) {
+                        const functionCallResults = (await functionCallResultsPromise)
+                            .filter((result) => result != null);
+                        this._ensureNotDisposed();
+                        if (abortController.signal.aborted)
+                            throw abortController.signal.reason;
+                        newContextWindowChatHistory = lastEvaluation.contextWindow;
+                        for (const { functionCall, functionDefinition, functionCallResult } of functionCallResults) {
+                            newChatHistory = addFunctionCallToChatHistory({
+                                chatHistory: newChatHistory,
+                                functionName: functionCall.functionName,
+                                functionDescription: functionDefinition.description,
+                                callParams: functionCall.params,
+                                callResult: functionCallResult,
+                                rawCall: functionCall.raw
+                            });
+                            newContextWindowChatHistory = addFunctionCallToChatHistory({
+                                chatHistory: newContextWindowChatHistory,
+                                functionName: functionCall.functionName,
+                                functionDescription: functionDefinition.description,
+                                callParams: functionCall.params,
+                                callResult: functionCallResult,
+                                rawCall: functionCall.raw
+                            });
+                        }
+                        lastEvaluation.cleanHistory = newChatHistory;
+                        lastEvaluation.contextWindow = newContextWindowChatHistory;
+                        continue;
+                    }
+                }
+                this._lastEvaluation = lastEvaluation;
+                this._chatHistory = newChatHistory;
+                this._chatHistoryStateRef = {};
+                const lastModelResponseItem = getLastModelResponseItem(newChatHistory);
+                const responseText = lastModelResponseItem.response
+                    .filter((item) => typeof item === "string")
+                    .join("");
+                if (metadata.stopReason === "customStopTrigger")
+                    return {
+                        response: lastModelResponseItem.response,
+                        responseText,
+                        stopReason: metadata.stopReason,
+                        customStopTrigger: metadata.customStopTrigger,
+                        remainingGenerationAfterStop: metadata.remainingGenerationAfterStop
+                    };
+                return {
+                    response: lastModelResponseItem.response,
+                    responseText,
+                    stopReason: metadata.stopReason,
+                    remainingGenerationAfterStop: metadata.remainingGenerationAfterStop
+                };
+            }
+        });
+    }
+    /**
+     * Preload a user prompt into the current context sequence state to make later inference of the model response begin sooner
+     * and feel faster.
+     *
+     * > **Note:** Preloading a long user prompt can incur context shifts, so consider limiting the length of prompts you preload
+     * @param prompt - the prompt to preload
+     * @param [options]
+     */
+    async preloadPrompt(prompt, options = {}) {
+        await this.completePromptWithMeta(prompt, {
+            ...options,
+            maxTokens: 0
+        });
+    }
+    /**
+     * Preload a user prompt into the current context sequence state and generate a completion for it.
+     *
+     * > **Note:** Preloading a long user prompt and completing a user prompt with a high number of `maxTokens` can incur context shifts,
+     * > so consider limiting the length of prompts you preload.
+     * >
+     * > Also, it's recommended to limit the number of tokens generated to a reasonable amount by configuring `maxTokens`.
+     * @param prompt - the prompt to preload
+     * @param [options]
+     */
+    async completePrompt(prompt, options = {}) {
+        const { completion } = await this.completePromptWithMeta(prompt, options);
+        return completion;
+    }
+    /**
+     * Create a smart completion engine that caches the prompt completions
+     * and reuses them when the user prompt matches the beginning of the cached prompt or completion.
+     *
+     * All completions are made and cache is used only for the current chat session state.
+     * You can create a single completion engine for an entire chat session.
+     */
+    createPromptCompletionEngine(options) {
+        return LlamaChatSessionPromptCompletionEngine._create(this, options);
+    }
+    /**
+     * See `completePrompt` for more information.
+     * @param prompt
+     * @param [options]
+     */
+    async completePromptWithMeta(prompt, { maxTokens, stopOnAbortSignal = false, functions, documentFunctionParams, onToken, signal, temperature, minP, topK, topP, grammar, trimWhitespaceSuffix = false, repeatPenalty, tokenBias, customStopTriggers, evaluationPriority } = {}) {
+        this._ensureNotDisposed();
+        if (grammar != null && grammar._llama !== this.model._llama)
+            throw new Error("The LlamaGrammar used by passed to this function was created with a different Llama instance than the one used by this sequence's model. Make sure you use the same Llama instance for both the model and the grammar.");
+        const abortController = wrapAbortSignal(signal);
+        this._preloadAndCompleteAbortControllers.add(abortController);
+        try {
+            return await withLock(this._chatLock, "evaluation", abortController.signal, async () => {
+                this._ensureNotDisposed();
+                if (this._chat == null)
+                    throw new DisposedError();
+                const { completion, lastEvaluation, metadata } = await this._chat.loadChatAndCompleteUserMessage(this._chatHistory, {
+                    initialUserPrompt: prompt,
+                    functions,
+                    documentFunctionParams,
+                    grammar,
+                    onToken,
+                    signal: abortController.signal,
+                    stopOnAbortSignal: true,
+                    repeatPenalty,
+                    minP,
+                    topK,
+                    topP,
+                    tokenBias,
+                    customStopTriggers,
+                    maxTokens,
+                    temperature,
+                    trimWhitespaceSuffix,
+                    contextShift: {
+                        ...this._contextShift,
+                        lastEvaluationMetadata: this._lastEvaluation?.contextShiftMetadata
+                    },
+                    evaluationPriority,
+                    lastEvaluationContextWindow: {
+                        history: this._lastEvaluation?.contextWindow,
+                        minimumOverlapPercentageToPreventContextShift: 0.8
+                    }
+                });
+                this._ensureNotDisposed();
+                this._lastEvaluation = {
+                    cleanHistory: this._chatHistory,
+                    contextWindow: lastEvaluation.contextWindow,
+                    contextShiftMetadata: lastEvaluation.contextShiftMetadata
+                };
+                if (!stopOnAbortSignal && metadata.stopReason === "abort" && abortController.signal?.aborted)
+                    throw abortController.signal.reason;
+                if (metadata.stopReason === "customStopTrigger")
+                    return {
+                        completion: completion,
+                        stopReason: metadata.stopReason,
+                        customStopTrigger: metadata.customStopTrigger,
+                        remainingGenerationAfterStop: metadata.remainingGenerationAfterStop
+                    };
+                return {
+                    completion: completion,
+                    stopReason: metadata.stopReason,
+                    remainingGenerationAfterStop: metadata.remainingGenerationAfterStop
+                };
+            });
+        }
+        finally {
+            this._preloadAndCompleteAbortControllers.delete(abortController);
+        }
+    }
+    getChatHistory() {
+        return structuredClone(this._chatHistory);
+    }
+    getLastEvaluationContextWindow() {
+        if (this._lastEvaluation == null)
+            return null;
+        return structuredClone(this._lastEvaluation?.contextWindow);
+    }
+    setChatHistory(chatHistory) {
+        this._chatHistory = structuredClone(chatHistory);
+        this._chatHistoryStateRef = {};
+        this._lastEvaluation = undefined;
+    }
+    /** @internal */
+    _stopAllPreloadAndPromptCompletions() {
+        for (const abortController of this._preloadAndCompleteAbortControllers)
+            abortController.abort();
+        this._preloadAndCompleteAbortControllers.clear();
+    }
+    /** @internal */
+    _ensureNotDisposed() {
+        if (this.disposed)
+            throw new DisposedError();
+    }
+}
+function addFunctionCallToChatHistory({ chatHistory, functionName, functionDescription, callParams, callResult, rawCall }) {
+    const newChatHistory = chatHistory.slice();
+    if (newChatHistory.length === 0 || newChatHistory[newChatHistory.length - 1].type !== "model")
+        newChatHistory.push({
+            type: "model",
+            response: []
+        });
+    const lastModelResponseItem = newChatHistory[newChatHistory.length - 1];
+    const newLastModelResponseItem = { ...lastModelResponseItem };
+    newChatHistory[newChatHistory.length - 1] = newLastModelResponseItem;
+    const modelResponse = newLastModelResponseItem.response.slice();
+    newLastModelResponseItem.response = modelResponse;
+    modelResponse.push({
+        type: "functionCall",
+        name: functionName,
+        description: functionDescription,
+        params: callParams,
+        result: callResult,
+        rawCall
+    });
+    return newChatHistory;
+}
+function getLastModelResponseItem(chatHistory) {
+    if (chatHistory.length === 0 || chatHistory[chatHistory.length - 1].type !== "model")
+        throw new Error("Expected chat history to end with a model response");
+    return chatHistory[chatHistory.length - 1];
+}
+//# sourceMappingURL=LlamaChatSession.js.map

package/dist/evaluator/LlamaChatSession/LlamaChatSession.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LlamaChatSession.js","sourceRoot":"","sources":["../../../src/evaluator/LlamaChatSession/LlamaChatSession.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,iBAAiB,EAAE,aAAa,EAAE,UAAU,EAAE,QAAQ,EAAC,MAAM,iBAAiB,CAAC;AACvF,OAAO,EAAC,uBAAuB,EAAC,MAAM,iBAAiB,CAAC;AAKxD,OAAO,EAAC,8BAA8B,EAAC,MAAM,+CAA+C,CAAC;AAG7F,OAAO,EAAC,SAAS,EAAiF,MAAM,2BAA2B,CAAC;AAIpI,OAAO,EAAC,eAAe,EAAC,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAC,iBAAiB,EAAC,MAAM,kCAAkC,CAAC;AACnE,OAAO,EACqC,sCAAsC,EACjF,MAAM,mDAAmD,CAAC;AAmO3D,MAAM,OAAO,gBAAgB;IACzB,gBAAgB,CAAkB,kBAAkB,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC/E,gBAAgB,CAAkB,oBAAoB,CAAU;IAChE,gBAAgB,CAAkB,aAAa,CAAuC;IACtF,gBAAgB,CAAkB,SAAS,GAAG,EAAE,CAAC;IACjD,gBAAgB,CAAS,YAAY,CAAoB;IACzD,gBAAgB,CAAS,eAAe,CAAuC;IAC/E,gBAAgB,CAAS,KAAK,CAAmB;IACjD,gBAAgB,CAAQ,oBAAoB,GAAG,EAAE,CAAC;IAClD,gBAAgB,CAAiB,mCAAmC,GAAG,IAAI,GAAG,EAAmB,CAAC;IAElF,SAAS,GAAG,IAAI,UAAU,EAAQ,CAAC;IAEnD;;OAEG;IACH,YAAmB,EACf,eAAe,EACf,WAAW,GAAG,MAAM,EACpB,YAAY,GAAG,uBAAuB,EACtC,oBAAoB,GAAG,KAAK,EAC5B,mBAAmB,GAAG,IAAI,EAC1B,YAAY,EACU;QACtB,IAAI,eAAe,IAAI,IAAI;YACvB,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAEtD,IAAI,eAAe,CAAC,QAAQ;YACxB,MAAM,IAAI,aAAa,EAAE,CAAC;QAE9B,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;QAElC,IAAI,CAAC,KAAK,GAAG,IAAI,SAAS,CAAC;YACvB,mBAAmB;YACnB,WAAW;YACX,eAAe;SAClB,CAAC,CAAC;QAEH,MAAM,iCAAiC,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,sBAAsB,CAAC;QACjG,IAAI,iCAAiC,IAAI,IAAI,IAAI,iCAAiC,IAAI,oBAAoB;YACtG,IAAI,CAAC,YAAY,GAAG,CAAC;oBACjB,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,YAAY;iBACrB,CAAC,CAAC;;YAEH,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;QAE3B,IAAI,CAAC,oBAAoB,GAAG,mBAAmB,CAAC;QAEhD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CACvB,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,EAAE;YACrC,IAAI,CAAC,OAAO,EAAE,CAAC;QACnB,CAAC,CAAC,CACL,CAAC;QACF,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;IAC9D,CAAC;IAEM,OAAO,CAAC,EAAC,eAAe,GAAG,IAAI,CAAC,oBAAoB,KAAiC,EAAE;QAC1F,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI;YAClB,OAAO;QAEX,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAC,eAAe,EAAC,CAAC,CAAC;QACtC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAElB,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,CAAC;IACtC,CAAC;IAED,cAAc;IACP,CAAC,MAAM,CAAC,OAAO,CAAC;QACnB,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;IAC1B,CAAC;IAED,IAAW,QAAQ;QACf,OAAO,IAAI,CAAC,KAAK,IAAI,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;IACrD,CAAC;IAED,IAAW,WAAW;QAClB,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI;YAClB,MAAM,IAAI,aAAa,EAAE,CAAC;QAE9B,OAAO,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;IAClC,CAAC;IAED,IAAW,QAAQ;QACf,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI;YAClB,MAAM,IAAI,aAAa,EAAE,CAAC;QAE9B,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;IAC/B,CAAC;IAED,IAAW,OAAO;QACd,OAAO,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;IACjC,CAAC;IAED,IAAW,KAAK;QACZ,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IAC/B,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,MAAM,CAA4E,MAAc,EAAE,EAC3G,SAAS,EACT,sBAAsB,EACtB,wBAAwB,EACxB,OAAO,EACP,MAAM,EACN,iBAAiB,GAAG,KAAK,EACzB,SAAS,EACT,WAAW,EACX,IAAI,EACJ,IAAI,EACJ,IAAI,EACJ,OAAO,EACP,oBAAoB,GAAG,KAAK,EAC5B,aAAa,EACb,SAAS,EACT,kBAAkB,KACiB,EAAE;QACrC,MAAM,EAAC,YAAY,EAAC,GAAG,MAAM,IAAI,CAAC,cAAc,CAAY,MAAM,EAAE;YAChE,uEAAuE;YACvE,SAAS,EAAE,SAAsB;YACjC,sBAAsB,EAAE,sBAAmC;YAC3D,wBAAwB,EAAE,wBAAqC;YAE/D,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,oBAAoB,EAAE,aAAa;YAC1H,SAAS,EAAE,kBAAkB;SAChC,CAAC,CAAC;QAEH,OAAO,YAAY,CAAC;IACxB,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,cAAc,CAA4E,MAAc,EAAE,EACnH,SAAS,EACT,sBAAsB,EACtB,wBAAwB,EACxB,OAAO,EACP,MAAM,EACN,iBAAiB,GAAG,KAAK,EACzB,SAAS,EACT,WAAW,EACX,IAAI,EACJ,IAAI,EACJ,IAAI,EACJ,OAAO,EACP,oBAAoB,GAAG,KAAK,EAC5B,aAAa,EACb,SAAS,EACT,kBAAkB,EAClB,kBAAkB,KACiB,EAAE;QACrC,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE1B,IAAI,OAAO,IAAI,IAAI,IAAI,OAAO,CAAC,MAAM,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM;YACvD,MAAM,IAAI,KAAK,CAAC,wNAAwN,CAAC,CAAC;QAE9O,IAAI,CAAC,mCAAmC,EAAE,CAAC;QAC3C,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE;YACnE,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC1B,IAAI,CAAC,mCAAmC,EAAE,CAAC;YAE3C,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI;gBAClB,MAAM,IAAI,aAAa,EAAE,CAAC;YAE9B,MAAM,eAAe,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;YAChD,IAAI,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC;YAC1C,IAAI,cAAc,GAAG,8BAA8B,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;YAC/E,IAAI,2BAA2B,GAAG,cAAc,EAAE,aAAa,IAAI,IAAI;gBACnE,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,8BAA8B,CAAC,cAAc,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC;YAE5E,cAAc,CAAC,IAAI,CAAC;gBAChB,IAAI,EAAE,OAAO;gBACb,QAAQ,EAAE,EAAE;aACf,CAAC,CAAC;YAEH,IAAI,2BAA2B,IAAI,IAAI;gBACnC,2BAA2B,CAAC,IAAI,CAAC;oBAC7B,IAAI,EAAE,OAAO;oBACb,QAAQ,EAAE,EAAE;iBACf,CAAC,CAAC;YAEP,iDAAiD;YACjD,OAAO,IAAI,EAAE,CAAC;gBACV,MAAM,uBAAuB,GAIvB,EAAE,CAAC;gBACT,IAAI,6BAA6B,GAAG,KAAK,CAAC;gBAC1C,IAAI,0BAA0B,GAAG,KAAK,CAAC;gBAEvC,MAAM,mBAAmB,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,gBAAgB,CAAC;gBAC5E,MAAM,EACF,cAAc,EAAE,qBAAqB,EACrC,QAAQ,EACX,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAY,cAAc,EAAE;oBAC7D,SAAS;oBACT,sBAAsB;oBACtB,wBAAwB;oBACxB,OAAO,EAAE,OAAoB,EAAE,uEAAuE;oBACtG,OAAO,EAAE,iBAAiB,CAAC,OAAO,CAAC;oBACnC,MAAM,EAAE,eAAe,CAAC,MAAM;oBAC9B,iBAAiB;oBACjB,aAAa;oBACb,IAAI;oBACJ,IAAI;oBACJ,IAAI;oBACJ,SAAS;oBACT,kBAAkB;oBAClB,SAAS;oBACT,WAAW;oBACX,oBAAoB;oBACpB,YAAY,EAAE;wBACV,GAAG,IAAI,CAAC,aAAa;wBACrB,sBAAsB,EAAE,cAAc,EAAE,oBAAoB;qBAC/D;oBACD,kBAAkB;oBAClB,2BAA2B,EAAE;wBACzB,OAAO,EAAE,2BAA2B;wBACpC,6CAA6C,EAAE,GAAG;qBACrD;oBACD,cAAc,EAAE,KAAK,EAAC,YAAY,EAAE,EAAE;wBAClC,uBAAuB,CAAC,IAAI,CACxB,CAAC,KAAK,IAAI,EAAE;4BACR,IAAI,CAAC;gCACD,MAAM,kBAAkB,GAAG,SAAS,EAAE,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;gCAElE,IAAI,kBAAkB,IAAI,IAAI;oCAC1B,MAAM,IAAI,KAAK,CACX,qCAAqC,YAAY,CAAC,YAAY,wBAAwB,CACzF,CAAC;gCAEN,MAAM,kBAAkB,GAAG,MAAM,kBAAkB,CAAC,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gCAEjF,OAAO;oCACH,YAAY;oCACZ,kBAAkB;oCAClB,kBAAkB;iCACrB,CAAC;4BACN,CAAC;4BAAC,OAAO,GAAG,EAAE,CAAC;gCACX,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oCAClC,0BAA0B,GAAG,IAAI,CAAC;oCAClC,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gCAC/B,CAAC;gCAED,IAAI,6BAA6B;oCAC7B,MAAM,GAAG,CAAC;gCAEd,OAAO,IAAI,CAAC;4BAChB,CAAC;wBACL,CAAC,CAAC,EAAE,CACP,CAAC;oBACN,CAAC;iBACJ,CAAC,CAAC;gBACH,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBAC1B,IAAI,eAAe,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,0BAA0B,IAAI,CAAC,iBAAiB,CAAC;oBACpF,MAAM,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC;gBAExC,IAAI,SAAS,IAAI,IAAI;oBACjB,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,gBAAgB,GAAG,mBAAmB,CAAC,CAAC,CAAC;gBAEjH,cAAc,GAAG,qBAAqB,CAAC;gBACvC,cAAc,GAAG,cAAc,CAAC,YAAY,CAAC;gBAE7C,IAAI,uBAAuB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACrC,6BAA6B,GAAG,IAAI,CAAC;oBACrC,MAAM,0BAA0B,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;oBACxE,MAAM,OAAO,CAAC,IAAI,CAAC;wBACf,0BAA0B;wBAC1B,IAAI,OAAO,CAAO,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE;4BACjC,eAAe,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;gCAClD,IAAI,0BAA0B,IAAI,CAAC,iBAAiB;oCAChD,MAAM,CAAC,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;;oCAEtC,MAAM,EAAE,CAAC;4BACjB,CAAC,CAAC,CAAC;4BAEH,IAAI,eAAe,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gCACjC,IAAI,0BAA0B,IAAI,CAAC,iBAAiB;oCAChD,MAAM,CAAC,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;;oCAEtC,MAAM,EAAE,CAAC;4BACjB,CAAC;wBACL,CAAC,CAAC;qBACL,CAAC,CAAC;oBACH,IAAI,CAAC,kBAAkB,EAAE,CAAC;oBAE1B,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;wBAClC,MAAM,mBAAmB,GAAG,CAAC,MAAM,0BAA0B,CAAC;6BACzD,MAAM,CAAC,CAAC,MAAM,EAA0C,EAAE,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC;wBAChF,IAAI,CAAC,kBAAkB,EAAE,CAAC;wBAE1B,IAAI,eAAe,CAAC,MAAM,CAAC,OAAO;4BAC9B,MAAM,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC;wBAExC,2BAA2B,GAAG,cAAc,CAAC,aAAa,CAAC;wBAE3D,KAAK,MAAM,EAAC,YAAY,EAAE,kBAAkB,EAAE,kBAAkB,EAAC,IAAI,mBAAmB,EAAE,CAAC;4BACvF,cAAc,GAAG,4BAA4B,CAAC;gCAC1C,WAAW,EAAE,cAAc;gCAC3B,YAAY,EAAE,YAAY,CAAC,YAAY;gCACvC,mBAAmB,EAAE,kBAAkB,CAAC,WAAW;gCACnD,UAAU,EAAE,YAAY,CAAC,MAAM;gCAC/B,UAAU,EAAE,kBAAkB;gCAC9B,OAAO,EAAE,YAAY,CAAC,GAAG;6BAC5B,CAAC,CAAC;4BAEH,2BAA2B,GAAG,4BAA4B,CAAC;gCACvD,WAAW,EAAE,2BAA2B;gCACxC,YAAY,EAAE,YAAY,CAAC,YAAY;gCACvC,mBAAmB,EAAE,kBAAkB,CAAC,WAAW;gCACnD,UAAU,EAAE,YAAY,CAAC,MAAM;gCAC/B,UAAU,EAAE,kBAAkB;gCAC9B,OAAO,EAAE,YAAY,CAAC,GAAG;6BAC5B,CAAC,CAAC;wBACP,CAAC;wBAED,cAAc,CAAC,YAAY,GAAG,cAAc,CAAC;wBAC7C,cAAc,CAAC,aAAa,GAAG,2BAA2B,CAAC;wBAE3D,SAAS;oBACb,CAAC;gBACL,CAAC;gBAED,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;gBACtC,IAAI,CAAC,YAAY,GAAG,cAAc,CAAC;gBACnC,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAC;gBAE/B,MAAM,qBAAqB,GAAG,wBAAwB,CAAC,cAAc,CAAC,CAAC;gBACvE,MAAM,YAAY,GAAG,qBAAqB,CAAC,QAAQ;qBAC9C,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC;qBAC1D,IAAI,CAAC,EAAE,CAAC,CAAC;gBAEd,IAAI,QAAQ,CAAC,UAAU,KAAK,mBAAmB;oBAC3C,OAAO;wBACH,QAAQ,EAAE,qBAAqB,CAAC,QAAQ;wBACxC,YAAY;wBACZ,UAAU,EAAE,QAAQ,CAAC,UAAU;wBAC/B,iBAAiB,EAAE,QAAQ,CAAC,iBAAiB;wBAC7C,4BAA4B,EAAE,QAAQ,CAAC,4BAA4B;qBACtE,CAAC;gBAEN,OAAO;oBACH,QAAQ,EAAE,qBAAqB,CAAC,QAAQ;oBACxC,YAAY;oBACZ,UAAU,EAAE,QAAQ,CAAC,UAAU;oBAC/B,4BAA4B,EAAE,QAAQ,CAAC,4BAA4B;iBACtE,CAAC;YACN,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;OAOG;IACI,KAAK,CAAC,aAAa,CAAC,MAAc,EAAE,UAAyC,EAAE;QAClF,MAAM,IAAI,CAAC,sBAAsB,CAAC,MAAM,EAAE;YACtC,GAAG,OAAO;YACV,SAAS,EAAE,CAAC;SACf,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;OASG;IACI,KAAK,CAAC,cAAc,CAAC,MAAc,EAAE,UAA0C,EAAE;QACpF,MAAM,EAAC,UAAU,EAAC,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAExE,OAAO,UAAU,CAAC;IACtB,CAAC;IAED;;;;;;OAMG;IACI,4BAA4B,CAAC,OAAgD;QAChF,OAAO,sCAAsC,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACzE,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,sBAAsB,CAAC,MAAc,EAAE,EAChD,SAAS,EACT,iBAAiB,GAAG,KAAK,EAEzB,SAAS,EACT,sBAAsB,EACtB,OAAO,EACP,MAAM,EACN,WAAW,EACX,IAAI,EACJ,IAAI,EACJ,IAAI,EACJ,OAAO,EACP,oBAAoB,GAAG,KAAK,EAC5B,aAAa,EACb,SAAS,EACT,kBAAkB,EAClB,kBAAkB,KACc,EAAE;QAClC,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE1B,IAAI,OAAO,IAAI,IAAI,IAAI,OAAO,CAAC,MAAM,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM;YACvD,MAAM,IAAI,KAAK,CAAC,wNAAwN,CAAC,CAAC;QAE9O,MAAM,eAAe,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;QAChD,IAAI,CAAC,mCAAmC,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAE9D,IAAI,CAAC;YACD,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,EAAE,eAAe,CAAC,MAAM,EAAE,KAAK,IAAI,EAAE;gBACnF,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBAE1B,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI;oBAClB,MAAM,IAAI,aAAa,EAAE,CAAC;gBAE9B,MAAM,EAAC,UAAU,EAAE,cAAc,EAAE,QAAQ,EAAC,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC,IAAI,CAAC,YAAY,EAAE;oBAC9G,iBAAiB,EAAE,MAAM;oBACzB,SAAS;oBACT,sBAAsB;oBACtB,OAAO;oBACP,OAAO;oBACP,MAAM,EAAE,eAAe,CAAC,MAAM;oBAC9B,iBAAiB,EAAE,IAAI;oBACvB,aAAa;oBACb,IAAI;oBACJ,IAAI;oBACJ,IAAI;oBACJ,SAAS;oBACT,kBAAkB;oBAClB,SAAS;oBACT,WAAW;oBACX,oBAAoB;oBACpB,YAAY,EAAE;wBACV,GAAG,IAAI,CAAC,aAAa;wBACrB,sBAAsB,EAAE,IAAI,CAAC,eAAe,EAAE,oBAAoB;qBACrE;oBACD,kBAAkB;oBAClB,2BAA2B,EAAE;wBACzB,OAAO,EAAE,IAAI,CAAC,eAAe,EAAE,aAAa;wBAC5C,6CAA6C,EAAE,GAAG;qBACrD;iBACJ,CAAC,CAAC;gBACH,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBAE1B,IAAI,CAAC,eAAe,GAAG;oBACnB,YAAY,EAAE,IAAI,CAAC,YAAY;oBAC/B,aAAa,EAAE,cAAc,CAAC,aAAa;oBAC3C,oBAAoB,EAAE,cAAc,CAAC,oBAAoB;iBAC5D,CAAC;gBAEF,IAAI,CAAC,iBAAiB,IAAI,QAAQ,CAAC,UAAU,KAAK,OAAO,IAAI,eAAe,CAAC,MAAM,EAAE,OAAO;oBACxF,MAAM,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC;gBAExC,IAAI,QAAQ,CAAC,UAAU,KAAK,mBAAmB;oBAC3C,OAAO;wBACH,UAAU,EAAE,UAAU;wBACtB,UAAU,EAAE,QAAQ,CAAC,UAAU;wBAC/B,iBAAiB,EAAE,QAAQ,CAAC,iBAAiB;wBAC7C,4BAA4B,EAAE,QAAQ,CAAC,4BAA4B;qBACtE,CAAC;gBAEN,OAAO;oBACH,UAAU,EAAE,UAAU;oBACtB,UAAU,EAAE,QAAQ,CAAC,UAAU;oBAC/B,4BAA4B,EAAE,QAAQ,CAAC,4BAA4B;iBACtE,CAAC;YACN,CAAC,CAAC,CAAC;QACP,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,mCAAmC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;QACrE,CAAC;IACL,CAAC;IAEM,cAAc;QACjB,OAAO,eAAe,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC9C,CAAC;IAEM,8BAA8B;QACjC,IAAI,IAAI,CAAC,eAAe,IAAI,IAAI;YAC5B,OAAO,IAAI,CAAC;QAEhB,OAAO,eAAe,CAAC,IAAI,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;IAChE,CAAC;IAEM,cAAc,CAAC,WAA8B;QAChD,IAAI,CAAC,YAAY,GAAG,eAAe,CAAC,WAAW,CAAC,CAAC;QACjD,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAC;QAC/B,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;IACrC,CAAC;IAED,gBAAgB;IACR,mCAAmC;QACvC,KAAK,MAAM,eAAe,IAAI,IAAI,CAAC,mCAAmC;YAClE,eAAe,CAAC,KAAK,EAAE,CAAC;QAE5B,IAAI,CAAC,mCAAmC,CAAC,KAAK,EAAE,CAAC;IACrD,CAAC;IAED,gBAAgB;IACR,kBAAkB;QACtB,IAAI,IAAI,CAAC,QAAQ;YACb,MAAM,IAAI,aAAa,EAAE,CAAC;IAClC,CAAC;CACJ;AAED,SAAS,4BAA4B,CAAC,EAClC,WAAW,EACX,YAAY,EACZ,mBAAmB,EACnB,UAAU,EACV,UAAU,EACV,OAAO,EAQV;IACG,MAAM,cAAc,GAAG,WAAW,CAAC,KAAK,EAAE,CAAC;IAC3C,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,IAAI,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO;QACzF,cAAc,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,OAAO;YACb,QAAQ,EAAE,EAAE;SACf,CAAC,CAAC;IAEP,MAAM,qBAAqB,GAAG,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAsB,CAAC;IAC7F,MAAM,wBAAwB,GAAG,EAAC,GAAG,qBAAqB,EAAC,CAAC;IAC5D,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,wBAAwB,CAAC;IAErE,MAAM,aAAa,GAAG,wBAAwB,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IAChE,wBAAwB,CAAC,QAAQ,GAAG,aAAa,CAAC;IAElD,aAAa,CAAC,IAAI,CAAC;QACf,IAAI,EAAE,cAAc;QACpB,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,mBAAmB;QAChC,MAAM,EAAE,UAAU;QAClB,MAAM,EAAE,UAAU;QAClB,OAAO;KACV,CAAC,CAAC;IAEH,OAAO,cAAc,CAAC;AAC1B,CAAC;AAED,SAAS,wBAAwB,CAAC,WAA8B;IAC5D,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,IAAI,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO;QAChF,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IAE1E,OAAO,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAsB,CAAC;AACpE,CAAC"}

package/dist/evaluator/LlamaChatSession/utils/LlamaChatSessionPromptCompletionEngine.d.ts

@@ -0,0 +1,39 @@
+import type { LLamaChatCompletePromptOptions } from "../LlamaChatSession.js";
+export type LLamaChatPromptCompletionEngineOptions = {
+    /**
+     * Max tokens to allow for preloading a prompt and generating a completion for it.
+     *
+     * Defaults to `256` or half of the context size, whichever is smaller.
+     */
+    maxPreloadTokens?: number;
+    onGeneration?(prompt: string, completion: string): void;
+    /**
+     * Max number of completions to cache.
+     *
+     * Defaults to `100`.
+     */
+    maxCachedCompletions?: number;
+    temperature?: LLamaChatCompletePromptOptions["temperature"];
+    minP?: LLamaChatCompletePromptOptions["minP"];
+    topK?: LLamaChatCompletePromptOptions["topK"];
+    topP?: LLamaChatCompletePromptOptions["topP"];
+    trimWhitespaceSuffix?: LLamaChatCompletePromptOptions["trimWhitespaceSuffix"];
+    evaluationPriority?: LLamaChatCompletePromptOptions["evaluationPriority"];
+    repeatPenalty?: LLamaChatCompletePromptOptions["repeatPenalty"];
+    tokenBias?: LLamaChatCompletePromptOptions["tokenBias"];
+    customStopTriggers?: LLamaChatCompletePromptOptions["customStopTriggers"];
+    grammar?: LLamaChatCompletePromptOptions["grammar"];
+    functions?: LLamaChatCompletePromptOptions["functions"];
+    documentFunctionParams?: LLamaChatCompletePromptOptions["documentFunctionParams"];
+};
+export declare class LlamaChatSessionPromptCompletionEngine {
+    private constructor();
+    dispose(): void;
+    /**
+     * Get completion for the prompt from the cache,
+     * and begin preloading this prompt into the context sequence and completing it.
+     *
+     * On completion progress, `onGeneration` (configured for this engine instance) will be called.
+     */
+    complete(prompt: string): string;
+}

package/dist/evaluator/LlamaChatSession/utils/LlamaChatSessionPromptCompletionEngine.js

@@ -0,0 +1,186 @@
+import { DisposeAggregator, DisposedError } from "lifecycle-utils";
+import { getConsoleLogPrefix } from "../../../utils/getConsoleLogPrefix.js";
+import { LruCache } from "../../../utils/LruCache.js";
+import { safeEventCallback } from "../../../utils/safeEventCallback.js";
+const defaultMaxPreloadTokens = 256;
+const defaultMaxCachedCompletions = 100;
+export class LlamaChatSessionPromptCompletionEngine {
+    /** @internal */ _chatSession;
+    /** @internal */ _maxPreloadTokens;
+    /** @internal */ _maxCachedCompletions;
+    /** @internal */ _onGeneration;
+    /** @internal */ _completionOptions;
+    /** @internal */ _completionCaches = new WeakMap();
+    /** @internal */ _disposeAggregator = new DisposeAggregator();
+    /** @internal */ _currentCompletionAbortController = new AbortController();
+    /** @internal */ _lastPrompt;
+    /** @internal */ _disposed = false;
+    constructor(chatSession, { maxPreloadTokens = defaultMaxPreloadTokens, onGeneration, maxCachedCompletions = defaultMaxCachedCompletions, ...options }) {
+        this._chatSession = chatSession;
+        this._maxPreloadTokens = Math.max(1, maxPreloadTokens);
+        this._maxCachedCompletions = Math.max(1, maxCachedCompletions);
+        this._onGeneration = safeEventCallback(onGeneration);
+        this._completionOptions = options;
+        this.dispose = this.dispose.bind(this);
+        this._disposeAggregator.add(this._chatSession.onDispose.createListener(this.dispose));
+        this._disposeAggregator.add(() => {
+            this._disposed = true;
+            this._currentCompletionAbortController.abort();
+        });
+    }
+    dispose() {
+        if (this._disposed)
+            return;
+        this._disposeAggregator.dispose();
+    }
+    /**
+     * Get completion for the prompt from the cache,
+     * and begin preloading this prompt into the context sequence and completing it.
+     *
+     * On completion progress, `onGeneration` (configured for this engine instance) will be called.
+     */
+    complete(prompt) {
+        if (this._disposed)
+            throw new DisposedError();
+        const completionCache = this._getCurrentCompletionCache();
+        const completion = completionCache.getCompletion(prompt);
+        if (this._lastPrompt == null || !(this._lastPrompt + (completion ?? "")).startsWith(prompt)) {
+            this._lastPrompt = prompt;
+            this._restartCompletion(completionCache);
+        }
+        this._lastPrompt = prompt;
+        return completion ?? "";
+    }
+    /** @internal */
+    _getCurrentCompletionCache() {
+        const completionCache = this._completionCaches.get(this._chatSession._chatHistoryStateRef);
+        if (completionCache != null)
+            return completionCache;
+        const newCompletionCache = new CompletionCache(this._maxCachedCompletions);
+        this._completionCaches.set(this._chatSession._chatHistoryStateRef, newCompletionCache);
+        return newCompletionCache;
+    }
+    /** @internal */
+    _restartCompletion(completionCache) {
+        if (this._disposed)
+            return;
+        this._currentCompletionAbortController.abort();
+        this._currentCompletionAbortController = new AbortController();
+        const prompt = this._lastPrompt;
+        if (prompt == null)
+            return;
+        const existingCompletion = completionCache.getCompletion(prompt);
+        const promptToComplete = prompt + (existingCompletion ?? "");
+        const currentPromptTokens = this._chatSession.model.tokenize(promptToComplete, false, "trimLeadingSpace").length;
+        const leftTokens = Math.max(0, this._maxPreloadTokens - currentPromptTokens);
+        if (leftTokens === 0)
+            return;
+        const currentAbortController = this._currentCompletionAbortController;
+        const currentAbortSignal = this._currentCompletionAbortController.signal;
+        const currentCompletion = [];
+        void this._chatSession.completePrompt(promptToComplete, {
+            ...this._completionOptions,
+            stopOnAbortSignal: false,
+            maxTokens: leftTokens,
+            signal: currentAbortSignal,
+            onToken: (chunk) => {
+                currentCompletion.push(...chunk);
+                const completion = (existingCompletion ?? "") + this._chatSession.model.detokenize(currentCompletion);
+                completionCache.putCompletion(prompt, completion);
+                if (this._getCurrentCompletionCache() !== completionCache) {
+                    currentAbortController.abort();
+                    return;
+                }
+                if (this._lastPrompt === prompt)
+                    this._onGeneration?.(prompt, completion);
+            }
+        })
+            .then(() => {
+            if (this._lastPrompt !== prompt && this._getCurrentCompletionCache() === completionCache)
+                return this._restartCompletion(completionCache);
+        })
+            .catch((err) => {
+            if (currentAbortSignal.aborted && err === currentAbortSignal.reason)
+                return;
+            console.error(getConsoleLogPrefix(false, false), err);
+        });
+    }
+    /** @internal */
+    static _create(chatSession, options = {}) {
+        return new LlamaChatSessionPromptCompletionEngine(chatSession, options);
+    }
+}
+class CompletionCache {
+    /** @internal */ _cache;
+    /** @internal */ _rootNode = [new Map()];
+    constructor(maxInputs) {
+        this._cache = new LruCache(maxInputs, {
+            onDelete: (key) => {
+                this._deleteInput(key);
+            }
+        });
+    }
+    get maxInputs() {
+        return this._cache.maxSize;
+    }
+    getCompletion(input) {
+        let node = this._rootNode;
+        for (let i = 0; i < input.length; i++) {
+            if (node == null)
+                return null;
+            const [next, completion] = node;
+            const char = input[i];
+            if (!next.has(char)) {
+                if (completion != null && completion.startsWith(input.slice(i))) {
+                    this._cache.get(input.slice(0, i));
+                    return completion.slice(input.length - i);
+                }
+            }
+            node = next.get(char);
+        }
+        if (node == null)
+            return null;
+        const [, possibleCompletion] = node;
+        if (possibleCompletion != null) {
+            this._cache.get(input);
+            return possibleCompletion;
+        }
+        return null;
+    }
+    putCompletion(input, completion) {
+        this._cache.set(input, null);
+        let node = this._rootNode;
+        for (let i = 0; i < input.length; i++) {
+            const [next] = node;
+            const char = input[i];
+            if (!next.has(char))
+                next.set(char, [new Map()]);
+            node = next.get(char);
+        }
+        const currentCompletion = node[1];
+        if (currentCompletion != null && currentCompletion.startsWith(completion))
+            return currentCompletion;
+        node[1] = completion;
+        return completion;
+    }
+    /** @internal */
+    _deleteInput(input) {
+        let lastNodeWithMultipleChildren = this._rootNode;
+        let lastNodeWithMultipleChildrenDeleteChar = input[0];
+        let node = this._rootNode;
+        for (let i = 0; i < input.length; i++) {
+            const [next] = node;
+            const char = input[i];
+            if (next.size > 1) {
+                lastNodeWithMultipleChildren = node;
+                lastNodeWithMultipleChildrenDeleteChar = char;
+            }
+            if (!next.has(char))
+                return;
+            node = next.get(char);
+        }
+        if (lastNodeWithMultipleChildrenDeleteChar !== "")
+            lastNodeWithMultipleChildren[0].delete(lastNodeWithMultipleChildrenDeleteChar);
+    }
+}
+//# sourceMappingURL=LlamaChatSessionPromptCompletionEngine.js.map