node-llama-cpp 2.8.0 → 3.0.0-beta.1

package/dist/llamaEvaluator/LlamaContext/LlamaContext.js

@@ -0,0 +1,640 @@
+import { DisposeAggregator, EventRelay, withLock, DisposedError } from "lifecycle-utils";
+import { removeNullFields } from "../../utils/removeNullFields.js";
+import { AddonContext } from "../LlamaBins.js";
+import { resolveBatchItemsPrioritizingStrategy } from "./utils/resolveBatchItemsPrioritizingStrategy.js";
+export class LlamaContext {
+    /** @internal */ _ctx;
+    /** @internal */ _onReclaimUnusedSequenceId = new EventRelay();
+    /** @internal */ _model;
+    /** @internal */ _contextSize;
+    /** @internal */ _batchSize;
+    /** @internal */ _totalSequences;
+    /** @internal */ _unusedSequenceIds = [];
+    /** @internal */ _batchingOptions;
+    /** @internal */ _queuedDecodeSequenceIds = new Set();
+    /** @internal */ _queuedDecodes = [];
+    /** @internal */ _disposeAggregator = new DisposeAggregator();
+    /** @internal */ _nextGeneratedSequenceId = 0;
+    /** @internal */ _dispatchDecodeScheduled = false;
+    /** @internal */ _batchDispatchPending = false;
+    /** @internal */ _currentDispatchBatchHandle = {};
+    /** @internal */ _allocatedContextSize;
+    /** @internal */ _disposed = false;
+    onDispose = new EventRelay();
+    /**
+     * @param options
+     */
+    constructor({ model, sequences = 1, seed = null, contextSize = model.trainContextSize, batchSize = contextSize, f16Kv, logitsAll, embedding, threads = 6, batching: { dispatchSchedule: batchingDispatchSchedule = "nextTick", itemsPrioritizingStrategy: batchingItemsPrioritizingStrategy = "maximumParallelism" } = {} }) {
+        if (model.disposed)
+            throw new DisposedError();
+        this._model = model;
+        this._totalSequences = Math.max(1, Math.floor(sequences));
+        this._contextSize = Math.max(2, contextSize);
+        this._batchSize = Math.max(batchSize, this._totalSequences);
+        this._ctx = new AddonContext(this._model._model, removeNullFields({
+            seed: seed != null ? Math.max(-1, Math.floor(seed)) : undefined,
+            contextSize: contextSize * this._totalSequences,
+            batchSize: this._batchSize,
+            f16Kv,
+            logitsAll,
+            embedding,
+            threads
+        }));
+        this._batchingOptions = {
+            dispatchSchedule: batchingDispatchSchedule,
+            itemsPrioritizingStrategy: batchingItemsPrioritizingStrategy
+        };
+        this._reclaimUnusedSequenceId = this._reclaimUnusedSequenceId.bind(this);
+        this._disposeAggregator.add(this._onReclaimUnusedSequenceId);
+        this._disposeAggregator.add(this.onDispose.dispatchEvent);
+        this._disposeAggregator.add(() => {
+            this._ctx.dispose();
+        });
+        this._disposeAggregator.add(this.model.onDispose.createListener(disposeContextIfReferenced.bind(null, new WeakRef(this))));
+    }
+    dispose() {
+        if (this._disposed)
+            return;
+        this._disposed = true;
+        this._disposeAggregator.dispose();
+    }
+    /** @hidden */
+    [Symbol.dispose]() {
+        return this.dispose();
+    }
+    get disposed() {
+        return this._disposed;
+    }
+    get model() {
+        return this._model;
+    }
+    get contextSize() {
+        return this._contextSize;
+    }
+    get batchSize() {
+        return this._batchSize;
+    }
+    getAllocatedContextSize() {
+        this._ensureNotDisposed();
+        if (this._allocatedContextSize == null)
+            this._allocatedContextSize = this._ctx.getContextSize();
+        return this._allocatedContextSize;
+    }
+    get totalSequences() {
+        return this._totalSequences;
+    }
+    get sequencesLeft() {
+        return this._totalSequences - this._nextGeneratedSequenceId + this._unusedSequenceIds.length;
+    }
+    /**
+     * Before calling this method, make sure to call `sequencesLeft` to check if there are any sequences left.
+     * When there are no sequences left, this method will throw an error.
+     * @param [options]
+     */
+    getSequence({ prependBos = true, contextShift: { size: contextShiftSize = Math.min(100, Math.ceil(this.contextSize / 2)), strategy: contextShiftStrategy = "eraseBeginning" } = {} } = {}) {
+        this._ensureNotDisposed();
+        const nextSequenceId = this._popSequenceId();
+        if (nextSequenceId == null)
+            throw new Error("No sequences left");
+        return LlamaContextSequence._create({
+            sequenceId: nextSequenceId,
+            context: this,
+            prependBos,
+            contextShift: {
+                size: contextShiftSize,
+                strategy: contextShiftStrategy
+            }
+        });
+    }
+    dispatchPendingBatch() {
+        this._currentDispatchBatchHandle = {};
+        this._dispatchDecodeScheduled = false;
+        if (this._batchDispatchPending)
+            return;
+        this._batchDispatchPending = true;
+        void withLock(this, "context", async () => {
+            this._currentDispatchBatchHandle = {};
+            this._dispatchDecodeScheduled = false;
+            this._batchDispatchPending = false;
+            let prioritizeStrategy;
+            try {
+                this._ensureNotDisposed();
+                prioritizeStrategy = resolveBatchItemsPrioritizingStrategy(this._batchingOptions.itemsPrioritizingStrategy);
+            }
+            catch (err) {
+                this._dispatchErrorForQueuedDecodesAndDequeue(new Set(this._queuedDecodes), err);
+                return;
+            }
+            let shouldHaveAnotherBatch = this._queuedDecodes.length > 0;
+            while (shouldHaveAnotherBatch) {
+                const batchItemToQueuedDecodeMap = new Map();
+                const batchItemsList = [];
+                for (const queuedDecode of this._queuedDecodes) {
+                    const batchItem = {
+                        tokens: queuedDecode.tokens,
+                        evaluationPriority: queuedDecode.evaluationPriority
+                    };
+                    batchItemToQueuedDecodeMap.set(batchItem, queuedDecode);
+                    batchItemsList.push(batchItem);
+                }
+                let prioritizedItems;
+                try {
+                    prioritizedItems = prioritizeStrategy({
+                        items: batchItemsList,
+                        size: this._batchSize
+                    });
+                }
+                catch (err) {
+                    this._dispatchErrorForQueuedDecodesAndDequeue(new Set(this._queuedDecodes), err);
+                    return;
+                }
+                let batchTokenSlotsLeft = this._batchSize;
+                const afterDecodeActions = [];
+                const queuedDecodesToDelete = new Set();
+                const currentQueuedDecodeItems = new Set();
+                const currentBatchItems = [];
+                let currentBatchSize = 0;
+                for (const prioritizedItem of prioritizedItems) {
+                    const queuedDecode = batchItemToQueuedDecodeMap.get(prioritizedItem.item);
+                    if (queuedDecode == null)
+                        throw new Error("Received invalid batch item. Make sure you keep the original object reference " +
+                            "of the batch item on `item` on `PrioritizedBatchItem` in your custom prioritization strategy");
+                    const processAmount = Math.min(queuedDecode.tokens.length, prioritizedItem.processAmount, batchTokenSlotsLeft);
+                    if (processAmount <= 0)
+                        continue;
+                    batchTokenSlotsLeft -= processAmount;
+                    currentBatchItems.push({
+                        queuedDecode,
+                        processAmount
+                    });
+                    currentBatchSize += processAmount;
+                }
+                if (currentBatchSize !== 0)
+                    this._ctx.initBatch(currentBatchSize);
+                for (const { queuedDecode, processAmount } of currentBatchItems) {
+                    let batchLogitIndex;
+                    try {
+                        batchLogitIndex = this._ctx.addToBatch(queuedDecode.sequenceId, queuedDecode.firstTokenSequenceIndex, Uint32Array.from(queuedDecode.tokens.slice(0, processAmount)), queuedDecode.generateLogitAtTheEnd && processAmount === queuedDecode.tokens.length);
+                    }
+                    catch (err) {
+                        this._dispatchErrorForQueuedDecodesAndDequeue(new Set([queuedDecode]), err);
+                        continue;
+                    }
+                    currentQueuedDecodeItems.add(queuedDecode);
+                    if (queuedDecode.tokens.length === processAmount) {
+                        queuedDecodesToDelete.add(queuedDecode);
+                        afterDecodeActions.push({
+                            batchLogitIndex,
+                            response: queuedDecode.response,
+                            onDone: queuedDecode.onDone
+                        });
+                    }
+                    else {
+                        queuedDecode.tokens = queuedDecode.tokens.slice(processAmount);
+                        queuedDecode.firstTokenSequenceIndex += processAmount;
+                    }
+                    if (batchTokenSlotsLeft === 0)
+                        break;
+                }
+                for (let i = 0; i < this._queuedDecodes.length; i++) {
+                    const queuedDecode = this._queuedDecodes[i];
+                    if (queuedDecodesToDelete.has(queuedDecode)) {
+                        this._queuedDecodes.splice(i, 1);
+                        this._queuedDecodeSequenceIds.delete(queuedDecode.sequenceId);
+                        i--;
+                    }
+                }
+                shouldHaveAnotherBatch = this._queuedDecodes.length > 0;
+                try {
+                    if (currentBatchSize !== 0)
+                        await this._ctx.decodeBatch();
+                }
+                catch (err) {
+                    this._dispatchErrorForQueuedDecodesAndDequeue(currentQueuedDecodeItems, err);
+                    return;
+                }
+                for (const action of afterDecodeActions) {
+                    const [accept, reject] = action.response;
+                    if (action.onDone != null && action.batchLogitIndex != null) {
+                        try {
+                            accept(action.onDone(action.batchLogitIndex ?? null));
+                        }
+                        catch (err) {
+                            reject(err);
+                        }
+                    }
+                    accept(undefined);
+                }
+            }
+        });
+    }
+    /** @internal */
+    async _decodeTokens({ sequenceId, firstTokenSequenceIndex, tokens, generateLogitAtTheEnd = false, evaluationPriority = 5 }, onDone) {
+        return await new Promise((accept, reject) => {
+            this._queuedDecodes.push({
+                sequenceId,
+                tokens,
+                firstTokenSequenceIndex,
+                generateLogitAtTheEnd,
+                evaluationPriority,
+                response: [accept, reject],
+                onDone
+            });
+            this._queuedDecodeSequenceIds.add(sequenceId);
+            this._scheduleDecode();
+        });
+    }
+    /** @internal */
+    _reclaimUnusedSequenceId(sequenceId) {
+        if (this._disposed)
+            return;
+        void withLock(this, "context", async () => {
+            this._ctx.disposeSequence(sequenceId);
+            this._unusedSequenceIds.push(sequenceId);
+            this._onReclaimUnusedSequenceId.dispatchEvent();
+        });
+    }
+    /** @internal */
+    _popSequenceId() {
+        if (this._unusedSequenceIds.length > 0)
+            return this._unusedSequenceIds.shift();
+        if (this._nextGeneratedSequenceId < this._totalSequences) {
+            const sequenceId = this._nextGeneratedSequenceId;
+            this._nextGeneratedSequenceId++;
+            return sequenceId;
+        }
+        return null;
+    }
+    /** @internal */
+    _scheduleDecode() {
+        if (this._dispatchDecodeScheduled || this._batchDispatchPending)
+            return;
+        this._dispatchDecodeScheduled = true;
+        const currentPendingBatchHandle = this._currentDispatchBatchHandle;
+        const dispatch = () => {
+            if (this._currentDispatchBatchHandle !== currentPendingBatchHandle)
+                return;
+            this.dispatchPendingBatch();
+        };
+        const dispatchSchedule = this._batchingOptions.dispatchSchedule;
+        if (this._queuedDecodeSequenceIds.size === this._totalSequences)
+            dispatch();
+        if (dispatchSchedule === "nextTick")
+            setTimeout(dispatch, 0);
+        else
+            dispatchSchedule(dispatch);
+    }
+    /** @internal */
+    _dispatchErrorForQueuedDecodesAndDequeue(queuedDecodes, err) {
+        for (const pendingDecode of queuedDecodes) {
+            const [, reject] = pendingDecode.response;
+            reject(err);
+        }
+        for (let i = 0; i < this._queuedDecodes.length; i++) {
+            const item = this._queuedDecodes[i];
+            if (queuedDecodes.has(item)) {
+                this._queuedDecodes.splice(i, 1);
+                this._queuedDecodeSequenceIds.delete(item.sequenceId);
+                i--;
+            }
+        }
+    }
+    /** @internal */
+    _ensureNotDisposed() {
+        if (this._disposed)
+            throw new DisposedError();
+    }
+}
+export class LlamaContextSequence {
+    /** @internal */ _sequenceId;
+    /** @internal */ _gcRegistry;
+    /** @internal */ _context;
+    /** @internal */ _prependBos;
+    /** @internal */ _contextShift;
+    /** @internal */ _disposeAggregator = new DisposeAggregator();
+    /** @internal */ _contextTokens = [];
+    /** @internal */ _contextTokenPriorities = [];
+    /** @internal */ _nextTokenIndex = 0;
+    /** @internal */ _prependTokens = [];
+    /** @internal */ _prependTokenPriorities = [];
+    /** @internal */ _disposed = false;
+    onDispose = new EventRelay();
+    constructor({ sequenceId, context, prependBos, contextShift }) {
+        this._sequenceId = sequenceId;
+        this._context = context;
+        this._contextShift = contextShift;
+        this._gcRegistry = new FinalizationRegistry(this._context._reclaimUnusedSequenceId);
+        this._prependTokens = [];
+        this._prependTokenPriorities = [];
+        if (prependBos && this._context.model.tokens.bos != null) {
+            this._prependTokens.unshift(this._context.model.tokens.bos);
+            this._prependTokenPriorities.unshift(1);
+            this._prependBos = true;
+        }
+        else
+            this._prependBos = false;
+        this._gcRegistry.register(this, sequenceId);
+        this._disposeAggregator.add(() => this._gcRegistry.unregister(this));
+        this._disposeAggregator.add(this.onDispose.dispatchEvent);
+        this._disposeAggregator.add(this.model.onDispose.createListener(disposeContextSequenceIfReferenced.bind(null, new WeakRef(this))));
+        this._disposeAggregator.add(() => {
+            this._context._reclaimUnusedSequenceId(this._sequenceId);
+        });
+    }
+    dispose() {
+        if (this._disposed)
+            return;
+        this._disposeAggregator.dispose();
+        this._contextTokens.length = 0;
+        this._contextTokenPriorities.length = 0;
+        this._disposed = true;
+    }
+    /** @hidden */
+    [Symbol.dispose]() {
+        return this.dispose();
+    }
+    get disposed() {
+        return this._disposed;
+    }
+    get context() {
+        return this._context;
+    }
+    get model() {
+        return this._context.model;
+    }
+    get prependBos() {
+        return this._prependBos;
+    }
+    get nextTokenIndex() {
+        return this._nextTokenIndex;
+    }
+    get contextTokens() {
+        return this._contextTokens.slice();
+    }
+    get contextTokenPriorities() {
+        return this._contextTokenPriorities.slice();
+    }
+    /**
+     * Clear the history of the sequence.
+     * If `prependBos` was enabled, the BOS token will be prepended to the sequence again.
+     */
+    async clearHistory() {
+        this._ensureNotDisposed();
+        await this.eraseContextTokenRanges([{ start: 0, end: this._nextTokenIndex }]);
+        this._prependTokens.length = 0;
+        this._prependTokenPriorities.length = 0;
+        if (this._prependBos && this._context.model.tokens.bos != null) {
+            this._prependTokens.push(this._context.model.tokens.bos);
+            this._prependTokenPriorities.push(1);
+        }
+    }
+    /**
+     * Erase context tokens in the provided ranges to free up space for new tokens to be generated.
+     * the start and end of each range are exclusive.
+     * For example, the range `{start: 0, end: 1}` will remove the token at the `0` index only.
+     */
+    async eraseContextTokenRanges(ranges) {
+        this._ensureNotDisposed();
+        await withLock(this._context, "context", async () => {
+            this._ensureNotDisposed();
+            if (ranges.length === 0)
+                return;
+            const resolvedRanges = ranges
+                .map(({ start, end }) => {
+                if (start === end)
+                    return null;
+                if (start > end)
+                    [start, end] = [end, start];
+                if (end > this._nextTokenIndex)
+                    end = this._nextTokenIndex;
+                if (start >= this._nextTokenIndex)
+                    return null;
+                return { start, end };
+            })
+                .filter((range) => range != null)
+                .sort((a, b) => a.start - b.start)
+                .reduce((ranges, range) => {
+                if (ranges.length === 0)
+                    return [range];
+                const lastRange = ranges[ranges.length - 1];
+                if (lastRange.end >= range.start) {
+                    lastRange.end = Math.max(lastRange.end, range.end);
+                    return ranges;
+                }
+                ranges.push(range);
+                return ranges;
+            }, []);
+            let removedTokens = 0;
+            let lastDeleteRangeEndPos = null;
+            for (const range of resolvedRanges) {
+                this._contextTokens.splice(range.start - removedTokens, range.end - range.start);
+                this._contextTokenPriorities.splice(range.start - removedTokens, range.end - range.start);
+                this._context._ctx.removeTokenCellsFromSequence(this._sequenceId, range.start, range.end);
+                if (lastDeleteRangeEndPos != null && removedTokens > 0 && lastDeleteRangeEndPos !== range.start)
+                    this._context._ctx.shiftSequenceTokenCells(this._sequenceId, lastDeleteRangeEndPos, range.start, -removedTokens);
+                removedTokens += range.end - range.start;
+                lastDeleteRangeEndPos = range.end;
+            }
+            if (lastDeleteRangeEndPos != null && removedTokens > 0 && lastDeleteRangeEndPos !== this._nextTokenIndex)
+                this._context._ctx.shiftSequenceTokenCells(this._sequenceId, lastDeleteRangeEndPos, this._nextTokenIndex, -removedTokens);
+            this._nextTokenIndex -= removedTokens;
+        });
+    }
+    /**
+     * @param tokens
+     * @param [options]
+     */
+    evaluate(tokens, { temperature = 0, topK = 40, topP = 0.95, grammarEvaluationState, repeatPenalty, evaluationPriority = 5, tokenPriority = 1 } = {}) {
+        return this._evaluate(tokens, {
+            temperature,
+            topK,
+            topP,
+            grammarEvaluationState,
+            repeatPenalty,
+            evaluationPriority,
+            tokenPriority
+        });
+    }
+    /**
+     * Evaluate the provided tokens into the context sequence without generating new tokens.
+     * @param tokens
+     * @param [options]
+     */
+    async evaluateWithoutGeneratingNewTokens(tokens, { evaluationPriority = 5, tokenPriority = 1 } = {}) {
+        const iterator = this._evaluate(tokens, {
+            generateNewTokens: false,
+            evaluationPriority,
+            tokenPriority
+        });
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        for await (const token of iterator) {
+            // Array.from doesn't work with async generators, so we have to iterate over the generator
+        }
+    }
+    /** @internal */
+    async *_evaluate(tokens, { temperature = 0, topK = 40, topP = 0.95, grammarEvaluationState, repeatPenalty, evaluationPriority = 5, tokenPriority = 1, generateNewTokens = true } = {}) {
+        this._ensureNotDisposed();
+        if (!(tokenPriority instanceof Array))
+            tokenPriority = Array(tokens.length).fill(tokenPriority);
+        let evalTokens = tokens;
+        if (generateNewTokens && tokens.length === 0)
+            return;
+        if (this._prependTokens.length > 0) {
+            evalTokens = this._prependTokens.concat(tokens);
+            tokenPriority = this._prependTokenPriorities.concat(tokenPriority);
+            this._prependTokens = [];
+            this._prependTokenPriorities = [];
+        }
+        if (evalTokens.length === 0)
+            return;
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+            this._ensureNotDisposed();
+            // Evaluate to get the next token.
+            const nextToken = await this._decodeTokens(evalTokens, generateNewTokens, evaluationPriority, tokenPriority, (batchLogitIndex) => {
+                return this._context._ctx.sampleToken(batchLogitIndex, removeNullFields({
+                    temperature,
+                    topK,
+                    topP,
+                    repeatPenalty: repeatPenalty?.penalty,
+                    repeatPenaltyTokens: repeatPenalty?.punishTokens instanceof Function
+                        ? repeatPenalty.punishTokens()
+                        : repeatPenalty?.punishTokens,
+                    repeatPenaltyPresencePenalty: repeatPenalty?.presencePenalty,
+                    repeatPenaltyFrequencyPenalty: repeatPenalty?.frequencyPenalty,
+                    grammarEvaluationState: grammarEvaluationState?._state
+                }));
+            });
+            if (nextToken == null)
+                return;
+            // the model finished generating text
+            if (nextToken === this._context.model.tokens.eos)
+                break;
+            yield nextToken;
+            // Create tokens for the next eval.
+            evalTokens = [nextToken];
+        }
+    }
+    /** @internal */
+    async _decodeTokens(tokens, generateLogit, evaluationPriority, tokenPriority, onDecodeDone) {
+        this._ensureNotDisposed();
+        const tokensLeftToDecode = tokens.slice();
+        return await withLock(this, "evaluate", async () => {
+            while (tokensLeftToDecode.length > 0) {
+                this._ensureNotDisposed();
+                let freeSpace = this._context.contextSize - this._nextTokenIndex;
+                if (freeSpace === 0) {
+                    await this._freeUpSpaceForTokens();
+                    freeSpace = this._context.contextSize - this._nextTokenIndex;
+                    if (freeSpace === 0)
+                        throw new Error("Failed to free up space for new tokens");
+                }
+                const tokensToDecode = tokensLeftToDecode.splice(0, freeSpace);
+                const generateLogitAtTheEnd = generateLogit && tokensLeftToDecode.length === 0;
+                const nextToken = await this._context._decodeTokens({
+                    sequenceId: this._sequenceId,
+                    tokens: tokensToDecode,
+                    firstTokenSequenceIndex: this._nextTokenIndex,
+                    generateLogitAtTheEnd,
+                    evaluationPriority
+                }, !generateLogitAtTheEnd
+                    ? undefined
+                    : onDecodeDone);
+                this._nextTokenIndex += tokensToDecode.length;
+                this._contextTokens = this._contextTokens.concat(tokensToDecode);
+                for (let i = 0; i < tokensToDecode.length; i++) {
+                    this._contextTokenPriorities.push(tokenPriority[i] ?? tokenPriority[tokenPriority.length - 1] ?? 1);
+                }
+                if (generateLogitAtTheEnd && nextToken != null)
+                    return nextToken;
+            }
+            return null;
+        });
+    }
+    /** @internal */
+    async _freeUpSpaceForTokens() {
+        this._ensureNotDisposed();
+        const size = Math.min(this._nextTokenIndex, Math.max(1, this._contextShift.size instanceof Function
+            ? await this._contextShift.size(this)
+            : this._contextShift.size));
+        this._ensureNotDisposed();
+        if (this._contextShift.strategy === "eraseLowestTokenPriorityBeginning") {
+            let leftTokensToErase = Math.min(size, this._contextTokenPriorities.length);
+            let previousLowestPriority = null;
+            let indexesToErase = [];
+            while (leftTokensToErase > 0 && this._contextTokenPriorities.length > 0) {
+                let currentLowestPriorityIndexes = [];
+                let currentLowestPriorityFound = null;
+                for (let i = 0; leftTokensToErase > currentLowestPriorityIndexes.length && i < this._contextTokenPriorities.length; i++) {
+                    const tokenPriority = this._contextTokenPriorities[i];
+                    if (currentLowestPriorityFound == null || (tokenPriority < currentLowestPriorityFound && (previousLowestPriority == null || tokenPriority > previousLowestPriority))) {
+                        currentLowestPriorityFound = tokenPriority;
+                        currentLowestPriorityIndexes = [i];
+                    }
+                    else if (tokenPriority === currentLowestPriorityFound) {
+                        currentLowestPriorityIndexes.push(i);
+                    }
+                }
+                previousLowestPriority = currentLowestPriorityFound;
+                indexesToErase = indexesToErase.concat(currentLowestPriorityIndexes);
+                leftTokensToErase -= currentLowestPriorityIndexes.length;
+            }
+            await this.eraseContextTokenRanges(indexesToErase.reduce((ranges, index) => {
+                if (ranges.length === 0)
+                    return [{ start: index, end: index + 1 }];
+                const lastRange = ranges[ranges.length - 1];
+                if (lastRange.end >= index) {
+                    lastRange.end = Math.max(lastRange.end, index + 1);
+                    return ranges;
+                }
+                ranges.push({ start: index, end: index + 1 });
+                return ranges;
+            }, []));
+        }
+        else if (this._contextShift.strategy === "eraseBeginning") {
+            await this.eraseContextTokenRanges([{ start: 0, end: size }]);
+        }
+        else {
+            const ranges = await this._contextShift.strategy({
+                sequence: this,
+                size
+            });
+            if (ranges == null)
+                throw new Error("Invalid delete ranges");
+            await this.eraseContextTokenRanges(ranges);
+            if (this.nextTokenIndex >= this._context.contextSize)
+                await this.eraseContextTokenRanges([{ start: 0, end: size }]);
+        }
+    }
+    /** @internal */
+    _ensureNotDisposed() {
+        if (this._disposed)
+            throw new DisposedError();
+    }
+    /**
+     * We need this to make it impossible to manually create instances of this class outside the code of this library
+     * @internal
+     */
+    static _create({ sequenceId, context, prependBos = true, contextShift: { size: contextShiftSize = Math.min(100, Math.ceil(context.contextSize / 2)), strategy: contextShiftStrategy = "eraseBeginning" } = {} }) {
+        return new LlamaContextSequence({
+            sequenceId,
+            context,
+            prependBos,
+            contextShift: {
+                size: contextShiftSize,
+                strategy: contextShiftStrategy
+            }
+        });
+    }
+}
+function disposeContextIfReferenced(contextRef) {
+    const context = contextRef.deref();
+    if (context != null)
+        context.dispose();
+}
+function disposeContextSequenceIfReferenced(contextRef) {
+    const context = contextRef.deref();
+    if (context != null)
+        context.dispose();
+}
+//# sourceMappingURL=LlamaContext.js.map