node-llama-cpp 3.0.0-beta.9 → 3.0.1

package/dist/utils/ThreadsSplitter.js

@@ -0,0 +1,177 @@
+import { DisposedError, DisposableHandle } from "lifecycle-utils";
+export class ThreadsSplitter {
+    _threadDemands = new MaxNumberCollection();
+    _threadFreeCallbacks = [];
+    _activeThreads = 0;
+    _totalWantedThreads = 0;
+    maxThreads;
+    /**
+     * Set to `0` to disable the limit
+     * @param maxThreads
+     */
+    constructor(maxThreads) {
+        this.maxThreads = Math.floor(Math.max(0, maxThreads));
+        this._removeWantedThreads = this._removeWantedThreads.bind(this);
+        this._removeThreadDemand = this._removeThreadDemand.bind(this);
+    }
+    createConsumer(wantedThreads, minThreads = 1) {
+        if (wantedThreads !== 0 && minThreads > wantedThreads)
+            minThreads = wantedThreads;
+        if (this.maxThreads !== 0 && wantedThreads === 0)
+            wantedThreads = this.maxThreads;
+        return new ThreadsSplitterConsumer(this, wantedThreads, minThreads);
+    }
+    normalizeThreadsValue(threads) {
+        if (this.maxThreads === 0)
+            return Math.floor(Math.max(0, threads));
+        return Math.floor(Math.max(0, Math.min(this.maxThreads, threads)));
+    }
+    /** @internal */
+    _getUpdatedActiveThreads(inUsed, wanted, demanded) {
+        const initialActiveThreads = this._activeThreads;
+        if (inUsed > wanted)
+            this._activeThreads -= inUsed - wanted;
+        const idealThreads = this._calculateIdealProportion(wanted, demanded);
+        let allocatedThreads = Math.min(inUsed, wanted); // already allocated
+        if (allocatedThreads === idealThreads) {
+            this._callOnActiveThreadsFreeIfCan(initialActiveThreads);
+            return idealThreads;
+        }
+        if (allocatedThreads > idealThreads) {
+            this._activeThreads -= allocatedThreads - idealThreads;
+            this._callOnActiveThreadsFreeIfCan(initialActiveThreads);
+            return idealThreads;
+        }
+        const neededThreads = idealThreads - allocatedThreads;
+        const availableThreads = this.maxThreads - this._activeThreads;
+        if (neededThreads <= availableThreads) {
+            this._activeThreads += neededThreads;
+            this._callOnActiveThreadsFreeIfCan(initialActiveThreads);
+            return idealThreads;
+        }
+        allocatedThreads += availableThreads;
+        this._activeThreads += availableThreads;
+        this._callOnActiveThreadsFreeIfCan(initialActiveThreads);
+        return allocatedThreads;
+    }
+    _callOnActiveThreadsFreeIfCan(lastActiveThreads) {
+        if (this._activeThreads >= lastActiveThreads)
+            return;
+        while (this._threadFreeCallbacks.length > 0)
+            this._threadFreeCallbacks.shift()?.();
+    }
+    _calculateIdealProportion(wantedThreads, demandedThreads) {
+        return Math.min(wantedThreads, Math.max(demandedThreads, Math.ceil((wantedThreads / this._totalWantedThreads) *
+            Math.max(1, this.maxThreads - (Math.max(demandedThreads, this._threadDemands.maxNumber) - demandedThreads)))));
+    }
+    /** @internal */
+    _waitForFreeThread() {
+        return new Promise(resolve => this._threadFreeCallbacks.push(resolve));
+    }
+    /** @internal */
+    _addWantedThreads(wantedThreads) {
+        this._totalWantedThreads += wantedThreads;
+    }
+    /** @internal */
+    _removeWantedThreads(wantedThreads) {
+        this._totalWantedThreads -= wantedThreads;
+    }
+    /** @internal */
+    _addThreadDemand(demandedThreads) {
+        this._threadDemands.add(demandedThreads);
+    }
+    /** @internal */
+    _removeThreadDemand(demandedThreads) {
+        const isHighestDemand = this._threadDemands.maxNumber === demandedThreads;
+        this._threadDemands.remove(demandedThreads);
+        if (demandedThreads !== 0 && isHighestDemand && this._threadDemands.maxNumber !== demandedThreads) {
+            while (this._threadFreeCallbacks.length > 0)
+                this._threadFreeCallbacks.shift()?.();
+        }
+    }
+}
+export class ThreadsSplitterConsumer {
+    _threadsSplitter;
+    _wantedThreads;
+    _demandedThreads;
+    _wantedThreadsGcRegistry;
+    _demandedThreadsGcRegistry;
+    _usedThreads = 0;
+    _disposed = false;
+    constructor(threadsSplitter, wantedThreads, minThreads) {
+        this._threadsSplitter = threadsSplitter;
+        this._wantedThreads = wantedThreads;
+        this._demandedThreads = minThreads;
+        this._threadsSplitter._addWantedThreads(this._wantedThreads);
+        this._threadsSplitter._addThreadDemand(this._demandedThreads);
+        this._wantedThreadsGcRegistry = new FinalizationRegistry(this._threadsSplitter._removeWantedThreads);
+        this._wantedThreadsGcRegistry.register(this, this._wantedThreads);
+        this._demandedThreadsGcRegistry = new FinalizationRegistry(this._threadsSplitter._removeThreadDemand);
+        this._demandedThreadsGcRegistry.register(this, this._demandedThreads);
+    }
+    [Symbol.dispose]() {
+        this.dispose();
+    }
+    dispose() {
+        if (this._disposed)
+            return;
+        this._disposed = true;
+        this._threadsSplitter._removeWantedThreads(this._wantedThreads);
+        this._threadsSplitter._removeThreadDemand(this._demandedThreads);
+        this._wantedThreadsGcRegistry.unregister(this);
+        this._demandedThreadsGcRegistry.unregister(this);
+    }
+    getAllocationToConsume() {
+        if (this._disposed)
+            throw new DisposedError();
+        if (this._threadsSplitter.maxThreads === 0)
+            return [this._wantedThreads, new DisposableHandle(() => { })];
+        return this._getAsyncAllocationToConsume();
+    }
+    async _getAsyncAllocationToConsume() {
+        do {
+            this._usedThreads = this._threadsSplitter._getUpdatedActiveThreads(this._usedThreads, this._wantedThreads, this._demandedThreads);
+            if (this._usedThreads < this._demandedThreads) {
+                this._usedThreads = this._threadsSplitter._getUpdatedActiveThreads(this._usedThreads, 0, 0);
+                await this._threadsSplitter._waitForFreeThread();
+            }
+        } while (this._usedThreads < this._demandedThreads);
+        return [this._usedThreads, new DisposableHandle(() => {
+                this._usedThreads = this._threadsSplitter._getUpdatedActiveThreads(this._usedThreads, 0, 0);
+            })];
+    }
+}
+class MaxNumberCollection {
+    _countMap = new Map();
+    _maxNumber = 0;
+    add(number) {
+        const count = this._countMap.get(number) ?? 0;
+        this._countMap.set(number, count + 1);
+        if (number > this._maxNumber)
+            this._maxNumber = number;
+    }
+    remove(number) {
+        const count = this._countMap.get(number);
+        if (count == null)
+            return;
+        if (count === 1) {
+            this._countMap.delete(number);
+            if (number === this._maxNumber)
+                this._maxNumber = this._findMaxNumber();
+        }
+        else
+            this._countMap.set(number, count - 1);
+    }
+    get maxNumber() {
+        return this._maxNumber;
+    }
+    _findMaxNumber() {
+        let maxNumber = 0;
+        for (const number of this._countMap.keys()) {
+            if (number > maxNumber)
+                maxNumber = number;
+        }
+        return maxNumber;
+    }
+}
+//# sourceMappingURL=ThreadsSplitter.js.map

package/dist/utils/ThreadsSplitter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ThreadsSplitter.js","sourceRoot":"","sources":["../../src/utils/ThreadsSplitter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAE,gBAAgB,EAAC,MAAM,iBAAiB,CAAC;AAGhE,MAAM,OAAO,eAAe;IACP,cAAc,GAAG,IAAI,mBAAmB,EAAE,CAAC;IAC3C,oBAAoB,GAAmB,EAAE,CAAC;IACnD,cAAc,GAAW,CAAC,CAAC;IAC3B,mBAAmB,GAAW,CAAC,CAAC;IACjC,UAAU,CAAS;IAE1B;;;OAGG;IACH,YAAmB,UAAkB;QACjC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC;QAEtD,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnE,CAAC;IAEM,cAAc,CAAC,aAAqB,EAAE,aAAqB,CAAC;QAC/D,IAAI,aAAa,KAAK,CAAC,IAAI,UAAU,GAAG,aAAa;YACjD,UAAU,GAAG,aAAa,CAAC;QAE/B,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC,IAAI,aAAa,KAAK,CAAC;YAC5C,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC;QAEpC,OAAO,IAAI,uBAAuB,CAAC,IAAI,EAAE,aAAa,EAAE,UAAU,CAAC,CAAC;IACxE,CAAC;IAEM,qBAAqB,CAAC,OAAe;QACxC,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC;YACrB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;QAE5C,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IACvE,CAAC;IAED,gBAAgB;IACT,wBAAwB,CAAC,MAAc,EAAE,MAAc,EAAE,QAAgB;QAC5E,MAAM,oBAAoB,GAAG,IAAI,CAAC,cAAc,CAAC;QACjD,IAAI,MAAM,GAAG,MAAM;YACf,IAAI,CAAC,cAAc,IAAI,MAAM,GAAG,MAAM,CAAC;QAE3C,MAAM,YAAY,GAAG,IAAI,CAAC,yBAAyB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACtE,IAAI,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,oBAAoB;QAErE,IAAI,gBAAgB,KAAK,YAAY,EAAE,CAAC;YACpC,IAAI,CAAC,6BAA6B,CAAC,oBAAoB,CAAC,CAAC;YACzD,OAAO,YAAY,CAAC;QACxB,CAAC;QAAC,IAAI,gBAAgB,GAAG,YAAY,EAAE,CAAC;YACpC,IAAI,CAAC,cAAc,IAAI,gBAAgB,GAAG,YAAY,CAAC;YACvD,IAAI,CAAC,6BAA6B,CAAC,oBAAoB,CAAC,CAAC;YACzD,OAAO,YAAY,CAAC;QACxB,CAAC;QAED,MAAM,aAAa,GAAG,YAAY,GAAG,gBAAgB,CAAC;QACtD,MAAM,gBAAgB,GAAG,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC;QAC/D,IAAI,aAAa,IAAI,gBAAgB,EAAE,CAAC;YACpC,IAAI,CAAC,cAAc,IAAI,aAAa,CAAC;YACrC,IAAI,CAAC,6BAA6B,CAAC,oBAAoB,CAAC,CAAC;YACzD,OAAO,YAAY,CAAC;QACxB,CAAC;QAED,gBAAgB,IAAI,gBAAgB,CAAC;QACrC,IAAI,CAAC,cAAc,IAAI,gBAAgB,CAAC;QAExC,IAAI,CAAC,6BAA6B,CAAC,oBAAoB,CAAC,CAAC;QACzD,OAAO,gBAAgB,CAAC;IAC5B,CAAC;IAEO,6BAA6B,CAAC,iBAAyB;QAC3D,IAAI,IAAI,CAAC,cAAc,IAAI,iBAAiB;YACxC,OAAO;QAEX,OAAO,IAAI,CAAC,oBAAoB,CAAC,MAAM,GAAG,CAAC;YACvC,IAAI,CAAC,oBAAoB,CAAC,KAAK,EAAE,EAAE,EAAE,CAAC;IAC9C,CAAC;IAEO,yBAAyB,CAAC,aAAqB,EAAE,eAAuB;QAC5E,OAAO,IAAI,CAAC,GAAG,CACX,aAAa,EACb,IAAI,CAAC,GAAG,CACJ,eAAe,EACf,IAAI,CAAC,IAAI,CACL,CAAC,aAAa,GAAG,IAAI,CAAC,mBAAmB,CAAC;YAC1C,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,GAAG,eAAe,CAAC,CAAC,CAC9G,CACJ,CACJ,CAAC;IACN,CAAC;IAED,gBAAgB;IACT,kBAAkB;QACrB,OAAO,IAAI,OAAO,CAAO,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IACjF,CAAC;IAED,gBAAgB;IACT,iBAAiB,CAAC,aAAqB;QAC1C,IAAI,CAAC,mBAAmB,IAAI,aAAa,CAAC;IAC9C,CAAC;IAED,gBAAgB;IACT,oBAAoB,CAAC,aAAqB;QAC7C,IAAI,CAAC,mBAAmB,IAAI,aAAa,CAAC;IAC9C,CAAC;IAED,gBAAgB;IACT,gBAAgB,CAAC,eAAuB;QAC3C,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IAC7C,CAAC;IAED,gBAAgB;IACT,mBAAmB,CAAC,eAAuB;QAC9C,MAAM,eAAe,GAAG,IAAI,CAAC,cAAc,CAAC,SAAS,KAAK,eAAe,CAAC;QAC1E,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;QAE5C,IAAI,eAAe,KAAK,CAAC,IAAI,eAAe,IAAI,IAAI,CAAC,cAAc,CAAC,SAAS,KAAK,eAAe,EAAE,CAAC;YAChG,OAAO,IAAI,CAAC,oBAAoB,CAAC,MAAM,GAAG,CAAC;gBACvC,IAAI,CAAC,oBAAoB,CAAC,KAAK,EAAE,EAAE,EAAE,CAAC;QAC9C,CAAC;IACL,CAAC;CACJ;AAED,MAAM,OAAO,uBAAuB;IACf,gBAAgB,CAAkB;IAClC,cAAc,CAAS;IACvB,gBAAgB,CAAS;IACzB,wBAAwB,CAA+B;IACvD,0BAA0B,CAA+B;IAClE,YAAY,GAAW,CAAC,CAAC;IACzB,SAAS,GAAY,KAAK,CAAC;IAEnC,YAAmB,eAAgC,EAAE,aAAqB,EAAE,UAAkB;QAC1F,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC;QACxC,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,gBAAgB,GAAG,UAAU,CAAC;QAEnC,IAAI,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC7D,IAAI,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAE9D,IAAI,CAAC,wBAAwB,GAAG,IAAI,oBAAoB,CAAC,IAAI,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,CAAC;QACrG,IAAI,CAAC,wBAAwB,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;QAElE,IAAI,CAAC,0BAA0B,GAAG,IAAI,oBAAoB,CAAC,IAAI,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,CAAC;QACtG,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC1E,CAAC;IAEM,CAAC,MAAM,CAAC,OAAO,CAAC;QACnB,IAAI,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;IAEM,OAAO;QACV,IAAI,IAAI,CAAC,SAAS;YACd,OAAO;QAEX,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QAEtB,IAAI,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAChE,IAAI,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAEjE,IAAI,CAAC,wBAAwB,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAC/C,IAAI,CAAC,0BAA0B,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IACrD,CAAC;IAEM,sBAAsB;QACzB,IAAI,IAAI,CAAC,SAAS;YACd,MAAM,IAAI,aAAa,EAAE,CAAC;QAE9B,IAAI,IAAI,CAAC,gBAAgB,CAAC,UAAU,KAAK,CAAC;YACtC,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,gBAAgB,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC;QAEjE,OAAO,IAAI,CAAC,4BAA4B,EAAE,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,4BAA4B;QACtC,GAAG,CAAC;YACA,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,wBAAwB,CAC9D,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,gBAAgB,CAChE,CAAC;YAEF,IAAI,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC5C,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,wBAAwB,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC5F,MAAM,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,EAAE,CAAC;YACrD,CAAC;QACL,CAAC,QAAQ,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,gBAAgB,EAAE;QAEpD,OAAO,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,gBAAgB,CAAC,GAAG,EAAE;gBACjD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,wBAAwB,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAChG,CAAC,CAAC,CAAC,CAAC;IACR,CAAC;CACJ;AAED,MAAM,mBAAmB;IACb,SAAS,GAAwB,IAAI,GAAG,EAAE,CAAC;IAC3C,UAAU,GAAW,CAAC,CAAC;IAExB,GAAG,CAAC,MAAc;QACrB,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC9C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;QAEtC,IAAI,MAAM,GAAG,IAAI,CAAC,UAAU;YACxB,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;IACjC,CAAC;IAEM,MAAM,CAAC,MAAc;QACxB,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,KAAK,IAAI,IAAI;YACb,OAAO;QAEX,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;YACd,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC9B,IAAI,MAAM,KAAK,IAAI,CAAC,UAAU;gBAC1B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QAChD,CAAC;;YACG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IAC9C,CAAC;IAED,IAAW,SAAS;QAChB,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAEO,cAAc;QAClB,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC;YACzC,IAAI,MAAM,GAAG,SAAS;gBAClB,SAAS,GAAG,MAAM,CAAC;QAC3B,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;CACJ"}

package/dist/utils/TokenStreamRegulator.d.ts

@@ -1,30 +1,36 @@
-import { Token } from "../types.js";
+import { Token, Tokenizer } from "../types.js";
 export declare class TokenStreamRegulator {
     addChunk({ tokens, text }: {
         tokens: Token[];
         text: string;
     }): QueuedTokenRelease;
     popFreeChunkTokens(): Token[];
-    getPartiallyFreeChunk(): {
+    getPartiallyFreeChunk(tokenizer: Tokenizer): {
         tokens: Token[];
         text: string;
     };
     getAllQueuedChunkTokens(): Token[];
+    getLastQueuedChunkTokens(maxTokens?: number): Token[];
+    clearQueue(): void;
 }
 export declare class QueuedTokenRelease {
-    readonly tokens: readonly Token[];
-    readonly text: string;
     private constructor();
+    get tokens(): readonly Token[];
+    get text(): string;
     get isFree(): boolean;
+    get hasTextLocks(): boolean;
+    get hasTokenLocks(): boolean;
     get isPartiallyFree(): boolean;
     getFreeTextIndex(): number;
     getFreeTokenIndex(): number;
     createTextIndexLock(startIndex: number): QueuedTokenReleaseLock;
     createTokenIndexLock(startIndex: number): QueuedTokenReleaseLock;
+    modifyTokensAndText(tokens: readonly Token[], text: string): void;
 }
 export declare class QueuedTokenReleaseLock {
     private constructor();
     get index(): number;
+    duplicate(): QueuedTokenReleaseLock;
     dispose(): void;
     [Symbol.dispose](): void;
 }

package/dist/utils/TokenStreamRegulator.js

@@ -1,5 +1,9 @@
+import { DisposedError } from "lifecycle-utils";
+import { maxRecentDetokenizerTokens } from "../consts.js";
+import { pushAll } from "./pushAll.js";
 export class TokenStreamRegulator {
     /** @internal */ _queue = [];
+    /** @internal */ _LastTokens = [];
     addChunk({ tokens, text }) {
         const queuedRelease = QueuedTokenRelease._create(tokens, text);
         this._queue.push(queuedRelease);
@@ -7,16 +11,65 @@ export class TokenStreamRegulator {
     }
     popFreeChunkTokens() {
         const res = [];
-        while (this._queue.length > 0 && this._queue[0].isFree)
-            res.push(...this._queue.shift().tokens);
+        while (this._queue.length > 0 && this._queue[0].isFree) {
+            const tokens = this._queue.shift().tokens;
+            pushAll(res, tokens);
+            pushAll(this._LastTokens, tokens);
+        }
+        if (this._LastTokens.length > maxRecentDetokenizerTokens)
+            this._LastTokens.splice(0, this._LastTokens.length - maxRecentDetokenizerTokens);
         return res;
     }
-    getPartiallyFreeChunk() {
+    getPartiallyFreeChunk(tokenizer) {
         if (this._queue.length > 0 && this._queue[0].isPartiallyFree) {
             const queuedRelease = this._queue[0];
+            if (queuedRelease.hasTextLocks && !queuedRelease.hasTokenLocks)
+                return {
+                    tokens: [],
+                    text: queuedRelease.text.slice(0, queuedRelease.getFreeTextIndex())
+                };
+            else if (queuedRelease.hasTokenLocks && !queuedRelease.hasTextLocks) {
+                const tokens = queuedRelease.tokens.slice(0, queuedRelease.getFreeTokenIndex());
+                return {
+                    tokens,
+                    text: tokenizer.detokenize(tokens, false, this._LastTokens)
+                };
+            }
+            const freeTokenIndex = queuedRelease.getFreeTokenIndex();
+            const tokens = queuedRelease.tokens.slice(0, freeTokenIndex);
+            const tokensText = tokenizer.detokenize(tokens, false, this._LastTokens);
+            const freeTextIndex = queuedRelease.getFreeTextIndex();
+            const text = queuedRelease.text.slice(0, freeTextIndex);
+            if (text.length > tokensText.length) {
+                return {
+                    tokens,
+                    text: tokensText
+                };
+            }
+            else if (text.length < tokensText.length) {
+                const resTokens = [];
+                let resTokensText = "";
+                const lastTokens = this._LastTokens.slice();
+                for (const token of tokens) {
+                    const tokenText = tokenizer.detokenize([token], false, lastTokens);
+                    lastTokens.push(token);
+                    if (resTokensText.length + tokenText.length > text.length) {
+                        const remainingText = text.slice(resTokensText.length);
+                        const remainingTokens = tokenizer(remainingText, false, "trimLeadingSpace");
+                        pushAll(resTokens, remainingTokens);
+                        break;
+                    }
+                    resTokens.push(token);
+                    resTokensText += tokenText;
+                }
+                return {
+                    tokens: resTokens,
+                    text
+                };
+            }
             return {
-                tokens: queuedRelease.tokens.slice(0, queuedRelease.getFreeTokenIndex()),
-                text: queuedRelease.text.slice(0, queuedRelease.getFreeTextIndex())
+                tokens: queuedRelease.tokens.slice(0, freeTokenIndex),
+                text: queuedRelease.text.slice(0, freeTextIndex)
             };
         }
         return {
@@ -27,21 +80,49 @@ export class TokenStreamRegulator {
     getAllQueuedChunkTokens() {
         return this._queue.flatMap((queuedRelease) => queuedRelease.tokens);
     }
+    getLastQueuedChunkTokens(maxTokens = maxRecentDetokenizerTokens) {
+        const res = [];
+        for (let i = this._queue.length - 1; i >= 0 && res.length < maxTokens; i--) {
+            const tokens = this._queue[i].tokens;
+            for (let j = tokens.length - 1; j >= 0 && res.length < maxTokens; j--)
+                res.unshift(tokens[j]);
+        }
+        return this._queue.flatMap((queuedRelease) => queuedRelease.tokens);
+    }
+    clearQueue() {
+        this._queue.length = 0;
+    }
 }
 export class QueuedTokenRelease {
     /** @internal */ _textLocks = new Set();
     /** @internal */ _tokenLocks = new Set();
-    tokens;
-    text;
+    /** @internal */ _tokens;
+    /** @internal */ _text;
     constructor(tokens, text) {
-        this.tokens = tokens;
-        this.text = text;
+        this._tokens = tokens;
+        this._text = text;
+    }
+    get tokens() {
+        return this._tokens;
+    }
+    get text() {
+        return this._text;
     }
     get isFree() {
         return this._textLocks.size === 0 && this._tokenLocks.size === 0;
     }
+    get hasTextLocks() {
+        return this._textLocks.size > 0;
+    }
+    get hasTokenLocks() {
+        return this._tokenLocks.size > 0;
+    }
     get isPartiallyFree() {
-        return this._textLocks.size !== 0 && this._tokenLocks.size !== 0;
+        if (this.isFree)
+            return true;
+        const freeTextIndex = this.getFreeTextIndex();
+        const freeTokenIndex = this.getFreeTokenIndex();
+        return freeTextIndex > 0 && freeTokenIndex > 0;
     }
     getFreeTextIndex() {
         if (this._textLocks.size === 0)
@@ -67,6 +148,10 @@ export class QueuedTokenRelease {
             this._tokenLocks.add(lock);
         return lock;
     }
+    modifyTokensAndText(tokens, text) {
+        this._tokens = tokens;
+        this._text = text;
+    }
     /** @internal */
     static _create(tokens, text) {
         return new QueuedTokenRelease(tokens, text);
@@ -82,6 +167,13 @@ export class QueuedTokenReleaseLock {
     get index() {
         return this._index;
     }
+    duplicate() {
+        if (!this._locks.has(this))
+            throw new DisposedError();
+        const lock = QueuedTokenReleaseLock._create(this._index, this._locks);
+        this._locks.add(lock);
+        return lock;
+    }
     dispose() {
         this._locks.delete(this);
     }

package/dist/utils/TokenStreamRegulator.js.map

@@ -1 +1 @@
-{"version":3,"file":"TokenStreamRegulator.js","sourceRoot":"","sources":["../../src/utils/TokenStreamRegulator.ts"],"names":[],"mappings":"AAEA,MAAM,OAAO,oBAAoB;IAC7B,gBAAgB,CAAkB,MAAM,GAAyB,EAAE,CAAC;IAE7D,QAAQ,CAAC,EAAC,MAAM,EAAE,IAAI,EAAkC;QAC3D,MAAM,aAAa,GAAG,kBAAkB,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAE/D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAEhC,OAAO,aAAa,CAAC;IACzB,CAAC;IAEM,kBAAkB;QACrB,MAAM,GAAG,GAAY,EAAE,CAAC;QAExB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM;YAClD,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAG,CAAC,MAAM,CAAC,CAAC;QAE7C,OAAO,GAAG,CAAC;IACf,CAAC;IAEM,qBAAqB;QACxB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,EAAE;YAC1D,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAErC,OAAO;gBACH,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,iBAAiB,EAAE,CAAC;gBACxE,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,gBAAgB,EAAE,CAAC;aACtE,CAAC;SACL;QAED,OAAO;YACH,MAAM,EAAE,EAAoB;YAC5B,IAAI,EAAE,EAAE;SACX,CAAC;IACN,CAAC;IAEM,uBAAuB;QAC1B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,aAAa,EAAE,EAAE,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IACxE,CAAC;CACJ;AAED,MAAM,OAAO,kBAAkB;IAC3B,gBAAgB,CAAkB,UAAU,GAAG,IAAI,GAAG,EAA0B,CAAC;IACjF,gBAAgB,CAAkB,WAAW,GAAG,IAAI,GAAG,EAA0B,CAAC;IAElE,MAAM,CAAmB;IACzB,IAAI,CAAS;IAE7B,YAAoB,MAAwB,EAAE,IAAY;QACtD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACrB,CAAC;IAED,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,CAAC;IACrE,CAAC;IAED,IAAW,eAAe;QACtB,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,CAAC;IACrE,CAAC;IAEM,gBAAgB;QACnB,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC;YAC1B,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;QAE5B,OAAO,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC;aACtB,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5E,CAAC;IAEM,iBAAiB;QACpB,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC;YAC3B,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QAE9B,OAAO,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC;aACvB,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC9E,CAAC;IAEM,mBAAmB,CAAC,UAAkB;QACzC,MAAM,IAAI,GAAG,sBAAsB,CAAC,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAEzE,IAAI,UAAU,IAAI,CAAC,IAAI,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM;YAChD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAE9B,OAAO,IAAI,CAAC;IAChB,CAAC;IAEM,oBAAoB,CAAC,UAAkB;QAC1C,MAAM,IAAI,GAAG,sBAAsB,CAAC,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAE1E,IAAI,UAAU,IAAI,CAAC,IAAI,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM;YAClD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAE/B,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,gBAAgB;IACT,MAAM,CAAC,OAAO,CAAC,MAAe,EAAE,IAAY;QAC/C,OAAO,IAAI,kBAAkB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAChD,CAAC;CACJ;AAED,MAAM,OAAO,sBAAsB;IAC/B,gBAAgB,CAAkB,MAAM,CAAC;IACzC,gBAAgB,CAAkB,MAAM,CAA8B;IAEtE,YAAoB,KAAa,EAAE,KAAkC;QACjE,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACxB,CAAC;IAED,IAAW,KAAK;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAEM,OAAO;QACV,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IAEM,CAAC,MAAM,CAAC,OAAO,CAAC;QACnB,IAAI,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;IAED,gBAAgB;IACT,MAAM,CAAC,OAAO,CAAC,MAAc,EAAE,KAAkC;QACpE,OAAO,IAAI,sBAAsB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACrD,CAAC;CACJ"}
+{"version":3,"file":"TokenStreamRegulator.js","sourceRoot":"","sources":["../../src/utils/TokenStreamRegulator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAC,0BAA0B,EAAC,MAAM,cAAc,CAAC;AACxD,OAAO,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAErC,MAAM,OAAO,oBAAoB;IAC7B,gBAAgB,CAAkB,MAAM,GAAyB,EAAE,CAAC;IACpE,gBAAgB,CAAkB,WAAW,GAAY,EAAE,CAAC;IAErD,QAAQ,CAAC,EAAC,MAAM,EAAE,IAAI,EAAkC;QAC3D,MAAM,aAAa,GAAG,kBAAkB,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAE/D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAEhC,OAAO,aAAa,CAAC;IACzB,CAAC;IAEM,kBAAkB;QACrB,MAAM,GAAG,GAAY,EAAE,CAAC;QAExB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,MAAM,EAAE,CAAC;YACtD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAG,CAAC,MAAM,CAAC;YAC3C,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QACtC,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,0BAA0B;YACpD,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,0BAA0B,CAAC,CAAC;QAErF,OAAO,GAAG,CAAC;IACf,CAAC;IAEM,qBAAqB,CAAC,SAAoB;QAC7C,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,eAAe,EAAE,CAAC;YAC5D,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC;YAEtC,IAAI,aAAa,CAAC,YAAY,IAAI,CAAC,aAAa,CAAC,aAAa;gBAC1D,OAAO;oBACH,MAAM,EAAE,EAAE;oBACV,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,gBAAgB,EAAE,CAAC;iBACtE,CAAC;iBACD,IAAI,aAAa,CAAC,aAAa,IAAI,CAAC,aAAa,CAAC,YAAY,EAAE,CAAC;gBAClE,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,iBAAiB,EAAE,CAAC,CAAC;gBAChF,OAAO;oBACH,MAAM;oBACN,IAAI,EAAE,SAAS,CAAC,UAAU,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC;iBAC9D,CAAC;YACN,CAAC;YAED,MAAM,cAAc,GAAG,aAAa,CAAC,iBAAiB,EAAE,CAAC;YACzD,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;YAC7D,MAAM,UAAU,GAAG,SAAS,CAAC,UAAU,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YAEzE,MAAM,aAAa,GAAG,aAAa,CAAC,gBAAgB,EAAE,CAAC;YACvD,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;YAExD,IAAI,IAAI,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;gBAClC,OAAO;oBACH,MAAM;oBACN,IAAI,EAAE,UAAU;iBACnB,CAAC;YACN,CAAC;iBAAM,IAAI,IAAI,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;gBACzC,MAAM,SAAS,GAAY,EAAE,CAAC;gBAC9B,IAAI,aAAa,GAAG,EAAE,CAAC;gBAEvB,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;gBAC5C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;oBACzB,MAAM,SAAS,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;oBACnE,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBAEvB,IAAI,aAAa,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;wBACxD,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;wBACvD,MAAM,eAAe,GAAG,SAAS,CAAC,aAAa,EAAE,KAAK,EAAE,kBAAkB,CAAC,CAAC;wBAC5E,OAAO,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;wBACpC,MAAM;oBACV,CAAC;oBAED,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACtB,aAAa,IAAI,SAAS,CAAC;gBAC/B,CAAC;gBAED,OAAO;oBACH,MAAM,EAAE,SAAS;oBACjB,IAAI;iBACP,CAAC;YACN,CAAC;YAED,OAAO;gBACH,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC;gBACrD,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC;aACnD,CAAC;QACN,CAAC;QAED,OAAO;YACH,MAAM,EAAE,EAAoB;YAC5B,IAAI,EAAE,EAAE;SACX,CAAC;IACN,CAAC;IAEM,uBAAuB;QAC1B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,aAAa,EAAE,EAAE,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IACxE,CAAC;IAEM,wBAAwB,CAAC,YAAoB,0BAA0B;QAC1E,MAAM,GAAG,GAAY,EAAE,CAAC;QAExB,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;YACzE,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC;YACtC,KAAK,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC,EAAE;gBACjE,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,aAAa,EAAE,EAAE,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IACxE,CAAC;IAEM,UAAU;QACb,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IAC3B,CAAC;CACJ;AAED,MAAM,OAAO,kBAAkB;IAC3B,gBAAgB,CAAkB,UAAU,GAAG,IAAI,GAAG,EAA0B,CAAC;IACjF,gBAAgB,CAAkB,WAAW,GAAG,IAAI,GAAG,EAA0B,CAAC;IAClF,gBAAgB,CAAS,OAAO,CAAmB;IACnD,gBAAgB,CAAS,KAAK,CAAS;IAEvC,YAAoB,MAAwB,EAAE,IAAY;QACtD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACtB,CAAC;IAED,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED,IAAW,IAAI;QACX,OAAO,IAAI,CAAC,KAAK,CAAC;IACtB,CAAC;IAED,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,CAAC;IACrE,CAAC;IAED,IAAW,YAAY;QACnB,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,CAAC;IACpC,CAAC;IAED,IAAW,aAAa;QACpB,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,CAAC;IACrC,CAAC;IAED,IAAW,eAAe;QACtB,IAAI,IAAI,CAAC,MAAM;YACX,OAAO,IAAI,CAAC;QAEhB,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC9C,MAAM,cAAc,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAChD,OAAO,aAAa,GAAG,CAAC,IAAI,cAAc,GAAG,CAAC,CAAC;IACnD,CAAC;IAEM,gBAAgB;QACnB,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC;YAC1B,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;QAE5B,OAAO,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC;aACtB,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5E,CAAC;IAEM,iBAAiB;QACpB,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC;YAC3B,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QAE9B,OAAO,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC;aACvB,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC9E,CAAC;IAEM,mBAAmB,CAAC,UAAkB;QACzC,MAAM,IAAI,GAAG,sBAAsB,CAAC,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAEzE,IAAI,UAAU,IAAI,CAAC,IAAI,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM;YAChD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAE9B,OAAO,IAAI,CAAC;IAChB,CAAC;IAEM,oBAAoB,CAAC,UAAkB;QAC1C,MAAM,IAAI,GAAG,sBAAsB,CAAC,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAE1E,IAAI,UAAU,IAAI,CAAC,IAAI,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM;YAClD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAE/B,OAAO,IAAI,CAAC;IAChB,CAAC;IAEM,mBAAmB,CAAC,MAAwB,EAAE,IAAY;QAC7D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACtB,CAAC;IAED,gBAAgB;IACT,MAAM,CAAC,OAAO,CAAC,MAAe,EAAE,IAAY;QAC/C,OAAO,IAAI,kBAAkB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAChD,CAAC;CACJ;AAED,MAAM,OAAO,sBAAsB;IAC/B,gBAAgB,CAAkB,MAAM,CAAC;IACzC,gBAAgB,CAAkB,MAAM,CAA8B;IAEtE,YAAoB,KAAa,EAAE,KAAkC;QACjE,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACxB,CAAC;IAED,IAAW,KAAK;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAEM,SAAS;QACZ,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC;YACtB,MAAM,IAAI,aAAa,EAAE,CAAC;QAE9B,MAAM,IAAI,GAAG,sBAAsB,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAEtE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAEtB,OAAO,IAAI,CAAC;IAChB,CAAC;IAEM,OAAO;QACV,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IAEM,CAAC,MAAM,CAAC,OAAO,CAAC;QACnB,IAAI,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;IAED,gBAAgB;IACT,MAAM,CAAC,OAAO,CAAC,MAAc,EAAE,KAAkC;QACpE,OAAO,IAAI,sBAAsB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACrD,CAAC;CACJ"}

package/dist/utils/UnsupportedError.d.ts

@@ -0,0 +1,2 @@
+export declare class UnsupportedError extends Error {
+}

package/dist/utils/UnsupportedError.js

@@ -0,0 +1,7 @@
+export class UnsupportedError extends Error {
+    /** @internal */
+    constructor(message = "UnsupportedError") {
+        super(message);
+    }
+}
+//# sourceMappingURL=UnsupportedError.js.map

package/dist/utils/UnsupportedError.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"UnsupportedError.js","sourceRoot":"","sources":["../../src/utils/UnsupportedError.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IACvC,gBAAgB;IAChB,YAAmB,UAAkB,kBAAkB;QACnD,KAAK,CAAC,OAAO,CAAC,CAAC;IACnB,CAAC;CACJ"}

package/dist/utils/appendUserMessageToChatHistory.d.ts

@@ -1,2 +1,6 @@
 import { ChatHistoryItem } from "../types.js";
+/**
+ * Appends a user message to the chat history.
+ * If the last message in the chat history is also a user message, the new message will be appended to it.
+ */
 export declare function appendUserMessageToChatHistory(chatHistory: readonly ChatHistoryItem[], message: string): ChatHistoryItem[];

package/dist/utils/appendUserMessageToChatHistory.js

@@ -1,3 +1,7 @@
+/**
+ * Appends a user message to the chat history.
+ * If the last message in the chat history is also a user message, the new message will be appended to it.
+ */
 export function appendUserMessageToChatHistory(chatHistory, message) {
     const newChatHistory = chatHistory.slice();
     if (newChatHistory.length > 0 && newChatHistory[newChatHistory.length - 1].type === "user") {

package/dist/utils/appendUserMessageToChatHistory.js.map

@@ -1 +1 @@
-{"version":3,"file":"appendUserMessageToChatHistory.js","sourceRoot":"","sources":["../../src/utils/appendUserMessageToChatHistory.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,8BAA8B,CAAC,WAAuC,EAAE,OAAe;IACnG,MAAM,cAAc,GAAG,WAAW,CAAC,KAAK,EAAE,CAAC;IAE3C,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,IAAI,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE;QACxF,MAAM,eAAe,GAAG,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAoB,CAAC;QAErF,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG;YACxC,GAAG,eAAe;YAClB,IAAI,EAAE,CAAC,eAAe,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;SACrD,CAAC;KACL;SAAM;QACH,cAAc,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,OAAO;SAChB,CAAC,CAAC;KACN;IAED,OAAO,cAAc,CAAC;AAC1B,CAAC"}
+{"version":3,"file":"appendUserMessageToChatHistory.js","sourceRoot":"","sources":["../../src/utils/appendUserMessageToChatHistory.ts"],"names":[],"mappings":"AAEA;;;GAGG;AACH,MAAM,UAAU,8BAA8B,CAAC,WAAuC,EAAE,OAAe;IACnG,MAAM,cAAc,GAAG,WAAW,CAAC,KAAK,EAAE,CAAC;IAE3C,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,IAAI,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC1F,MAAM,eAAe,GAAG,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAqB,CAAC;QAEtF,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG;YACxC,GAAG,eAAe;YAClB,IAAI,EAAE,CAAC,eAAe,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;SACrD,CAAC;IACN,CAAC;SAAM,CAAC;QACJ,cAAc,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,OAAO;SAChB,CAAC,CAAC;IACP,CAAC;IAED,OAAO,cAAc,CAAC;AAC1B,CAAC"}

package/dist/utils/clearTempFolder.js.map

@@ -1 +1 @@
-{"version":3,"file":"clearTempFolder.js","sourceRoot":"","sources":["../../src/utils/clearTempFolder.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,qBAAqB,EAAC,MAAM,cAAc,CAAC;AAEnD,MAAM,CAAC,KAAK,UAAU,eAAe;IACjC,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;QAC9B,IAAI;YACA,MAAM,EAAE,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;SAC1C;QAAC,OAAO,GAAG,EAAE;YACV,gGAAgG;SACnG;QAED,OAAO;KACV;IAED,MAAM,EAAE,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;AAC3C,CAAC"}
+{"version":3,"file":"clearTempFolder.js","sourceRoot":"","sources":["../../src/utils/clearTempFolder.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,qBAAqB,EAAC,MAAM,cAAc,CAAC;AAEnD,MAAM,CAAC,KAAK,UAAU,eAAe;IACjC,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;QAC/B,IAAI,CAAC;YACD,MAAM,EAAE,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;QAC3C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,gGAAgG;QACpG,CAAC;QAED,OAAO;IACX,CAAC;IAED,MAAM,EAAE,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;AAC3C,CAAC"}

package/dist/utils/cmake.js

@@ -4,6 +4,7 @@ import which from "which";
 import chalk from "chalk";
 import { chmodr } from "chmodrp";
 import { defaultXpacksCacheDirectory, defaultXpacksStoreDirectory, llamaDirectory, localXpacksCacheDirectory, localXpacksStoreDirectory, xpackDirectory, xpmVersion } from "../config.js";
+import { logDistroInstallInstruction } from "../bindings/utils/logDistroInstallInstruction.js";
 import { spawnCommand } from "./spawnCommand.js";
 import withStatusLogs from "./withStatusLogs.js";
 import { withLockfile } from "./withLockfile.js";
@@ -18,8 +19,10 @@ export async function hasBuiltinCmake() {
 }
 export async function getCmakePath() {
     try {
-        const resolvedPath = await which("cmake");
-        if (resolvedPath !== "")
+        const resolvedPath = await which("cmake", {
+            nothrow: true
+        });
+        if (resolvedPath !== "" && resolvedPath != null)
             return resolvedPath;
     }
     catch (err) { }
@@ -48,14 +51,24 @@ export async function downloadCmakeIfNeeded(wrapWithStatusLogs = false) {
     catch (err) { }
     if (!wrapWithStatusLogs)
         await downloadCmake({ progressLogs: wrapWithStatusLogs });
-    else
-        await withStatusLogs({
-            loading: chalk.blue("Downloading cmake"),
-            success: chalk.blue("Downloaded cmake"),
-            fail: chalk.blue("Failed to download cmake")
-        }, async () => {
-            await downloadCmake({ progressLogs: wrapWithStatusLogs });
-        });
+    else {
+        try {
+            await withStatusLogs({
+                loading: chalk.blue("Downloading cmake"),
+                success: chalk.blue("Downloaded cmake"),
+                fail: chalk.blue("Failed to download cmake")
+            }, async () => {
+                await downloadCmake({ progressLogs: wrapWithStatusLogs });
+            });
+        }
+        catch (err) {
+            await logDistroInstallInstruction('To install "cmake", ', {
+                linuxPackages: { apt: ["cmake"], apk: ["cmake"] },
+                macOsPackages: { brew: ["cmake"] }
+            });
+            throw err;
+        }
+    }
 }
 export async function clearLocalCmake() {
     await fs.remove(localXpacksStoreDirectory);

package/dist/utils/cmake.js.map

@@ -1 +1 @@
-{"version":3,"file":"cmake.js","sourceRoot":"","sources":["../../src/utils/cmake.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAC,MAAM,EAAC,MAAM,SAAS,CAAC;AAC/B,OAAO,EACH,2BAA2B,EAAE,2BAA2B,EAAE,cAAc,EAAE,yBAAyB,EAAE,yBAAyB,EAC9H,cAAc,EAAE,UAAU,EAC7B,MAAM,cAAc,CAAC;AACtB,OAAO,EAAC,YAAY,EAAC,MAAM,mBAAmB,CAAC;AAC/C,OAAO,cAAc,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAC,YAAY,EAAC,MAAM,mBAAmB,CAAC;AAG/C,MAAM,CAAC,KAAK,UAAU,eAAe;IACjC,IAAI;QACA,MAAM,YAAY,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;QAC1C,OAAO,YAAY,KAAK,EAAE,CAAC;KAC9B;IAAC,OAAO,GAAG,EAAE;QACV,OAAO,KAAK,CAAC;KAChB;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY;IAC9B,IAAI;QACA,MAAM,YAAY,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;QAE1C,IAAI,YAAY,KAAK,EAAE;YACnB,OAAO,YAAY,CAAC;KAC3B;IAAC,OAAO,GAAG,EAAE,GAAE;IAEhB,IAAI;QACA,IAAI,YAAY,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE;YACpC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC;SAC7D,CAAC,CAAC;QAEH,IAAI,YAAY,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC3C,YAAY,GAAG,CAAC,MAAM,mBAAmB,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC,IAAI,EAAE,CAAC;aAC3E,IAAI,YAAY,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;YAClD,MAAM,WAAW,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;YAEnE,IAAI,MAAM,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC;gBAChC,YAAY,GAAG,CAAC,MAAM,mBAAmB,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC,IAAI,EAAE,CAAC;SAClF;QAED,IAAI,YAAY,KAAK,EAAE;YACnB,OAAO,YAAY,CAAC;KAC3B;IAAC,OAAO,GAAG,EAAE,GAAE;IAEhB,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;AACvC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,qBAA8B,KAAK;IAC3E,IAAI;QACA,MAAM,YAAY,EAAE,CAAC;QACrB,OAAO;KACV;IAAC,OAAO,GAAG,EAAE,GAAE;IAEhB,IAAI,CAAC,kBAAkB;QACnB,MAAM,aAAa,CAAC,EAAC,YAAY,EAAE,kBAAkB,EAAC,CAAC,CAAC;;QAExD,MAAM,cAAc,CAAC;YACjB,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC;YACxC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC;YACvC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC;SAC/C,EAAE,KAAK,IAAI,EAAE;YACV,MAAM,aAAa,CAAC,EAAC,YAAY,EAAE,kBAAkB,EAAC,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;AACX,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe;IACjC,MAAM,EAAE,CAAC,MAAM,CAAC,yBAAyB,CAAC,CAAC;IAC3C,MAAM,EAAE,CAAC,MAAM,CAAC,yBAAyB,CAAC,CAAC;IAC3C,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC,CAAC;AACzD,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB;IACrC,IAAI;QACA,MAAM,MAAM,CAAC,yBAAyB,EAAE,KAAK,CAAC,CAAC;QAC/C,MAAM,MAAM,CAAC,yBAAyB,EAAE,KAAK,CAAC,CAAC;QAC/C,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC,EAAE,KAAK,CAAC,CAAC;KAC5D;IAAC,OAAO,GAAG,EAAE,GAAE;AACpB,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,EAAC,YAAY,GAAG,IAAI,KAA8B,EAAE;IAC7E,MAAM,YAAY,CAAC;QACf,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,cAAc,CAAC;KAC1D,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,MAAM,GAAsB;YAC9B,GAAG,OAAO,CAAC,GAAG;YACd,mBAAmB,EAAE,2BAA2B;YAChD,mBAAmB,EAAE,2BAA2B;SACnD,CAAC;QAEF,MAAM,YAAY,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,UAAU,EAAE,EAAE,SAAS,EAAE,+BAA+B,EAAE,WAAW,CAAC,EAAE,cAAc,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;QAEvK,MAAM,EAAE,CAAC,MAAM,CAAC,yBAAyB,CAAC,CAAC;QAC3C,MAAM,mBAAmB,EAAE,CAAC;IAChC,CAAC,CAAC,CAAC;AACP,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,WAAmB,EAAE,OAAe;IACnE,MAAM,WAAW,GAAW,MAAM,EAAE,CAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IACnE,MAAM,oBAAoB,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;IAEvD,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,OAAO,CAAC;QACvC,OAAO,IAAI,CAAC;IAEhB,MAAM,kBAAkB,GAAG,oBAAoB,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrE,MAAM,qBAAqB,GAAG,WAAW,CAAC,kBAAkB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAE/E,IAAI,qBAAqB,KAAK,GAAG,IAAI,qBAAqB,KAAK,GAAG;QAC9D,OAAO,IAAI,CAAC;IAEhB,MAAM,oBAAoB,GAAG,WAAW,CAAC,WAAW,CAAC,qBAAqB,EAAE,kBAAkB,CAAC,CAAC;IAEhG,MAAM,OAAO,GAAG,WAAW,CAAC,KAAK,CAAC,oBAAoB,GAAG,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAEjG,IAAI,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC;QAC7B,OAAO,IAAI,CAAC;IAEhB,OAAO,OAAO,CAAC;AACnB,CAAC"}
+{"version":3,"file":"cmake.js","sourceRoot":"","sources":["../../src/utils/cmake.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAC,MAAM,EAAC,MAAM,SAAS,CAAC;AAC/B,OAAO,EACH,2BAA2B,EAAE,2BAA2B,EAAE,cAAc,EAAE,yBAAyB,EAAE,yBAAyB,EAC9H,cAAc,EAAE,UAAU,EAC7B,MAAM,cAAc,CAAC;AACtB,OAAO,EAAC,2BAA2B,EAAC,MAAM,kDAAkD,CAAC;AAC7F,OAAO,EAAC,YAAY,EAAC,MAAM,mBAAmB,CAAC;AAC/C,OAAO,cAAc,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAC,YAAY,EAAC,MAAM,mBAAmB,CAAC;AAG/C,MAAM,CAAC,KAAK,UAAU,eAAe;IACjC,IAAI,CAAC;QACD,MAAM,YAAY,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;QAC1C,OAAO,YAAY,KAAK,EAAE,CAAC;IAC/B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,OAAO,KAAK,CAAC;IACjB,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY;IAC9B,IAAI,CAAC;QACD,MAAM,YAAY,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE;YACtC,OAAO,EAAE,IAAI;SAChB,CAAC,CAAC;QAEH,IAAI,YAAY,KAAK,EAAE,IAAI,YAAY,IAAI,IAAI;YAC3C,OAAO,YAAY,CAAC;IAC5B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC,CAAA,CAAC;IAEhB,IAAI,CAAC;QACD,IAAI,YAAY,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE;YACpC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC;SAC7D,CAAC,CAAC;QAEH,IAAI,YAAY,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC3C,YAAY,GAAG,CAAC,MAAM,mBAAmB,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC,IAAI,EAAE,CAAC;aAC3E,IAAI,YAAY,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACnD,MAAM,WAAW,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;YAEnE,IAAI,MAAM,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC;gBAChC,YAAY,GAAG,CAAC,MAAM,mBAAmB,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC,IAAI,EAAE,CAAC;QACnF,CAAC;QAED,IAAI,YAAY,KAAK,EAAE;YACnB,OAAO,YAAY,CAAC;IAC5B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC,CAAA,CAAC;IAEhB,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;AACvC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,qBAA8B,KAAK;IAC3E,IAAI,CAAC;QACD,MAAM,YAAY,EAAE,CAAC;QACrB,OAAO;IACX,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC,CAAA,CAAC;IAEhB,IAAI,CAAC,kBAAkB;QACnB,MAAM,aAAa,CAAC,EAAC,YAAY,EAAE,kBAAkB,EAAC,CAAC,CAAC;SACvD,CAAC;QACF,IAAI,CAAC;YACD,MAAM,cAAc,CAAC;gBACjB,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC;gBACxC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC;gBACvC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC;aAC/C,EAAE,KAAK,IAAI,EAAE;gBACV,MAAM,aAAa,CAAC,EAAC,YAAY,EAAE,kBAAkB,EAAC,CAAC,CAAC;YAC5D,CAAC,CAAC,CAAC;QACP,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,2BAA2B,CAAC,sBAAsB,EAAE;gBACtD,aAAa,EAAE,EAAC,GAAG,EAAE,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,EAAC;gBAC/C,aAAa,EAAE,EAAC,IAAI,EAAE,CAAC,OAAO,CAAC,EAAC;aACnC,CAAC,CAAC;YACH,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe;IACjC,MAAM,EAAE,CAAC,MAAM,CAAC,yBAAyB,CAAC,CAAC;IAC3C,MAAM,EAAE,CAAC,MAAM,CAAC,yBAAyB,CAAC,CAAC;IAC3C,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC,CAAC;AACzD,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB;IACrC,IAAI,CAAC;QACD,MAAM,MAAM,CAAC,yBAAyB,EAAE,KAAK,CAAC,CAAC;QAC/C,MAAM,MAAM,CAAC,yBAAyB,EAAE,KAAK,CAAC,CAAC;QAC/C,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC,EAAE,KAAK,CAAC,CAAC;IAC7D,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC,CAAA,CAAC;AACpB,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,EAAC,YAAY,GAAG,IAAI,KAA8B,EAAE;IAC7E,MAAM,YAAY,CAAC;QACf,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,cAAc,CAAC;KAC1D,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,MAAM,GAAsB;YAC9B,GAAG,OAAO,CAAC,GAAG;YACd,mBAAmB,EAAE,2BAA2B;YAChD,mBAAmB,EAAE,2BAA2B;SACnD,CAAC;QAEF,MAAM,YAAY,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,UAAU,EAAE,EAAE,SAAS,EAAE,+BAA+B,EAAE,WAAW,CAAC,EAAE,cAAc,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;QAEvK,MAAM,EAAE,CAAC,MAAM,CAAC,yBAAyB,CAAC,CAAC;QAC3C,MAAM,mBAAmB,EAAE,CAAC;IAChC,CAAC,CAAC,CAAC;AACP,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,WAAmB,EAAE,OAAe;IACnE,MAAM,WAAW,GAAW,MAAM,EAAE,CAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IACnE,MAAM,oBAAoB,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;IAEvD,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,OAAO,CAAC;QACvC,OAAO,IAAI,CAAC;IAEhB,MAAM,kBAAkB,GAAG,oBAAoB,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrE,MAAM,qBAAqB,GAAG,WAAW,CAAC,kBAAkB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAE/E,IAAI,qBAAqB,KAAK,GAAG,IAAI,qBAAqB,KAAK,GAAG;QAC9D,OAAO,IAAI,CAAC;IAEhB,MAAM,oBAAoB,GAAG,WAAW,CAAC,WAAW,CAAC,qBAAqB,EAAE,kBAAkB,CAAC,CAAC;IAEhG,MAAM,OAAO,GAAG,WAAW,CAAC,KAAK,CAAC,oBAAoB,GAAG,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAEjG,IAAI,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC;QAC7B,OAAO,IAAI,CAAC;IAEhB,OAAO,OAAO,CAAC;AACnB,CAAC"}

package/dist/utils/compareTokens.d.ts

@@ -1,2 +1,2 @@
 import { Token } from "../types.js";
-export declare function compareTokens(token1: Token, token2: Token): boolean;
+export declare function compareTokens(token1?: Token, token2?: Token): boolean;

package/dist/utils/compareTokens.js.map

@@ -1 +1 @@
-{"version":3,"file":"compareTokens.js","sourceRoot":"","sources":["../../src/utils/compareTokens.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,aAAa,CAAC,MAAa,EAAE,MAAa;IACtD,OAAO,MAAM,KAAK,MAAM,CAAC;AAC7B,CAAC"}
+{"version":3,"file":"compareTokens.js","sourceRoot":"","sources":["../../src/utils/compareTokens.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,aAAa,CAAC,MAAc,EAAE,MAAc;IACxD,OAAO,MAAM,KAAK,MAAM,CAAC;AAC7B,CAAC"}