gpt-tokenizer 2.3.0 → 2.4.0

package/esm/BytePairEncodingCore.d.ts

@@ -4,28 +4,41 @@ export interface BytePairEncodingConfig {
     specialTokenMapping?: Map<string, number>;
     tokenSplitRegex: RegExp;
 }
+export declare const decoder: TextDecoder;
 export declare class BytePairEncodingCore {
-    readonly bytePairEncoderSize: number;
-    private bytePairEncoder;
-    private bytePairEncoderSortedLookup;
-    private bytePairRanksDecoder;
+    readonly mergeableBytePairRankCount: number;
+    /**
+     * an array where the index is the BPE rank,
+     * and the value is the string or the array of bytes that it decodes to
+     * it may contain holes if token is unused
+     */
+    private bytePairRankDecoder;
+    private bytePairNonUtfRankDecoder;
+    private bytePairNonUtfSortedEncoder;
+    /**
+     * a reverse map of the bytePairRankDecoder,
+     * where the key is the string and the value is the rank
+     * values that cannot be represented as a string are present in `bytePairNonUtfSortedEncoder`
+     */
+    private bytePairStringRankEncoder;
     private tokenSplitRegex;
     private specialTokensEncoder;
     private specialTokensDecoder;
     private specialTokenPatternRegex;
-    private stringDecoder;
     private textEncoder;
     constructor({ mergeableBytePairRanks: bytePairEncoder, specialTokenMapping: specialTokenEncoder, tokenSplitRegex, }: BytePairEncodingConfig);
-    getBpeRankFromString(key: string): number | undefined;
-    getBpeRankFromStringOrThrow(key: string): number;
-    getBpeRankFromBytes(key: Uint8Array): number | undefined;
-    getBpeRankFromBytesOrThrow(key: Uint8Array): number;
-    binarySearch(key: Uint8Array): number;
-    encodeNative(text: string, allowedSpecial?: Set<string>): Generator<number[], number, undefined>;
-    findNextSpecialStartIndex(text: string, allowedSpecial: Set<string> | undefined, startIndex: number, specialRegex: RegExp): number | undefined;
-    decodeNative(tokens: Iterable<number>): Generator<Uint8Array | string, void, void>;
-    decodeNativeAsync(tokens: AsyncIterable<number>): AsyncGenerator<Uint8Array | string>;
-    tryDecodeToken(tokenRank: number): Uint8Array | string | undefined;
-    bytePairEncode(input: string): number[];
-    bytePairMerge(piece: Uint8Array, getByteForRange: (start: number, end: number) => number): number[];
+    encodeNativeGenerator(text: string, allowedSpecial?: Set<string>): Generator<number[], number, undefined>;
+    encodeNative(text: string, allowedSpecial?: Set<string>): number[];
+    decodeNativeGenerator(tokens: Iterable<number>): Generator<Uint8Array | string, void, void>;
+    decodeNative(tokens: Iterable<number>): string;
+    decodeNativeAsyncIterable(tokens: AsyncIterable<number>): AsyncGenerator<Uint8Array | string>;
+    private getBpeRankFromString;
+    private getBpeRankFromStringOrThrow;
+    private getBpeRankFromBytes;
+    private getBpeRankFromBytesOrThrow;
+    private binarySearch;
+    private findNextSpecialStartIndex;
+    private tryDecodeToken;
+    private bytePairEncode;
+    private bytePairMerge;
 }

package/esm/BytePairEncodingCore.js

@@ -1,50 +1,175 @@
 /* eslint-disable no-continue */
 import { compareUint8Arrays, isAscii, tryConvertToString } from './utfUtil.js';
 import { escapeRegExp } from './util.js';
+const emptyBuffer = new Uint8Array(0);
+export const decoder = new TextDecoder('utf8');
 export class BytePairEncodingCore {
-    bytePairEncoderSize;
-    bytePairEncoder;
-    bytePairEncoderSortedLookup;
-    bytePairRanksDecoder = new Map();
+    mergeableBytePairRankCount;
+    /**
+     * an array where the index is the BPE rank,
+     * and the value is the string or the array of bytes that it decodes to
+     * it may contain holes if token is unused
+     */
+    bytePairRankDecoder;
+    bytePairNonUtfRankDecoder = new Map();
+    bytePairNonUtfSortedEncoder;
+    /**
+     * a reverse map of the bytePairRankDecoder,
+     * where the key is the string and the value is the rank
+     * values that cannot be represented as a string are present in `bytePairNonUtfSortedEncoder`
+     */
+    bytePairStringRankEncoder;
     tokenSplitRegex;
     specialTokensEncoder;
     specialTokensDecoder;
     specialTokenPatternRegex;
-    stringDecoder;
     textEncoder = new TextEncoder();
     constructor({ mergeableBytePairRanks: bytePairEncoder, specialTokenMapping: specialTokenEncoder, tokenSplitRegex, }) {
-        this.bytePairEncoder = bytePairEncoder;
-        this.stringDecoder = new Map();
+        this.bytePairRankDecoder = bytePairEncoder;
+        this.bytePairStringRankEncoder = new Map();
         // size without array holes (which may be present in the encoder)
-        this.bytePairEncoderSize = Object.keys(bytePairEncoder).length;
+        this.mergeableBytePairRankCount = Object.keys(bytePairEncoder).length;
         const binaryLookup = [];
         // forEach skips array holes:
         bytePairEncoder.forEach((value, rank) => {
             if (typeof value === 'string') {
-                this.stringDecoder.set(value, rank);
+                this.bytePairStringRankEncoder.set(value, rank);
                 return;
             }
             const byteArray = new Uint8Array(value);
             binaryLookup.push([byteArray, rank]);
-            this.bytePairRanksDecoder.set(rank, byteArray);
+            this.bytePairNonUtfRankDecoder.set(rank, byteArray);
         });
-        this.bytePairEncoderSortedLookup = binaryLookup.sort((a, b) => compareUint8Arrays(a[0], b[0]));
+        this.bytePairNonUtfSortedEncoder = binaryLookup.sort((a, b) => compareUint8Arrays(a[0], b[0]));
         this.specialTokensEncoder = specialTokenEncoder ?? new Map();
         this.specialTokensDecoder = specialTokenEncoder
             ? new Map([...specialTokenEncoder].map(([key, value]) => [value, key]))
             : new Map();
         this.tokenSplitRegex = tokenSplitRegex;
-        const parts = [...this.specialTokensEncoder.keys()].map(escapeRegExp);
-        const joinedParts = parts.join('|');
+        const escapedSpecialTokens = [...this.specialTokensEncoder.keys()].map(escapeRegExp);
+        const allSpecialTokensRegex = escapedSpecialTokens.join('|');
         try {
-            this.specialTokenPatternRegex = new RegExp(joinedParts);
+            this.specialTokenPatternRegex = new RegExp(allSpecialTokensRegex);
         }
         catch {
             throw new Error('Invalid regular expression pattern.');
         }
     }
+    *encodeNativeGenerator(text, allowedSpecial) {
+        let startIndex = 0;
+        let lastTokenLength = 0;
+        while (true) {
+            const nextSpecialStartIndex = this.findNextSpecialStartIndex(text, allowedSpecial, startIndex);
+            const endIndex = nextSpecialStartIndex !== undefined
+                ? nextSpecialStartIndex
+                : text.length;
+            const textSegment = text.slice(startIndex, endIndex - startIndex);
+            for (const [match] of textSegment.matchAll(this.tokenSplitRegex)) {
+                const token = this.getBpeRankFromString(match);
+                if (token !== undefined) {
+                    lastTokenLength = 1;
+                    yield [token];
+                    continue;
+                }
+                const tokens = this.bytePairEncode(match);
+                lastTokenLength = tokens.length;
+                yield tokens;
+            }
+            if (nextSpecialStartIndex !== undefined) {
+                const specialToken = text.slice(Math.max(0, nextSpecialStartIndex));
+                const specialTokenValue = this.specialTokensEncoder.get(specialToken);
+                if (specialTokenValue === undefined) {
+                    throw new Error(`Special token "${specialToken}" is not in the special token encoder.`);
+                }
+                yield [specialTokenValue];
+                startIndex = nextSpecialStartIndex + specialToken.length;
+                lastTokenLength = 0;
+            }
+            else {
+                break;
+            }
+        }
+        return lastTokenLength;
+    }
+    encodeNative(text, allowedSpecial) {
+        let startIndex = 0;
+        const tokensArray = []; // Flat list to collect the tokens
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+            const nextSpecialStartIndex = this.findNextSpecialStartIndex(text, allowedSpecial, startIndex);
+            const endIndex = nextSpecialStartIndex !== undefined
+                ? nextSpecialStartIndex
+                : text.length;
+            const textSegment = text.slice(startIndex, endIndex - startIndex);
+            for (const [match] of textSegment.matchAll(this.tokenSplitRegex)) {
+                const token = this.getBpeRankFromString(match);
+                if (token !== undefined) {
+                    tokensArray.push(token);
+                    continue;
+                }
+                const tokens = this.bytePairEncode(match);
+                tokensArray.push(...tokens);
+            }
+            if (nextSpecialStartIndex !== undefined) {
+                const specialToken = text.slice(Math.max(0, nextSpecialStartIndex));
+                const specialTokenValue = this.specialTokensEncoder.get(specialToken);
+                if (specialTokenValue === undefined) {
+                    throw new Error(`Special token "${specialToken}" is not in the special token encoder.`);
+                }
+                tokensArray.push(specialTokenValue);
+                startIndex = nextSpecialStartIndex + specialToken.length;
+            }
+            else {
+                break;
+            }
+        }
+        return tokensArray;
+    }
+    *decodeNativeGenerator(tokens) {
+        for (const token of tokens) {
+            const tokenBytes = this.tryDecodeToken(token);
+            if (tokenBytes) {
+                yield tokenBytes;
+            }
+        }
+    }
+    decodeNative(tokens) {
+        let decoded = '';
+        let intBuffer = emptyBuffer;
+        for (const token of tokens) {
+            const tokenBytes = this.tryDecodeToken(token);
+            if (tokenBytes === undefined) {
+                throw new Error(`Token ${token} is not in the byte pair encoder.`);
+            }
+            if (typeof tokenBytes === 'string') {
+                if (intBuffer !== emptyBuffer) {
+                    decoded += decoder.decode(intBuffer, { stream: true });
+                    intBuffer = emptyBuffer;
+                }
+                decoded += tokenBytes;
+            }
+            else {
+                const newBuffer = new Uint8Array(intBuffer.length + tokenBytes.length);
+                newBuffer.set(intBuffer);
+                newBuffer.set(tokenBytes, intBuffer.length);
+                intBuffer = newBuffer;
+            }
+        }
+        if (intBuffer !== emptyBuffer) {
+            decoded += decoder.decode(intBuffer, { stream: true });
+        }
+        return decoded;
+    }
+    async *decodeNativeAsyncIterable(tokens) {
+        for await (const token of tokens) {
+            const tokenBytesOrString = this.tryDecodeToken(token);
+            if (tokenBytesOrString) {
+                yield tokenBytesOrString;
+            }
+        }
+    }
     getBpeRankFromString(key) {
-        return this.stringDecoder.get(key);
+        return this.bytePairStringRankEncoder.get(key);
     }
     getBpeRankFromStringOrThrow(key) {
         const value = this.getBpeRankFromString(key);
@@ -61,7 +186,7 @@ export class BytePairEncodingCore {
         // Perform binary search on the binary keys
         const index = this.binarySearch(key);
         if (index !== -1) {
-            return this.bytePairEncoderSortedLookup[index][1];
+            return this.bytePairNonUtfSortedEncoder[index][1];
         }
         return undefined;
     }
@@ -75,13 +200,14 @@ export class BytePairEncodingCore {
     // Binary search on the binary keys
     binarySearch(key) {
         let low = 0;
-        let high = this.bytePairEncoderSortedLookup.length - 1;
+        let high = this.bytePairNonUtfSortedEncoder.length - 1;
         while (low <= high) {
             // eslint-disable-next-line no-bitwise
             const mid = (low + high) >>> 1;
-            const midKey = this.bytePairEncoderSortedLookup[mid][0];
+            const midKey = this.bytePairNonUtfSortedEncoder[mid][0];
             let cmp = 0;
-            for (let i = 0; i < Math.min(midKey.length, key.length); i++) {
+            const maxLength = Math.min(midKey.length, key.length);
+            for (let i = 0; i < maxLength; i++) {
                 cmp = midKey[i] - key[i];
                 if (cmp !== 0)
                     break;
@@ -101,80 +227,28 @@ export class BytePairEncodingCore {
         }
         return -1;
     }
-    *encodeNative(text, allowedSpecial) {
-        let startIndex = 0;
-        let lastTokenLength = 0;
-        while (true) {
-            const nextSpecialStartIndex = this.findNextSpecialStartIndex(text, allowedSpecial, startIndex, this.specialTokenPatternRegex);
-            const endIndex = nextSpecialStartIndex !== undefined
-                ? nextSpecialStartIndex
-                : text.length;
-            const textSegment = text.slice(startIndex, endIndex - startIndex);
-            for (const [match] of textSegment.matchAll(this.tokenSplitRegex)) {
-                const token = this.getBpeRankFromString(match);
-                if (token !== undefined) {
-                    lastTokenLength = 1;
-                    yield [token];
-                    continue;
-                }
-                const tokens = this.bytePairEncode(match);
-                lastTokenLength = tokens.length;
-                yield tokens;
-            }
-            if (nextSpecialStartIndex !== undefined) {
-                const specialToken = text.slice(Math.max(0, nextSpecialStartIndex));
-                const specialTokenValue = this.specialTokensEncoder.get(specialToken);
-                if (specialTokenValue === undefined) {
-                    throw new Error(`Special token "${specialToken}" is not in the special token encoder.`);
-                }
-                yield [specialTokenValue];
-                startIndex = nextSpecialStartIndex + specialToken.length;
-                lastTokenLength = 0;
-            }
-            else {
-                break;
-            }
-        }
-        return lastTokenLength;
-    }
-    findNextSpecialStartIndex(text, allowedSpecial, startIndex, specialRegex) {
+    findNextSpecialStartIndex(text, allowedSpecial, startIndex) {
         let searchIndex = startIndex;
         // eslint-disable-next-line no-constant-condition
         while (true) {
-            const nextSpecialMatch = specialRegex.exec(text.slice(Math.max(0, searchIndex)));
+            const nextSpecialMatch = this.specialTokenPatternRegex.exec(text.slice(Math.max(0, searchIndex)));
             if (!nextSpecialMatch) {
                 return undefined;
             }
-            const [specialToken] = nextSpecialMatch;
+            const specialToken = nextSpecialMatch[0];
             if (allowedSpecial?.has(specialToken)) {
                 return nextSpecialMatch.index + searchIndex;
             }
             searchIndex = nextSpecialMatch.index + searchIndex + 1;
         }
     }
-    *decodeNative(tokens) {
-        for (const token of tokens) {
-            const tokenBytes = this.tryDecodeToken(token);
-            if (tokenBytes) {
-                yield tokenBytes;
-            }
-        }
-    }
-    async *decodeNativeAsync(tokens) {
-        for await (const token of tokens) {
-            const tokenBytesOrString = this.tryDecodeToken(token);
-            if (tokenBytesOrString) {
-                yield tokenBytesOrString;
-            }
-        }
-    }
     tryDecodeToken(tokenRank) {
-        const value = this.bytePairEncoder[tokenRank];
+        const value = this.bytePairRankDecoder[tokenRank];
         if (typeof value === 'string') {
             return value;
         }
         if (typeof value === 'object') {
-            const fromBinary = this.bytePairRanksDecoder.get(tokenRank);
+            const fromBinary = this.bytePairNonUtfRankDecoder.get(tokenRank);
             if (fromBinary) {
                 return fromBinary;
             }
@@ -186,80 +260,74 @@ export class BytePairEncodingCore {
             return [this.getBpeRankFromStringOrThrow(input)];
         }
         const inputBytes = this.textEncoder.encode(input);
-        return this.bytePairMerge(inputBytes, (start, end) => {
-            const key = inputBytes.subarray(start, end);
-            return this.getBpeRankFromBytesOrThrow(key);
-        });
+        return this.bytePairMerge(inputBytes);
     }
     bytePairMerge(
     // Input array of bytes to process
-    piece, 
-    // Function to apply to each final segment after merging
-    getByteForRange) {
-        // Create an array of partition objects. Each partition tracks the start index in 'piece'
-        // and a rank value for adjacent pairs (initially set to positive infinity).
-        const partitions = Array.from({ length: piece.length + 1 }, (_, i) => ({
-            start: i,
-            rank: Number.POSITIVE_INFINITY, // Rank starts at infinity (unmerged)
-        }));
-        // Helper function to get the rank of a byte pair starting at 'startIndex'.
-        // 'skip' determines how far we look ahead (usually 0, for consecutive pairs).
-        const getRank = (startIndex, skip) => {
-            if (startIndex + skip + 2 >= partitions.length) {
-                // Avoid out-of-bounds errors, return undefined when no valid pair exists
-                return undefined;
+    piece) {
+        // 'starts' holds the start indices of each partition
+        const starts = [];
+        // 'ranks' holds the BPE ranks of each partition pair
+        const ranks = [];
+        // Helper function to get the rank of a byte pair starting at 'startIndex'
+        const getRank = (startIndex, pairStart = starts[startIndex], pairEnd = starts[startIndex + 2]) => {
+            if (pairEnd === undefined) {
+                // No valid pair exists
+                return Number.POSITIVE_INFINITY;
             }
-            // Get the byte pair by extracting a subarray starting at 'startIndex' and ending at
-            // the start of the partition after 'skip + 2'.
-            const key = piece.subarray(partitions[startIndex].start, partitions[startIndex + skip + 2].start);
-            // Retrieve the rank of this byte pair from the BPE rank function
-            return this.getBpeRankFromBytes(key);
+            // Extract the byte pair
+            const key = piece.subarray(pairStart, pairEnd);
+            // Retrieve the BPE rank of this byte pair (if it exists)
+            const rank = this.getBpeRankFromBytes(key);
+            return rank ?? Number.POSITIVE_INFINITY;
         };
-        // Initialize the ranks for all adjacent pairs in the array
-        for (let i = 0; i < partitions.length - 2; i++) {
-            // Get the rank for the pair starting at index 'i'
-            const rank = getRank(i, 0);
-            if (rank !== undefined) {
-                // Assign the rank to the partition at index 'i'
-                partitions[i].rank = rank;
+        // Initialize the 'starts' array with all possible start indices
+        for (let i = 0; i <= piece.length; i++) {
+            starts.push(i);
+            if (i < piece.length - 1) {
+                // Initialize the BPE values for all adjacent pairs
+                ranks.push(getRank(i, i, i + 2));
+            }
+            else {
+                // Initialize BPE values to infinity for the last pair
+                ranks.push(Number.POSITIVE_INFINITY);
             }
         }
         // Iteratively merge byte pairs until no more useful merges can be done
-        while (partitions.length > 1) {
-            let minRank = Number.POSITIVE_INFINITY;
-            let minRankIdx = 0;
-            // Find the partition with the minimum rank, i.e., the most important pair to merge next
-            let i = 0;
-            for (const partition of partitions) {
-                if (partition.rank < minRank) {
-                    minRank = partition.rank;
-                    minRankIdx = i;
+        while (starts.length > 1) {
+            let lowestRank = Number.POSITIVE_INFINITY;
+            let lowestPartitionIndex = -1;
+            // Find the partition with the minimum rank
+            for (let i = 0; i < ranks.length - 1; i++) {
+                const rank = ranks[i];
+                if (rank < lowestRank) {
+                    lowestRank = rank;
+                    lowestPartitionIndex = i;
                 }
-                i++;
             }
             // If no valid pair is left to merge, exit the loop
-            if (minRank === Number.POSITIVE_INFINITY) {
+            if (lowestRank === Number.POSITIVE_INFINITY ||
+                lowestPartitionIndex === -1) {
                 break;
             }
-            // Update the rank of the partition after the merged one
-            partitions[minRankIdx].rank =
-                getRank(minRankIdx, 1) ?? Number.POSITIVE_INFINITY;
-            // Update the rank of the partition before the merged one (if exists)
-            if (minRankIdx > 0) {
-                partitions[minRankIdx - 1].rank =
-                    getRank(minRankIdx - 1, 1) ?? Number.POSITIVE_INFINITY;
+            // Merge the pair at 'lowestPartitionIndex' by removing the next start index
+            starts.splice(lowestPartitionIndex + 1, 1);
+            // Remove the BPE value of the merged pair
+            ranks.splice(lowestPartitionIndex, 1);
+            // Update the current merged pair's rank
+            ranks[lowestPartitionIndex] = getRank(lowestPartitionIndex);
+            // Update the rank of the previous pair, if it exists
+            if (lowestPartitionIndex > 0) {
+                ranks[lowestPartitionIndex - 1] = getRank(lowestPartitionIndex - 1);
             }
-            // Merge by removing the partition after the one we just merged
-            partitions.splice(minRankIdx + 1, 1);
         }
         // Create the final output by applying the transform function to each partitioned range
         const output = [];
-        for (let i = 0; i < partitions.length - 1; i++) {
-            output.push(getByteForRange(
-            // start index
-            partitions[i].start, 
-            // end index
-            partitions[i + 1].start));
+        for (let i = 0; i < starts.length - 1; i++) {
+            const pairStart = starts[i];
+            const pairEnd = starts[i + 1];
+            const bpeValue = this.getBpeRankFromBytesOrThrow(piece.subarray(pairStart, pairEnd));
+            output.push(bpeValue);
         }
         return output;
     }

package/esm/BytePairEncodingCore.js.map

@@ -1 +1 @@
-{"version":3,"file":"BytePairEncodingCore.js","sourceRoot":"","sources":["../src/BytePairEncodingCore.ts"],"names":[],"mappings":"AAAA,gCAAgC;AAEhC,OAAO,EAAE,kBAAkB,EAAE,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AAC9E,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAA;AAUxC,MAAM,OAAO,oBAAoB;IACtB,mBAAmB,CAAQ;IAC5B,eAAe,CAAkB;IACjC,2BAA2B,CAAiC;IAC5D,oBAAoB,GAAG,IAAI,GAAG,EAAsB,CAAA;IACpD,eAAe,CAAQ;IACvB,oBAAoB,CAAqB;IACzC,oBAAoB,CAAqB;IACzC,wBAAwB,CAAQ;IAChC,aAAa,CAAqB;IAClC,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;IAEvC,YAAY,EACV,sBAAsB,EAAE,eAAe,EACvC,mBAAmB,EAAE,mBAAmB,EACxC,eAAe,GACQ;QACvB,IAAI,CAAC,eAAe,GAAG,eAAe,CAAA;QACtC,IAAI,CAAC,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAA;QAE9C,iEAAiE;QACjE,IAAI,CAAC,mBAAmB,GAAG,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,MAAM,CAAA;QAC9D,MAAM,YAAY,GAA2B,EAAE,CAAA;QAC/C,6BAA6B;QAC7B,eAAe,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YACtC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;gBACnC,OAAM;YACR,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,CAAA;YACvC,YAAY,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAA;YACpC,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;QAChD,CAAC,CAAC,CAAA;QACF,IAAI,CAAC,2BAA2B,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC5D,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAC/B,CAAA;QACD,IAAI,CAAC,oBAAoB,GAAG,mBAAmB,IAAI,IAAI,GAAG,EAAkB,CAAA;QAC5E,IAAI,CAAC,oBAAoB,GAAG,mBAAmB;YAC7C,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,mBAAmB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;YACvE,CAAC,CAAC,IAAI,GAAG,EAAkB,CAAA;QAC7B,IAAI,CAAC,eAAe,GAAG,eAAe,CAAA;QAEtC,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;QACrE,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACnC,IAAI,CAAC;YACH,IAAI,CAAC,wBAAwB,GAAG,IAAI,MAAM,CAAC,WAAW,CAAC,CAAA;QACzD,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAA;QACxD,CAAC;IACH,CAAC;IAED,oBAAoB,CAAC,GAAW;QAC9B,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACpC,CAAC;IAED,2BAA2B,CAAC,GAAW;QACrC,MAAM,KAAK,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,CAAA;QAC5C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CACb,wDAAwD,GAAG,EAAE,CAC9D,CAAA;QACH,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,mBAAmB,CAAC,GAAe;QACjC,MAAM,WAAW,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAA;QAE3C,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAA;QAC/C,CAAC;QAED,2CAA2C;QAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAA;QACpC,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;YACjB,OAAO,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAAE,CAAC,CAAC,CAAC,CAAA;QACpD,CAAC;QAED,OAAO,SAAS,CAAA;IAClB,CAAC;IAED,0BAA0B,CAAC,GAAe;QACxC,MAAM,KAAK,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAA;QAC3C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CACb,wDAAwD,GAAG,CAAC,QAAQ,EAAE,EAAE,CACzE,CAAA;QACH,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,mCAAmC;IACnC,YAAY,CAAC,GAAe;QAC1B,IAAI,GAAG,GAAG,CAAC,CAAA;QACX,IAAI,IAAI,GAAG,IAAI,CAAC,2BAA2B,CAAC,MAAM,GAAG,CAAC,CAAA;QAEtD,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;YACnB,sCAAsC;YACtC,MAAM,GAAG,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,2BAA2B,CAAC,GAAG,CAAE,CAAC,CAAC,CAAC,CAAA;YACxD,IAAI,GAAG,GAAG,CAAC,CAAA;YACX,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7D,GAAG,GAAG,MAAM,CAAC,CAAC,CAAE,GAAG,GAAG,CAAC,CAAC,CAAE,CAAA;gBAC1B,IAAI,GAAG,KAAK,CAAC;oBAAE,MAAK;YACtB,CAAC;YACD,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC;gBACd,GAAG,GAAG,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAA;YAClC,CAAC;YACD,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC;gBACd,OAAO,GAAG,CAAA;YACZ,CAAC;YACD,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;gBACZ,GAAG,GAAG,GAAG,GAAG,CAAC,CAAA;YACf,CAAC;iBAAM,CAAC;gBACN,IAAI,GAAG,GAAG,GAAG,CAAC,CAAA;YAChB,CAAC;QACH,CAAC;QACD,OAAO,CAAC,CAAC,CAAA;IACX,CAAC;IAED,CAAC,YAAY,CACX,IAAY,EACZ,cAA4B;QAE5B,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,IAAI,eAAe,GAAG,CAAC,CAAA;QAEvB,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,qBAAqB,GAAG,IAAI,CAAC,yBAAyB,CAC1D,IAAI,EACJ,cAAc,EACd,UAAU,EACV,IAAI,CAAC,wBAAwB,CAC9B,CAAA;YAED,MAAM,QAAQ,GACZ,qBAAqB,KAAK,SAAS;gBACjC,CAAC,CAAC,qBAAqB;gBACvB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAA;YAEjB,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,GAAG,UAAU,CAAC,CAAA;YAEjE,KAAK,MAAM,CAAC,KAAK,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC;gBACjE,MAAM,KAAK,GAAG,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAA;gBAC9C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACxB,eAAe,GAAG,CAAC,CAAA;oBACnB,MAAM,CAAC,KAAK,CAAC,CAAA;oBAEb,SAAQ;gBACV,CAAC;gBAED,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;gBACzC,eAAe,GAAG,MAAM,CAAC,MAAM,CAAA;gBAC/B,MAAM,MAAM,CAAA;YACd,CAAC;YAED,IAAI,qBAAqB,KAAK,SAAS,EAAE,CAAC;gBACxC,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,qBAAqB,CAAC,CAAC,CAAA;gBACnE,MAAM,iBAAiB,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;gBACrE,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;oBACpC,MAAM,IAAI,KAAK,CACb,kBAAkB,YAAY,wCAAwC,CACvE,CAAA;gBACH,CAAC;gBACD,MAAM,CAAC,iBAAiB,CAAC,CAAA;gBACzB,UAAU,GAAG,qBAAqB,GAAG,YAAY,CAAC,MAAM,CAAA;gBACxD,eAAe,GAAG,CAAC,CAAA;YACrB,CAAC;iBAAM,CAAC;gBACN,MAAK;YACP,CAAC;QACH,CAAC;QAED,OAAO,eAAe,CAAA;IACxB,CAAC;IAED,yBAAyB,CACvB,IAAY,EACZ,cAAuC,EACvC,UAAkB,EAClB,YAAoB;QAEpB,IAAI,WAAW,GAAG,UAAU,CAAA;QAE5B,iDAAiD;QACjD,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,gBAAgB,GAAG,YAAY,CAAC,IAAI,CACxC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,CACrC,CAAA;YAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACtB,OAAO,SAAS,CAAA;YAClB,CAAC;YAED,MAAM,CAAC,YAAY,CAAC,GAAG,gBAAgB,CAAA;YAEvC,IAAI,cAAc,EAAE,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;gBACtC,OAAO,gBAAgB,CAAC,KAAK,GAAG,WAAW,CAAA;YAC7C,CAAC;YAED,WAAW,GAAG,gBAAgB,CAAC,KAAK,GAAG,WAAW,GAAG,CAAC,CAAA;QACxD,CAAC;IACH,CAAC;IAED,CAAC,YAAY,CACX,MAAwB;QAExB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;YAC7C,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,UAAU,CAAA;YAClB,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,CAAC,iBAAiB,CACtB,MAA6B;QAE7B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACjC,MAAM,kBAAkB,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;YACrD,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,kBAAkB,CAAA;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAED,cAAc,CAAC,SAAiB;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAA;QAC7C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,OAAO,KAAK,CAAA;QACd,CAAC;QACD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,MAAM,UAAU,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;YAC3D,IAAI,UAAU,EAAE,CAAC;gBACf,OAAO,UAAU,CAAA;YACnB,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;IACjD,CAAC;IAED,cAAc,CAAC,KAAa;QAC1B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;YACzD,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAC,CAAA;QAClD,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAEjD,OAAO,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;YACnD,MAAM,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;YAC3C,OAAO,IAAI,CAAC,0BAA0B,CAAC,GAAG,CAAC,CAAA;QAC7C,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,aAAa;IACX,kCAAkC;IAClC,KAAiB;IACjB,wDAAwD;IACxD,eAAuD;QAEvD,yFAAyF;QACzF,4EAA4E;QAC5E,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACrE,KAAK,EAAE,CAAC;YACR,IAAI,EAAE,MAAM,CAAC,iBAAiB,EAAE,qCAAqC;SACtE,CAAC,CAAC,CAAA;QAEH,2EAA2E;QAC3E,8EAA8E;QAC9E,MAAM,OAAO,GAAG,CAAC,UAAkB,EAAE,IAAY,EAAsB,EAAE;YACvE,IAAI,UAAU,GAAG,IAAI,GAAG,CAAC,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;gBAC/C,yEAAyE;gBACzE,OAAO,SAAS,CAAA;YAClB,CAAC;YAED,oFAAoF;YACpF,+CAA+C;YAC/C,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,CACxB,UAAU,CAAC,UAAU,CAAE,CAAC,KAAK,EAC7B,UAAU,CAAC,UAAU,GAAG,IAAI,GAAG,CAAC,CAAE,CAAC,KAAK,CACzC,CAAA;YAED,iEAAiE;YACjE,OAAO,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAA;QACtC,CAAC,CAAA;QAED,2DAA2D;QAC3D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,kDAAkD;YAClD,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;YAC1B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;gBACvB,gDAAgD;gBAChD,UAAU,CAAC,CAAC,CAAE,CAAC,IAAI,GAAG,IAAI,CAAA;YAC5B,CAAC;QACH,CAAC;QAED,uEAAuE;QACvE,OAAO,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,IAAI,OAAO,GAAG,MAAM,CAAC,iBAAiB,CAAA;YACtC,IAAI,UAAU,GAAG,CAAC,CAAA;YAElB,wFAAwF;YACxF,IAAI,CAAC,GAAG,CAAC,CAAA;YACT,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;gBACnC,IAAI,SAAS,CAAC,IAAI,GAAG,OAAO,EAAE,CAAC;oBAC7B,OAAO,GAAG,SAAS,CAAC,IAAI,CAAA;oBACxB,UAAU,GAAG,CAAC,CAAA;gBAChB,CAAC;gBACD,CAAC,EAAE,CAAA;YACL,CAAC;YAED,mDAAmD;YACnD,IAAI,OAAO,KAAK,MAAM,CAAC,iBAAiB,EAAE,CAAC;gBACzC,MAAK;YACP,CAAC;YAED,wDAAwD;YACxD,UAAU,CAAC,UAAU,CAAE,CAAC,IAAI;gBAC1B,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,iBAAiB,CAAA;YAEpD,qEAAqE;YACrE,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;gBACnB,UAAU,CAAC,UAAU,GAAG,CAAC,CAAE,CAAC,IAAI;oBAC9B,OAAO,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,iBAAiB,CAAA;YAC1D,CAAC;YAED,+DAA+D;YAC/D,UAAU,CAAC,MAAM,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;QACtC,CAAC;QAED,uFAAuF;QACvF,MAAM,MAAM,GAAa,EAAE,CAAA;QAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,MAAM,CAAC,IAAI,CACT,eAAe;YACb,cAAc;YACd,UAAU,CAAC,CAAC,CAAE,CAAC,KAAK;YACpB,YAAY;YACZ,UAAU,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,KAAK,CACzB,CACF,CAAA;QACH,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC;CACF"}
+{"version":3,"file":"BytePairEncodingCore.js","sourceRoot":"","sources":["../src/BytePairEncodingCore.ts"],"names":[],"mappings":"AAAA,gCAAgC;AAEhC,OAAO,EAAE,kBAAkB,EAAE,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AAC9E,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAA;AAUxC,MAAM,WAAW,GAAG,IAAI,UAAU,CAAC,CAAC,CAAC,CAAA;AAErC,MAAM,CAAC,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC,CAAA;AAE9C,MAAM,OAAO,oBAAoB;IACtB,0BAA0B,CAAQ;IAC3C;;;;OAIG;IACK,mBAAmB,CAAkB;IACrC,yBAAyB,GAAG,IAAI,GAAG,EAAsB,CAAA;IACzD,2BAA2B,CAAiC;IACpE;;;;OAIG;IACK,yBAAyB,CAAqB;IAC9C,eAAe,CAAQ;IACvB,oBAAoB,CAAqB;IACzC,oBAAoB,CAAqB;IACzC,wBAAwB,CAAQ;IAChC,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;IAEvC,YAAY,EACV,sBAAsB,EAAE,eAAe,EACvC,mBAAmB,EAAE,mBAAmB,EACxC,eAAe,GACQ;QACvB,IAAI,CAAC,mBAAmB,GAAG,eAAe,CAAA;QAC1C,IAAI,CAAC,yBAAyB,GAAG,IAAI,GAAG,EAAkB,CAAA;QAE1D,iEAAiE;QACjE,IAAI,CAAC,0BAA0B,GAAG,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,MAAM,CAAA;QACrE,MAAM,YAAY,GAA2B,EAAE,CAAA;QAC/C,6BAA6B;QAC7B,eAAe,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YACtC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;gBAC/C,OAAM;YACR,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,CAAA;YACvC,YAAY,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAA;YACpC,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;QACrD,CAAC,CAAC,CAAA;QACF,IAAI,CAAC,2BAA2B,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC5D,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAC/B,CAAA;QACD,IAAI,CAAC,oBAAoB,GAAG,mBAAmB,IAAI,IAAI,GAAG,EAAkB,CAAA;QAC5E,IAAI,CAAC,oBAAoB,GAAG,mBAAmB;YAC7C,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,mBAAmB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;YACvE,CAAC,CAAC,IAAI,GAAG,EAAkB,CAAA;QAC7B,IAAI,CAAC,eAAe,GAAG,eAAe,CAAA;QAEtC,MAAM,oBAAoB,GAAG,CAAC,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CACpE,YAAY,CACb,CAAA;QACD,MAAM,qBAAqB,GAAG,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAC5D,IAAI,CAAC;YACH,IAAI,CAAC,wBAAwB,GAAG,IAAI,MAAM,CAAC,qBAAqB,CAAC,CAAA;QACnE,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAA;QACxD,CAAC;IACH,CAAC;IAED,CAAC,qBAAqB,CACpB,IAAY,EACZ,cAA4B;QAE5B,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,IAAI,eAAe,GAAG,CAAC,CAAA;QAEvB,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,qBAAqB,GAAG,IAAI,CAAC,yBAAyB,CAC1D,IAAI,EACJ,cAAc,EACd,UAAU,CACX,CAAA;YAED,MAAM,QAAQ,GACZ,qBAAqB,KAAK,SAAS;gBACjC,CAAC,CAAC,qBAAqB;gBACvB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAA;YAEjB,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,GAAG,UAAU,CAAC,CAAA;YAEjE,KAAK,MAAM,CAAC,KAAK,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC;gBACjE,MAAM,KAAK,GAAG,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAA;gBAC9C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACxB,eAAe,GAAG,CAAC,CAAA;oBACnB,MAAM,CAAC,KAAK,CAAC,CAAA;oBAEb,SAAQ;gBACV,CAAC;gBAED,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;gBACzC,eAAe,GAAG,MAAM,CAAC,MAAM,CAAA;gBAC/B,MAAM,MAAM,CAAA;YACd,CAAC;YAED,IAAI,qBAAqB,KAAK,SAAS,EAAE,CAAC;gBACxC,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,qBAAqB,CAAC,CAAC,CAAA;gBACnE,MAAM,iBAAiB,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;gBACrE,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;oBACpC,MAAM,IAAI,KAAK,CACb,kBAAkB,YAAY,wCAAwC,CACvE,CAAA;gBACH,CAAC;gBACD,MAAM,CAAC,iBAAiB,CAAC,CAAA;gBACzB,UAAU,GAAG,qBAAqB,GAAG,YAAY,CAAC,MAAM,CAAA;gBACxD,eAAe,GAAG,CAAC,CAAA;YACrB,CAAC;iBAAM,CAAC;gBACN,MAAK;YACP,CAAC;QACH,CAAC;QAED,OAAO,eAAe,CAAA;IACxB,CAAC;IAED,YAAY,CAAC,IAAY,EAAE,cAA4B;QACrD,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,MAAM,WAAW,GAAa,EAAE,CAAA,CAAC,kCAAkC;QAEnE,iDAAiD;QACjD,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,qBAAqB,GAAG,IAAI,CAAC,yBAAyB,CAC1D,IAAI,EACJ,cAAc,EACd,UAAU,CACX,CAAA;YAED,MAAM,QAAQ,GACZ,qBAAqB,KAAK,SAAS;gBACjC,CAAC,CAAC,qBAAqB;gBACvB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAA;YAEjB,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,GAAG,UAAU,CAAC,CAAA;YAEjE,KAAK,MAAM,CAAC,KAAK,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC;gBACjE,MAAM,KAAK,GAAG,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAA;gBAC9C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACxB,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;oBAEvB,SAAQ;gBACV,CAAC;gBAED,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;gBACzC,WAAW,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAA;YAC7B,CAAC;YAED,IAAI,qBAAqB,KAAK,SAAS,EAAE,CAAC;gBACxC,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,qBAAqB,CAAC,CAAC,CAAA;gBACnE,MAAM,iBAAiB,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;gBACrE,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;oBACpC,MAAM,IAAI,KAAK,CACb,kBAAkB,YAAY,wCAAwC,CACvE,CAAA;gBACH,CAAC;gBACD,WAAW,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;gBACnC,UAAU,GAAG,qBAAqB,GAAG,YAAY,CAAC,MAAM,CAAA;YAC1D,CAAC;iBAAM,CAAC;gBACN,MAAK;YACP,CAAC;QACH,CAAC;QAED,OAAO,WAAW,CAAA;IACpB,CAAC;IAED,CAAC,qBAAqB,CACpB,MAAwB;QAExB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;YAC7C,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,UAAU,CAAA;YAClB,CAAC;QACH,CAAC;IACH,CAAC;IAED,YAAY,CAAC,MAAwB;QACnC,IAAI,OAAO,GAAG,EAAE,CAAA;QAChB,IAAI,SAAS,GAAG,WAAW,CAAA;QAE3B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;YAC7C,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CAAC,SAAS,KAAK,mCAAmC,CAAC,CAAA;YACpE,CAAC;YACD,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;gBACnC,IAAI,SAAS,KAAK,WAAW,EAAE,CAAC;oBAC9B,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;oBACtD,SAAS,GAAG,WAAW,CAAA;gBACzB,CAAC;gBACD,OAAO,IAAI,UAAU,CAAA;YACvB,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAA;gBACtE,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;gBACxB,SAAS,CAAC,GAAG,CAAC,UAAU,EAAE,SAAS,CAAC,MAAM,CAAC,CAAA;gBAC3C,SAAS,GAAG,SAAS,CAAA;YACvB,CAAC;QACH,CAAC;QAED,IAAI,SAAS,KAAK,WAAW,EAAE,CAAC;YAC9B,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;QACxD,CAAC;QACD,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,KAAK,CAAC,CAAC,yBAAyB,CAC9B,MAA6B;QAE7B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACjC,MAAM,kBAAkB,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;YACrD,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,kBAAkB,CAAA;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAEO,oBAAoB,CAAC,GAAW;QACtC,OAAO,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IAChD,CAAC;IAEO,2BAA2B,CAAC,GAAW;QAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,CAAA;QAC5C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CACb,wDAAwD,GAAG,EAAE,CAC9D,CAAA;QACH,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAEO,mBAAmB,CAAC,GAAe;QACzC,MAAM,WAAW,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAA;QAE3C,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAA;QAC/C,CAAC;QAED,2CAA2C;QAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAA;QACpC,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;YACjB,OAAO,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAAE,CAAC,CAAC,CAAC,CAAA;QACpD,CAAC;QAED,OAAO,SAAS,CAAA;IAClB,CAAC;IAEO,0BAA0B,CAAC,GAAe;QAChD,MAAM,KAAK,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAA;QAC3C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CACb,wDAAwD,GAAG,CAAC,QAAQ,EAAE,EAAE,CACzE,CAAA;QACH,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,mCAAmC;IAC3B,YAAY,CAAC,GAAe;QAClC,IAAI,GAAG,GAAG,CAAC,CAAA;QACX,IAAI,IAAI,GAAG,IAAI,CAAC,2BAA2B,CAAC,MAAM,GAAG,CAAC,CAAA;QAEtD,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;YACnB,sCAAsC;YACtC,MAAM,GAAG,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,2BAA2B,CAAC,GAAG,CAAE,CAAC,CAAC,CAAC,CAAA;YACxD,IAAI,GAAG,GAAG,CAAC,CAAA;YACX,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;YACrD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnC,GAAG,GAAG,MAAM,CAAC,CAAC,CAAE,GAAG,GAAG,CAAC,CAAC,CAAE,CAAA;gBAC1B,IAAI,GAAG,KAAK,CAAC;oBAAE,MAAK;YACtB,CAAC;YACD,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC;gBACd,GAAG,GAAG,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAA;YAClC,CAAC;YACD,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC;gBACd,OAAO,GAAG,CAAA;YACZ,CAAC;YACD,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;gBACZ,GAAG,GAAG,GAAG,GAAG,CAAC,CAAA;YACf,CAAC;iBAAM,CAAC;gBACN,IAAI,GAAG,GAAG,GAAG,CAAC,CAAA;YAChB,CAAC;QACH,CAAC;QACD,OAAO,CAAC,CAAC,CAAA;IACX,CAAC;IAEO,yBAAyB,CAC/B,IAAY,EACZ,cAAuC,EACvC,UAAkB;QAElB,IAAI,WAAW,GAAG,UAAU,CAAA;QAE5B,iDAAiD;QACjD,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,gBAAgB,GAAG,IAAI,CAAC,wBAAwB,CAAC,IAAI,CACzD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,CACrC,CAAA;YAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACtB,OAAO,SAAS,CAAA;YAClB,CAAC;YAED,MAAM,YAAY,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAA;YAExC,IAAI,cAAc,EAAE,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;gBACtC,OAAO,gBAAgB,CAAC,KAAK,GAAG,WAAW,CAAA;YAC7C,CAAC;YAED,WAAW,GAAG,gBAAgB,CAAC,KAAK,GAAG,WAAW,GAAG,CAAC,CAAA;QACxD,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,SAAiB;QACtC,MAAM,KAAK,GAAG,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAA;QACjD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,OAAO,KAAK,CAAA;QACd,CAAC;QACD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,MAAM,UAAU,GAAG,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;YAChE,IAAI,UAAU,EAAE,CAAC;gBACf,OAAO,UAAU,CAAA;YACnB,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;IACjD,CAAC;IAEO,cAAc,CAAC,KAAa;QAClC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;YACzD,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAC,CAAA;QAClD,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAEjD,OAAO,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAA;IACvC,CAAC;IAEO,aAAa;IACnB,kCAAkC;IAClC,KAAiB;QAEjB,qDAAqD;QACrD,MAAM,MAAM,GAAa,EAAE,CAAA;QAC3B,qDAAqD;QACrD,MAAM,KAAK,GAAa,EAAE,CAAA;QAE1B,0EAA0E;QAC1E,MAAM,OAAO,GAAG,CACd,UAAkB,EAClB,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,EAC9B,OAAO,GAAG,MAAM,CAAC,UAAU,GAAG,CAAC,CAAC,EACxB,EAAE;YACV,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,uBAAuB;gBACvB,OAAO,MAAM,CAAC,iBAAiB,CAAA;YACjC,CAAC;YAED,wBAAwB;YACxB,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;YAE9C,yDAAyD;YACzD,MAAM,IAAI,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAA;YAC1C,OAAO,IAAI,IAAI,MAAM,CAAC,iBAAiB,CAAA;QACzC,CAAC,CAAA;QAED,gEAAgE;QAChE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACd,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,mDAAmD;gBACnD,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YAClC,CAAC;iBAAM,CAAC;gBACN,sDAAsD;gBACtD,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAA;YACtC,CAAC;QACH,CAAC;QAED,uEAAuE;QACvE,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,IAAI,UAAU,GAAG,MAAM,CAAC,iBAAiB,CAAA;YACzC,IAAI,oBAAoB,GAAG,CAAC,CAAC,CAAA;YAE7B,2CAA2C;YAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC1C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;gBACtB,IAAI,IAAI,GAAG,UAAU,EAAE,CAAC;oBACtB,UAAU,GAAG,IAAI,CAAA;oBACjB,oBAAoB,GAAG,CAAC,CAAA;gBAC1B,CAAC;YACH,CAAC;YAED,mDAAmD;YACnD,IACE,UAAU,KAAK,MAAM,CAAC,iBAAiB;gBACvC,oBAAoB,KAAK,CAAC,CAAC,EAC3B,CAAC;gBACD,MAAK;YACP,CAAC;YAED,4EAA4E;YAC5E,MAAM,CAAC,MAAM,CAAC,oBAAoB,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;YAC1C,0CAA0C;YAC1C,KAAK,CAAC,MAAM,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAA;YAErC,wCAAwC;YACxC,KAAK,CAAC,oBAAoB,CAAC,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAA;YAE3D,qDAAqD;YACrD,IAAI,oBAAoB,GAAG,CAAC,EAAE,CAAC;gBAC7B,KAAK,CAAC,oBAAoB,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,oBAAoB,GAAG,CAAC,CAAC,CAAA;YACrE,CAAC;QACH,CAAC;QAED,uFAAuF;QACvF,MAAM,MAAM,GAAa,EAAE,CAAA;QAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAA;YAC3B,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,0BAA0B,CAC9C,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CACnC,CAAA;YACD,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACvB,CAAC;QACD,OAAO,MAAM,CAAA;IACf,CAAC;CACF"}

package/esm/GptEncoding.d.ts

@@ -20,17 +20,18 @@ export declare class GptEncoding {
     static FimPrefix: string;
     static FimSuffix: string;
     modelName?: ModelName;
-    private decoder;
     private bytePairEncodingCoreProcessor;
     private specialTokenMapping;
     private specialTokensSet;
     private allSpecialTokenRegex;
+    private defaultSpecialTokenConfig;
     private constructor();
     static getEncodingApi(encodingName: EncodingName, getMergeableRanks: GetMergeableRanksFn): GptEncoding;
     static getEncodingApiForModel(modelName: ModelName, getMergeableRanks: GetMergeableRanksFn): GptEncoding;
     static getEncodingApiAsync(encodingName: EncodingName, getMergeableRanks: GetMergeableRanksAsyncFn): Promise<GptEncoding>;
     static getEncodingApiForModelAsync(modelName: ModelName, getMergeableRanks: GetMergeableRanksAsyncFn): Promise<GptEncoding>;
-    encodeGenerator(lineToEncode: string, { allowedSpecial, disallowedSpecial }?: EncodeOptions): Generator<number[], number, undefined>;
+    private processSpecialTokens;
+    encodeGenerator(lineToEncode: string, encodeOptions?: EncodeOptions): Generator<number[], number, undefined>;
     encode(lineToEncode: string, encodeOptions?: EncodeOptions): number[];
     /**
      * Progressively tokenizes an OpenAI chat.
@@ -52,7 +53,8 @@ export declare class GptEncoding {
      * @returns {false | number} false if token limit is exceeded, otherwise the number of tokens
      */
     isWithinTokenLimit(input: string | Iterable<ChatMessage>, tokenLimit: number): false | number;
-    decodeGenerator(inputTokensToDecode: Iterable<number>): Generator<string, void>;
-    decodeAsyncGenerator(inputTokensToDecode: AsyncIterable<number>): AsyncGenerator<string, void>;
     decode(inputTokensToDecode: Iterable<number>): string;
+    decodeGenerator(inputTokensToDecode: Iterable<number>): Generator<string, void, void>;
+    decodeAsyncGenerator(inputTokensToDecode: AsyncIterable<number>): AsyncGenerator<string, void>;
+    decodeAsync(inputTokensToDecode: AsyncIterable<number>): Promise<string>;
 }