gpt-tokenizer 4.0.0 → 4.0.2

package/cjs/utfUtil.js.map

@@ -1 +1 @@
-{"version":3,"file":"utfUtil.js","names":[],"sources":["../src/utfUtil.ts"],"sourcesContent":["export const isAscii = (codePoint: number) => codePoint <= 0x7f\n\nconst HIGH_SURROGATE_START = 55_296\nconst HIGH_SURROGATE_END = 56_319\n\nexport function endsWithIncompleteUtfPairSurrogate(string: string): boolean {\n  if (string.length === 0) return false\n  // Check if the last character is a high surrogate\n  const lastCharCode = string.charCodeAt(string.length - 1)\n  return (\n    lastCharCode >= HIGH_SURROGATE_START && lastCharCode <= HIGH_SURROGATE_END\n  )\n}\n\nfunction isValidUTF8(bytes: Uint8Array): boolean {\n  let i = 0\n  while (i < bytes.length) {\n    const byte1 = bytes[i]!\n\n    let numBytes = 0\n    let codePoint = 0\n\n    // Determine the number of bytes in the current UTF-8 character\n    if (byte1 <= 0x7f) {\n      // 1-byte character (ASCII)\n      numBytes = 1\n      codePoint = byte1\n    } else if ((byte1 & 0xe0) === 0xc0) {\n      // 2-byte character\n      numBytes = 2\n      codePoint = byte1 & 0x1f\n      if (byte1 <= 0xc1) return false // Overlong encoding not allowed\n    } else if ((byte1 & 0xf0) === 0xe0) {\n      // 3-byte character\n      numBytes = 3\n      codePoint = byte1 & 0x0f\n    } else if ((byte1 & 0xf8) === 0xf0) {\n      // 4-byte character\n      numBytes = 4\n      codePoint = byte1 & 0x07\n      if (byte1 > 0xf4) return false // Code points above U+10FFFF not allowed\n    } else {\n      // Invalid first byte of UTF-8 character\n      return false\n    }\n\n    // Ensure there are enough continuation bytes\n    if (i + numBytes > bytes.length) return false\n\n    // Process the continuation bytes\n    for (let j = 1; j < numBytes; j++) {\n      const byte = bytes[i + j]\n      if (byte === undefined || (byte & 0xc0) !== 0x80) return false // Continuation bytes must start with '10'\n      codePoint = (codePoint << 6) | (byte & 0x3f)\n    }\n\n    // Check for overlong encodings\n    if (numBytes === 2 && codePoint < 0x80) return false // Overlong 2-byte sequence\n    if (numBytes === 3 && codePoint < 2_048) return false // Overlong 3-byte sequence\n    if (numBytes === 4 && codePoint < 65_536) return false // Overlong 4-byte sequence\n\n    // Check for surrogate halves (U+D800 to U+DFFF)\n    if (codePoint >= 55_296 && codePoint <= 57_343) return false\n\n    // Check for code points above U+10FFFF\n    if (codePoint > 1_114_111) return false\n\n    // Move to the next character\n    i += numBytes\n  }\n  return true\n}\n\nconst textDecoder = new TextDecoder('utf8', { fatal: false })\n\nexport function tryConvertToString(arr: Uint8Array): string | undefined {\n  if (!isValidUTF8(arr)) {\n    return undefined\n  }\n  return textDecoder.decode(arr)\n}\n\n// Helper function to compare two Uint8Arrays lexicographically\nexport function compareUint8Arrays(a: Uint8Array, b: Uint8Array): number {\n  const len = Math.min(a.length, b.length)\n  for (let i = 0; i < len; i++) {\n    if (a[i] !== b[i]) {\n      return a[i]! - b[i]!\n    }\n  }\n  return a.length - b.length\n}\n"],"mappings":";;AAAA,MAAa,WAAW,cAAsB,aAAa;AAE3D,MAAM,uBAAuB;AAC7B,MAAM,qBAAqB;AAE3B,SAAgB,mCAAmC,QAAyB;CAC1E,IAAI,OAAO,WAAW,GAAG,OAAO;CAEhC,MAAM,eAAe,OAAO,WAAW,OAAO,SAAS,CAAC;CACxD,OACE,gBAAgB,wBAAwB,gBAAgB;AAE5D;AAEA,SAAS,YAAY,OAA4B;CAC/C,IAAI,IAAI;CACR,OAAO,IAAI,MAAM,QAAQ;EACvB,MAAM,QAAQ,MAAM;EAEpB,IAAI,WAAW;EACf,IAAI,YAAY;EAGhB,IAAI,SAAS,KAAM;GAEjB,WAAW;GACX,YAAY;EACd,OAAO,KAAK,QAAQ,SAAU,KAAM;GAElC,WAAW;GACX,YAAY,QAAQ;GACpB,IAAI,SAAS,KAAM,OAAO;EAC5B,OAAO,KAAK,QAAQ,SAAU,KAAM;GAElC,WAAW;GACX,YAAY,QAAQ;EACtB,OAAO,KAAK,QAAQ,SAAU,KAAM;GAElC,WAAW;GACX,YAAY,QAAQ;GACpB,IAAI,QAAQ,KAAM,OAAO;EAC3B,OAEE,OAAO;EAIT,IAAI,IAAI,WAAW,MAAM,QAAQ,OAAO;EAGxC,KAAK,IAAI,IAAI,GAAG,IAAI,UAAU,KAAK;GACjC,MAAM,OAAO,MAAM,IAAI;GACvB,IAAI,SAAS,KAAA,MAAc,OAAO,SAAU,KAAM,OAAO;GACzD,YAAa,aAAa,IAAM,OAAO;EACzC;EAGA,IAAI,aAAa,KAAK,YAAY,KAAM,OAAO;EAC/C,IAAI,aAAa,KAAK,YAAY,MAAO,OAAO;EAChD,IAAI,aAAa,KAAK,YAAY,OAAQ,OAAO;EAGjD,IAAI,aAAa,SAAU,aAAa,OAAQ,OAAO;EAGvD,IAAI,YAAY,SAAW,OAAO;EAGlC,KAAK;CACP;CACA,OAAO;AACT;AAEA,MAAM,cAAc,IAAI,YAAY,QAAQ,EAAE,OAAO,MAAM,CAAC;AAE5D,SAAgB,mBAAmB,KAAqC;CACtE,IAAI,CAAC,YAAY,GAAG,GAClB;CAEF,OAAO,YAAY,OAAO,GAAG;AAC/B;AAGA,SAAgB,mBAAmB,GAAe,GAAuB;CACvE,MAAM,MAAM,KAAK,IAAI,EAAE,QAAQ,EAAE,MAAM;CACvC,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,KACvB,IAAI,EAAE,OAAO,EAAE,IACb,OAAO,EAAE,KAAM,EAAE;CAGrB,OAAO,EAAE,SAAS,EAAE;AACtB"}
+{"version":3,"file":"utfUtil.js","names":[],"sources":["../src/utfUtil.ts"],"sourcesContent":["export const isAscii = (codePoint: number) => codePoint <= 0x7f\n\nconst HIGH_SURROGATE_START = 55_296\nconst HIGH_SURROGATE_END = 56_319\n\nexport function endsWithIncompleteUtfPairSurrogate(string: string): boolean {\n  if (string.length === 0) return false\n  // Check if the last character is a high surrogate\n  const lastCharCode = string.charCodeAt(string.length - 1)\n  return (\n    lastCharCode >= HIGH_SURROGATE_START && lastCharCode <= HIGH_SURROGATE_END\n  )\n}\n\nfunction isValidUTF8(bytes: Uint8Array): boolean {\n  let i = 0\n  while (i < bytes.length) {\n    const byte1 = bytes[i]!\n\n    let numBytes = 0\n    let codePoint = 0\n\n    // Determine the number of bytes in the current UTF-8 character\n    if (byte1 <= 0x7f) {\n      // 1-byte character (ASCII)\n      numBytes = 1\n      codePoint = byte1\n    } else if ((byte1 & 0xe0) === 0xc0) {\n      // 2-byte character\n      numBytes = 2\n      codePoint = byte1 & 0x1f\n      if (byte1 <= 0xc1) return false // Overlong encoding not allowed\n    } else if ((byte1 & 0xf0) === 0xe0) {\n      // 3-byte character\n      numBytes = 3\n      codePoint = byte1 & 0x0f\n    } else if ((byte1 & 0xf8) === 0xf0) {\n      // 4-byte character\n      numBytes = 4\n      codePoint = byte1 & 0x07\n      if (byte1 > 0xf4) return false // Code points above U+10FFFF not allowed\n    } else {\n      // Invalid first byte of UTF-8 character\n      return false\n    }\n\n    // Ensure there are enough continuation bytes\n    if (i + numBytes > bytes.length) return false\n\n    // Process the continuation bytes\n    for (let j = 1; j < numBytes; j++) {\n      const byte = bytes[i + j]\n      if (byte === undefined || (byte & 0xc0) !== 0x80) return false // Continuation bytes must start with '10'\n      codePoint = (codePoint << 6) | (byte & 0x3f)\n    }\n\n    // Check for overlong encodings\n    if (numBytes === 2 && codePoint < 0x80) return false // Overlong 2-byte sequence\n    if (numBytes === 3 && codePoint < 2_048) return false // Overlong 3-byte sequence\n    if (numBytes === 4 && codePoint < 65_536) return false // Overlong 4-byte sequence\n\n    // Check for surrogate halves (U+D800 to U+DFFF)\n    if (codePoint >= 55_296 && codePoint <= 57_343) return false\n\n    // Check for code points above U+10FFFF\n    if (codePoint > 1_114_111) return false\n\n    // Move to the next character\n    i += numBytes\n  }\n  return true\n}\n\nexport function startsWithByteOrderMark(bytes: Uint8Array): boolean {\n  return (\n    bytes.length >= 3 &&\n    bytes[0] === 0xef &&\n    bytes[1] === 0xbb &&\n    bytes[2] === 0xbf\n  )\n}\n\n// `ignoreBOM: true` keeps a leading U+FEFF in the output instead of silently\n// stripping it, so the bytes EF BB BF map to U+FEFF rather than to an empty string.\nconst textDecoder = new TextDecoder('utf8', { fatal: false, ignoreBOM: true })\n\nexport function tryConvertToString(arr: Uint8Array): string | undefined {\n  if (!isValidUTF8(arr)) {\n    return undefined\n  }\n  return textDecoder.decode(arr)\n}\n\n// Helper function to compare two Uint8Arrays lexicographically\nexport function compareUint8Arrays(a: Uint8Array, b: Uint8Array): number {\n  const len = Math.min(a.length, b.length)\n  for (let i = 0; i < len; i++) {\n    if (a[i] !== b[i]) {\n      return a[i]! - b[i]!\n    }\n  }\n  return a.length - b.length\n}\n"],"mappings":";;AAAA,MAAa,WAAW,cAAsB,aAAa;AAE3D,MAAM,uBAAuB;AAC7B,MAAM,qBAAqB;AAE3B,SAAgB,mCAAmC,QAAyB;CAC1E,IAAI,OAAO,WAAW,GAAG,OAAO;CAEhC,MAAM,eAAe,OAAO,WAAW,OAAO,SAAS,CAAC;CACxD,OACE,gBAAgB,wBAAwB,gBAAgB;AAE5D;AAEA,SAAS,YAAY,OAA4B;CAC/C,IAAI,IAAI;CACR,OAAO,IAAI,MAAM,QAAQ;EACvB,MAAM,QAAQ,MAAM;EAEpB,IAAI,WAAW;EACf,IAAI,YAAY;EAGhB,IAAI,SAAS,KAAM;GAEjB,WAAW;GACX,YAAY;EACd,OAAO,KAAK,QAAQ,SAAU,KAAM;GAElC,WAAW;GACX,YAAY,QAAQ;GACpB,IAAI,SAAS,KAAM,OAAO;EAC5B,OAAO,KAAK,QAAQ,SAAU,KAAM;GAElC,WAAW;GACX,YAAY,QAAQ;EACtB,OAAO,KAAK,QAAQ,SAAU,KAAM;GAElC,WAAW;GACX,YAAY,QAAQ;GACpB,IAAI,QAAQ,KAAM,OAAO;EAC3B,OAEE,OAAO;EAIT,IAAI,IAAI,WAAW,MAAM,QAAQ,OAAO;EAGxC,KAAK,IAAI,IAAI,GAAG,IAAI,UAAU,KAAK;GACjC,MAAM,OAAO,MAAM,IAAI;GACvB,IAAI,SAAS,KAAA,MAAc,OAAO,SAAU,KAAM,OAAO;GACzD,YAAa,aAAa,IAAM,OAAO;EACzC;EAGA,IAAI,aAAa,KAAK,YAAY,KAAM,OAAO;EAC/C,IAAI,aAAa,KAAK,YAAY,MAAO,OAAO;EAChD,IAAI,aAAa,KAAK,YAAY,OAAQ,OAAO;EAGjD,IAAI,aAAa,SAAU,aAAa,OAAQ,OAAO;EAGvD,IAAI,YAAY,SAAW,OAAO;EAGlC,KAAK;CACP;CACA,OAAO;AACT;AAEA,SAAgB,wBAAwB,OAA4B;CAClE,OACE,MAAM,UAAU,KAChB,MAAM,OAAO,OACb,MAAM,OAAO,OACb,MAAM,OAAO;AAEjB;AAIA,MAAM,cAAc,IAAI,YAAY,QAAQ;CAAE,OAAO;CAAO,WAAW;AAAK,CAAC;AAE7E,SAAgB,mBAAmB,KAAqC;CACtE,IAAI,CAAC,YAAY,GAAG,GAClB;CAEF,OAAO,YAAY,OAAO,GAAG;AAC/B;AAGA,SAAgB,mBAAmB,GAAe,GAAuB;CACvE,MAAM,MAAM,KAAK,IAAI,EAAE,QAAQ,EAAE,MAAM;CACvC,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,KACvB,IAAI,EAAE,OAAO,EAAE,IACb,OAAO,EAAE,KAAM,EAAE;CAGrB,OAAO,EAAE,SAAS,EAAE;AACtB"}

package/esm/BytePairEncodingCore.js

@@ -1,9 +1,9 @@
 import { DEFAULT_MERGE_CACHE_SIZE } from "./constants.js";
-import { compareUint8Arrays, isAscii, tryConvertToString } from "./utfUtil.js";
+import { compareUint8Arrays, isAscii, startsWithByteOrderMark, tryConvertToString } from "./utfUtil.js";
 import { escapeRegExp } from "./util.js";
 //#region src/BytePairEncodingCore.ts
 const emptyBuffer = /* @__PURE__ */ new Uint8Array(0);
-const decoder = new TextDecoder("utf8");
+const decoder = new TextDecoder("utf8", { ignoreBOM: true });
 var BytePairEncodingCore = class {
 	mergeableBytePairRankCount;
 	/**
@@ -42,6 +42,10 @@ var BytePairEncodingCore = class {
 			const byteArray = new Uint8Array(value);
 			binaryLookup.push([byteArray, rank]);
 			this.bytePairNonUtfRankDecoder.set(rank, byteArray);
+			if (startsWithByteOrderMark(byteArray)) {
+				const asString = tryConvertToString(byteArray);
+				if (asString !== void 0) this.bytePairStringRankEncoder.set(asString, rank);
+			}
 		});
 		this.bytePairNonUtfSortedEncoder = binaryLookup.sort((a, b) => compareUint8Arrays(a[0], b[0]));
 		this.specialTokensEncoder = specialTokensEncoder ?? /* @__PURE__ */ new Map();
@@ -258,41 +262,81 @@ var BytePairEncodingCore = class {
 		return result;
 	}
 	bytePairMerge(piece) {
-		const starts = [];
-		const ranks = [];
-		const getRank = (startIndex, pairStart = starts[startIndex], pairEnd = starts[startIndex + 2]) => {
-			if (pairEnd === void 0) return Number.POSITIVE_INFINITY;
-			const key = piece.subarray(pairStart, pairEnd);
-			return this.getBpeRankFromBytes(key) ?? Number.POSITIVE_INFINITY;
+		const { length } = piece;
+		const next = new Int32Array(length + 1);
+		const previous = new Int32Array(length + 1);
+		const pairRanks = new Float64Array(length + 1);
+		const live = new Uint8Array(length + 1);
+		const getPairRank = (startIndex) => {
+			const middle = next[startIndex];
+			if (middle >= length) return Number.POSITIVE_INFINITY;
+			return this.getBpeRankFromBytes(piece.subarray(startIndex, next[middle])) ?? Number.POSITIVE_INFINITY;
+		};
+		const heap = [];
+		const isLower = (a, b) => heap[a] < heap[b] || heap[a] === heap[b] && heap[a + 1] < heap[b + 1];
+		const swap = (a, b) => {
+			const rank = heap[a];
+			const startIndex = heap[a + 1];
+			heap[a] = heap[b];
+			heap[a + 1] = heap[b + 1];
+			heap[b] = rank;
+			heap[b + 1] = startIndex;
 		};
-		for (let i = 0; i <= piece.length; i++) {
-			starts.push(i);
-			if (i < piece.length - 1) ranks.push(getRank(i, i, i + 2));
-			else ranks.push(Number.POSITIVE_INFINITY);
+		const push = (rank, startIndex) => {
+			heap.push(rank, startIndex);
+			let index = heap.length - 2;
+			while (index > 0) {
+				const parent = (index >> 1) - 1 >> 1 << 1;
+				if (!isLower(index, parent)) break;
+				swap(index, parent);
+				index = parent;
+			}
+		};
+		const pop = () => {
+			const last = heap.length - 2;
+			swap(0, last);
+			const rank = heap[last];
+			const startIndex = heap[last + 1];
+			heap.length = last;
+			let index = 0;
+			while (true) {
+				const left = index * 2 + 2;
+				const right = left + 2;
+				let lowest = index;
+				if (left < heap.length && isLower(left, lowest)) lowest = left;
+				if (right < heap.length && isLower(right, lowest)) lowest = right;
+				if (lowest === index) break;
+				swap(index, lowest);
+				index = lowest;
+			}
+			return [rank, startIndex];
+		};
+		for (let i = 0; i <= length; i++) {
+			next[i] = i + 1;
+			previous[i] = i - 1;
+			live[i] = 1;
 		}
-		while (starts.length > 1) {
-			let lowestRank = Number.POSITIVE_INFINITY;
-			let lowestPartitionIndex = -1;
-			for (let i = 0; i < ranks.length - 1; i++) {
-				const rank = ranks[i];
-				if (rank < lowestRank) {
-					lowestRank = rank;
-					lowestPartitionIndex = i;
-				}
+		for (let i = 0; i < length - 1; i++) {
+			pairRanks[i] = getPairRank(i);
+			if (pairRanks[i] !== Number.POSITIVE_INFINITY) push(pairRanks[i], i);
+		}
+		while (heap.length > 0) {
+			const [rank, startIndex] = pop();
+			if (!live[startIndex] || pairRanks[startIndex] !== rank) continue;
+			const merged = next[startIndex];
+			live[merged] = 0;
+			next[startIndex] = next[merged];
+			previous[next[merged]] = startIndex;
+			pairRanks[startIndex] = getPairRank(startIndex);
+			if (pairRanks[startIndex] !== Number.POSITIVE_INFINITY) push(pairRanks[startIndex], startIndex);
+			const before = previous[startIndex];
+			if (before >= 0) {
+				pairRanks[before] = getPairRank(before);
+				if (pairRanks[before] !== Number.POSITIVE_INFINITY) push(pairRanks[before], before);
 			}
-			if (lowestRank === Number.POSITIVE_INFINITY || lowestPartitionIndex === -1) break;
-			starts.splice(lowestPartitionIndex + 1, 1);
-			ranks.splice(lowestPartitionIndex, 1);
-			ranks[lowestPartitionIndex] = getRank(lowestPartitionIndex);
-			if (lowestPartitionIndex > 0) ranks[lowestPartitionIndex - 1] = getRank(lowestPartitionIndex - 1);
 		}
 		const output = [];
-		for (let i = 0; i < starts.length - 1; i++) {
-			const pairStart = starts[i];
-			const pairEnd = starts[i + 1];
-			const bpeValue = this.getBpeRankFromBytesOrThrow(piece.subarray(pairStart, pairEnd));
-			output.push(bpeValue);
-		}
+		for (let i = 0; i < length; i = next[i]) output.push(this.getBpeRankFromBytesOrThrow(piece.subarray(i, next[i])));
 		return output;
 	}
 };

package/esm/BytePairEncodingCore.js.map

@@ -1 +1 @@
-{"version":3,"file":"BytePairEncodingCore.js","names":[],"sources":["../src/BytePairEncodingCore.ts"],"sourcesContent":["import { DEFAULT_MERGE_CACHE_SIZE } from './constants.js'\nimport { compareUint8Arrays, isAscii, tryConvertToString } from './utfUtil.js'\nimport { escapeRegExp } from './util.js'\n\nexport type RawBytePairRanks = readonly (string | readonly number[])[]\n\nexport interface BytePairEncodingConfig {\n  bytePairRankDecoder: RawBytePairRanks\n  specialTokensEncoder?: Map<string, number>\n  tokenSplitRegex: RegExp\n  /**\n   * LRU cache for merged tokens pairs.\n   * Increasing this value should make encoding similar strings faster,\n   * but will consume more memory.\n   * @default 100000\n   */\n  mergeCacheSize?: number\n}\n\nconst emptyBuffer = new Uint8Array(0)\n\nexport const decoder = new TextDecoder('utf8')\n\nexport class BytePairEncodingCore {\n  readonly mergeableBytePairRankCount: number\n  /**\n   * an array where the index is the BPE rank,\n   * and the value is the string or the array of bytes that it decodes to\n   * it may contain holes if token is unused\n   */\n  private bytePairRankDecoder: RawBytePairRanks\n  private bytePairNonUtfRankDecoder = new Map<number, Uint8Array>()\n  private bytePairNonUtfSortedEncoder: readonly [Uint8Array, number][]\n  /**\n   * a reverse map of the bytePairRankDecoder,\n   * where the key is the string and the value is the rank\n   * values that cannot be represented as a string are present in `bytePairNonUtfSortedEncoder`\n   */\n  private bytePairStringRankEncoder: Map<string, number>\n  private tokenSplitRegex: RegExp\n  private specialTokensEncoder: Map<string, number>\n  private specialTokensDecoder: Map<number, string>\n  private specialTokenPatternRegex: RegExp\n  private textEncoder = new TextEncoder()\n  private mergeCache?: Map<string, number[]>\n  private mergeCacheSize: number\n\n  constructor({\n    bytePairRankDecoder,\n    specialTokensEncoder,\n    tokenSplitRegex,\n    mergeCacheSize = DEFAULT_MERGE_CACHE_SIZE,\n  }: BytePairEncodingConfig) {\n    this.bytePairRankDecoder = bytePairRankDecoder\n    this.bytePairStringRankEncoder = new Map<string, number>()\n    this.mergeCacheSize = mergeCacheSize\n    if (mergeCacheSize > 0) {\n      this.mergeCache = new Map()\n    }\n\n    // size without array holes (which may be present in the encoder)\n    this.mergeableBytePairRankCount = Object.keys(bytePairRankDecoder).length\n    const binaryLookup: [Uint8Array, number][] = []\n    // forEach skips array holes:\n    bytePairRankDecoder.forEach((value, rank) => {\n      if (typeof value === 'string') {\n        this.bytePairStringRankEncoder.set(value, rank)\n        return\n      }\n      const byteArray = new Uint8Array(value)\n      binaryLookup.push([byteArray, rank])\n      this.bytePairNonUtfRankDecoder.set(rank, byteArray)\n    })\n    this.bytePairNonUtfSortedEncoder = binaryLookup.sort((a, b) =>\n      compareUint8Arrays(a[0], b[0]),\n    )\n    this.specialTokensEncoder =\n      specialTokensEncoder ?? new Map<string, number>()\n    this.specialTokensDecoder = specialTokensEncoder\n      ? new Map([...specialTokensEncoder].map(([key, value]) => [value, key]))\n      : new Map<number, string>()\n    this.tokenSplitRegex = tokenSplitRegex\n\n    const escapedSpecialTokens = [...this.specialTokensEncoder.keys()].map(\n      escapeRegExp,\n    )\n    const allSpecialTokensRegex = escapedSpecialTokens.join('|')\n    try {\n      this.specialTokenPatternRegex = new RegExp(allSpecialTokensRegex, 'y')\n    } catch {\n      throw new Error('Invalid regular expression pattern.')\n    }\n  }\n\n  setMergeCacheSize(newSize: number): void {\n    if (this.mergeCacheSize === 0 && newSize > 0) {\n      this.mergeCache = new Map()\n    }\n    this.mergeCacheSize = newSize\n    if (newSize === 0) {\n      this.mergeCache = undefined\n    }\n  }\n\n  clearMergeCache(): void {\n    this.mergeCache?.clear()\n  }\n\n  *encodeNativeGenerator(\n    text: string,\n    allowedSpecial?: Set<string>,\n  ): Generator<number[], number, undefined> {\n    let startIndex = 0\n    let lastTokenLength = 0\n\n    while (true) {\n      const nextSpecialMatch = this.findNextSpecialToken(\n        text,\n        allowedSpecial,\n        startIndex,\n      )\n      const nextSpecialStartIndex = nextSpecialMatch?.[0]\n\n      const endIndex = nextSpecialStartIndex ?? text.length\n\n      const textBeforeSpecial =\n        startIndex === 0 && endIndex === text.length\n          ? text\n          : text.slice(startIndex, endIndex)\n\n      for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {\n        const token = this.getBpeRankFromString(match)\n        if (token !== undefined) {\n          lastTokenLength = 1\n          yield [token]\n\n          continue\n        }\n\n        const tokens = this.bytePairEncode(match)\n        lastTokenLength = tokens.length\n        yield tokens\n      }\n\n      if (nextSpecialStartIndex !== undefined) {\n        const specialToken = nextSpecialMatch![1]\n        const specialTokenValue = this.specialTokensEncoder.get(specialToken)\n        if (specialTokenValue === undefined) {\n          throw new Error(\n            `Special token \"${specialToken}\" is not in the special token encoder.`,\n          )\n        }\n        yield [specialTokenValue]\n        startIndex = nextSpecialStartIndex + specialToken.length\n        lastTokenLength = 1\n      } else {\n        break\n      }\n    }\n\n    return lastTokenLength\n  }\n\n  encodeNative(text: string, allowedSpecial?: Set<string>): number[] {\n    let startIndex = 0\n    const tokensArray: number[] = [] // Flat list to collect the tokens\n\n    while (true) {\n      const nextSpecialMatch = this.findNextSpecialToken(\n        text,\n        allowedSpecial,\n        startIndex,\n      )\n      const nextSpecialStartIndex = nextSpecialMatch?.[0]\n\n      const endIndex = nextSpecialStartIndex ?? text.length\n\n      const textBeforeSpecial =\n        startIndex === 0 && endIndex === text.length\n          ? text\n          : text.slice(startIndex, endIndex)\n\n      for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {\n        const token = this.getBpeRankFromString(match)\n        if (token !== undefined) {\n          tokensArray.push(token)\n\n          continue\n        }\n\n        const tokens = this.bytePairEncode(match)\n        tokensArray.push(...tokens)\n      }\n\n      if (nextSpecialStartIndex !== undefined) {\n        const specialToken = nextSpecialMatch![1]\n        const specialTokenValue = this.specialTokensEncoder.get(specialToken)\n        if (specialTokenValue === undefined) {\n          throw new Error(\n            `Special token \"${specialToken}\" is not in the special token encoder.`,\n          )\n        }\n        tokensArray.push(specialTokenValue)\n        startIndex = nextSpecialStartIndex + specialToken.length\n      } else {\n        break\n      }\n    }\n\n    return tokensArray\n  }\n\n  countNative(text: string, allowedSpecial?: Set<string>): number {\n    let startIndex = 0\n    let tokensCount = 0\n\n    while (true) {\n      const nextSpecialMatch = this.findNextSpecialToken(\n        text,\n        allowedSpecial,\n        startIndex,\n      )\n      const nextSpecialStartIndex = nextSpecialMatch?.[0]\n\n      const endIndex = nextSpecialStartIndex ?? text.length\n\n      const textBeforeSpecial =\n        startIndex === 0 && endIndex === text.length\n          ? text\n          : text.slice(startIndex, endIndex)\n\n      for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {\n        const token = this.getBpeRankFromString(match)\n        if (token !== undefined) {\n          tokensCount++\n\n          continue\n        }\n\n        const tokens = this.bytePairEncode(match)\n        tokensCount += tokens.length\n      }\n\n      if (nextSpecialStartIndex !== undefined) {\n        const specialToken = nextSpecialMatch![1]\n        const specialTokenValue = this.specialTokensEncoder.get(specialToken)\n        if (specialTokenValue === undefined) {\n          throw new Error(\n            `Special token \"${specialToken}\" is not in the special token encoder.`,\n          )\n        }\n        tokensCount++\n        startIndex = nextSpecialStartIndex + specialToken.length\n      } else {\n        break\n      }\n    }\n\n    return tokensCount\n  }\n\n  *decodeNativeGenerator(\n    tokens: Iterable<number>,\n  ): Generator<Uint8Array | string, void, void> {\n    for (const token of tokens) {\n      const tokenBytes = this.tryDecodeToken(token)\n      if (tokenBytes) {\n        yield tokenBytes\n      }\n    }\n  }\n\n  decodeNative(tokens: Iterable<number>): string {\n    let decoded = ''\n    let intBuffer = emptyBuffer\n\n    for (const token of tokens) {\n      const tokenBytes = this.tryDecodeToken(token)\n      if (tokenBytes === undefined) {\n        throw new Error(`Token ${token} is not in the byte pair encoder.`)\n      }\n      if (typeof tokenBytes === 'string') {\n        if (intBuffer !== emptyBuffer) {\n          decoded += decoder.decode(intBuffer, { stream: true })\n          intBuffer = emptyBuffer\n        }\n        decoded += tokenBytes\n      } else {\n        const newBuffer = new Uint8Array(intBuffer.length + tokenBytes.length)\n        newBuffer.set(intBuffer)\n        newBuffer.set(tokenBytes, intBuffer.length)\n        intBuffer = newBuffer\n      }\n    }\n\n    if (intBuffer !== emptyBuffer) {\n      decoded += decoder.decode(intBuffer, { stream: true })\n    }\n    return decoded\n  }\n\n  async *decodeNativeAsyncIterable(\n    tokens: AsyncIterable<number>,\n  ): AsyncGenerator<Uint8Array | string> {\n    for await (const token of tokens) {\n      const tokenBytesOrString = this.tryDecodeToken(token)\n      if (tokenBytesOrString) {\n        yield tokenBytesOrString\n      }\n    }\n  }\n\n  private getBpeRankFromString(key: string): number | undefined {\n    return this.bytePairStringRankEncoder.get(key)\n  }\n\n  private getBpeRankFromStringOrThrow(key: string): number {\n    const value = this.getBpeRankFromString(key)\n    if (value === undefined) {\n      throw new Error(\n        `The byte-pair encoding does not contain a value for: ${key}`,\n      )\n    }\n    return value\n  }\n\n  private getBpeRankFromBytes(key: Uint8Array): number | undefined {\n    const keyAsString = tryConvertToString(key)\n\n    if (keyAsString !== undefined) {\n      return this.getBpeRankFromString(keyAsString)\n    }\n\n    // Perform binary search on the binary keys\n    const index = this.binarySearch(key)\n    if (index !== -1) {\n      return this.bytePairNonUtfSortedEncoder[index]![1]\n    }\n\n    return undefined\n  }\n\n  private getBpeRankFromBytesOrThrow(key: Uint8Array): number {\n    const value = this.getBpeRankFromBytes(key)\n    if (value === undefined) {\n      throw new Error(\n        `The byte-pair encoding does not contain a value for: ${key.toString()}`,\n      )\n    }\n    return value\n  }\n\n  // Binary search on the binary keys\n  private binarySearch(key: Uint8Array): number {\n    let low = 0\n    let high = this.bytePairNonUtfSortedEncoder.length - 1\n\n    while (low <= high) {\n      const mid = (low + high) >>> 1\n      const midKey = this.bytePairNonUtfSortedEncoder[mid]![0]\n      let cmp = 0\n      const maxLength = Math.min(midKey.length, key.length)\n      for (let i = 0; i < maxLength; i++) {\n        cmp = midKey[i]! - key[i]!\n        if (cmp !== 0) break\n      }\n      if (cmp === 0) {\n        cmp = midKey.length - key.length\n      }\n      if (cmp === 0) {\n        return mid\n      }\n      if (cmp < 0) {\n        low = mid + 1\n      } else {\n        high = mid - 1\n      }\n    }\n    return -1\n  }\n\n  private findNextSpecialToken(\n    text: string,\n    allowedSpecial: Set<string> | undefined,\n    startIndex: number,\n  ): [startIndex: number, token: string] | undefined {\n    let searchIndex = startIndex\n\n    while (true) {\n      this.specialTokenPatternRegex.lastIndex = searchIndex\n      const nextSpecialMatch = this.specialTokenPatternRegex.exec(text)\n\n      if (!nextSpecialMatch) {\n        return undefined\n      }\n\n      const specialToken = nextSpecialMatch[0]\n\n      if (allowedSpecial?.has(specialToken)) {\n        const specialTokenStartIndex = nextSpecialMatch.index + searchIndex\n        return [specialTokenStartIndex, specialToken]\n      }\n\n      searchIndex = nextSpecialMatch.index + searchIndex + 1\n    }\n  }\n\n  private tryDecodeToken(tokenRank: number): Uint8Array | string | undefined {\n    const value = this.bytePairRankDecoder[tokenRank]\n    if (typeof value === 'string') {\n      return value\n    }\n    if (typeof value === 'object') {\n      const fromBinary = this.bytePairNonUtfRankDecoder.get(tokenRank)\n      if (fromBinary) {\n        return fromBinary\n      }\n    }\n    return this.specialTokensDecoder.get(tokenRank)\n  }\n\n  private addToMergeCache(key: string, value: number[]): void {\n    if (!this.mergeCache) return\n\n    if (this.mergeCache.size >= this.mergeCacheSize) {\n      // Remove least recently used item (first item)\n      const firstKey = this.mergeCache.keys().next().value!\n      this.mergeCache.delete(firstKey)\n    }\n    this.mergeCache.set(key, value)\n  }\n\n  private bytePairEncode(input: string): number[] {\n    if (input.length === 1 && isAscii(input.codePointAt(0)!)) {\n      return [this.getBpeRankFromStringOrThrow(input)]\n    }\n\n    if (this.mergeCache?.has(input)) {\n      const result = this.mergeCache.get(input)!\n      // Move to end to mark as recently used\n      this.mergeCache.delete(input)\n      this.mergeCache.set(input, result)\n      return result\n    }\n\n    const inputBytes = this.textEncoder.encode(input)\n    const result = this.bytePairMerge(inputBytes)\n    this.addToMergeCache(input, result)\n    return result\n  }\n\n  private bytePairMerge(\n    // Input array of bytes to process\n    piece: Uint8Array,\n  ): number[] {\n    // 'starts' holds the start indices of each partition\n    const starts: number[] = []\n    // 'ranks' holds the BPE ranks of each partition pair\n    const ranks: number[] = []\n\n    // Helper function to get the rank of a byte pair starting at 'startIndex'\n    const getRank = (\n      startIndex: number,\n      pairStart = starts[startIndex],\n      pairEnd = starts[startIndex + 2],\n    ): number => {\n      if (pairEnd === undefined) {\n        // No valid pair exists\n        return Number.POSITIVE_INFINITY\n      }\n\n      // Extract the byte pair\n      const key = piece.subarray(pairStart, pairEnd)\n\n      // Retrieve the BPE rank of this byte pair (if it exists)\n      const rank = this.getBpeRankFromBytes(key)\n      return rank ?? Number.POSITIVE_INFINITY\n    }\n\n    // Initialize the 'starts' array with all possible start indices\n    for (let i = 0; i <= piece.length; i++) {\n      starts.push(i)\n      if (i < piece.length - 1) {\n        // Initialize the BPE values for all adjacent pairs\n        ranks.push(getRank(i, i, i + 2))\n      } else {\n        // Initialize BPE values to infinity for the last pair\n        ranks.push(Number.POSITIVE_INFINITY)\n      }\n    }\n\n    // Iteratively merge byte pairs until no more useful merges can be done\n    while (starts.length > 1) {\n      let lowestRank = Number.POSITIVE_INFINITY\n      let lowestPartitionIndex = -1\n\n      // Find the partition with the minimum rank\n      for (let i = 0; i < ranks.length - 1; i++) {\n        const rank = ranks[i]!\n        if (rank < lowestRank) {\n          lowestRank = rank\n          lowestPartitionIndex = i\n        }\n      }\n\n      // If no valid pair is left to merge, exit the loop\n      if (\n        lowestRank === Number.POSITIVE_INFINITY ||\n        lowestPartitionIndex === -1\n      ) {\n        break\n      }\n\n      // Merge the pair at 'lowestPartitionIndex' by removing the next start index\n      starts.splice(lowestPartitionIndex + 1, 1)\n      // Remove the BPE value of the merged pair\n      ranks.splice(lowestPartitionIndex, 1)\n\n      // Update the current merged pair's rank\n      ranks[lowestPartitionIndex] = getRank(lowestPartitionIndex)\n\n      // Update the rank of the previous pair, if it exists\n      if (lowestPartitionIndex > 0) {\n        ranks[lowestPartitionIndex - 1] = getRank(lowestPartitionIndex - 1)\n      }\n    }\n\n    // Create the final output by applying the transform function to each partitioned range\n    const output: number[] = []\n    for (let i = 0; i < starts.length - 1; i++) {\n      const pairStart = starts[i]\n      const pairEnd = starts[i + 1]\n      const bpeValue = this.getBpeRankFromBytesOrThrow(\n        piece.subarray(pairStart, pairEnd),\n      )\n      output.push(bpeValue)\n    }\n    return output\n  }\n}\n"],"mappings":";;;;AAmBA,MAAM,8BAAc,IAAI,WAAW,CAAC;AAEpC,MAAa,UAAU,IAAI,YAAY,MAAM;AAE7C,IAAa,uBAAb,MAAkC;CAChC;;;;;;CAMA;CACA,4CAAoC,IAAI,IAAwB;CAChE;;;;;;CAMA;CACA;CACA;CACA;CACA;CACA,cAAsB,IAAI,YAAY;CACtC;CACA;CAEA,YAAY,EACV,qBACA,sBACA,iBACA,iBAAiB,4BACQ;EACzB,KAAK,sBAAsB;EAC3B,KAAK,4CAA4B,IAAI,IAAoB;EACzD,KAAK,iBAAiB;EACtB,IAAI,iBAAiB,GACnB,KAAK,6BAAa,IAAI,IAAI;EAI5B,KAAK,6BAA6B,OAAO,KAAK,mBAAmB,CAAC,CAAC;EACnE,MAAM,eAAuC,CAAC;EAE9C,oBAAoB,SAAS,OAAO,SAAS;GAC3C,IAAI,OAAO,UAAU,UAAU;IAC7B,KAAK,0BAA0B,IAAI,OAAO,IAAI;IAC9C;GACF;GACA,MAAM,YAAY,IAAI,WAAW,KAAK;GACtC,aAAa,KAAK,CAAC,WAAW,IAAI,CAAC;GACnC,KAAK,0BAA0B,IAAI,MAAM,SAAS;EACpD,CAAC;EACD,KAAK,8BAA8B,aAAa,MAAM,GAAG,MACvD,mBAAmB,EAAE,IAAI,EAAE,EAAE,CAC/B;EACA,KAAK,uBACH,wCAAwB,IAAI,IAAoB;EAClD,KAAK,uBAAuB,uBACxB,IAAI,IAAI,CAAC,GAAG,oBAAoB,CAAC,CAAC,KAAK,CAAC,KAAK,WAAW,CAAC,OAAO,GAAG,CAAC,CAAC,oBACrE,IAAI,IAAoB;EAC5B,KAAK,kBAAkB;EAKvB,MAAM,wBAHuB,CAAC,GAAG,KAAK,qBAAqB,KAAK,CAAC,CAAC,CAAC,IACjE,YAE+C,CAAC,CAAC,KAAK,GAAG;EAC3D,IAAI;GACF,KAAK,2BAA2B,IAAI,OAAO,uBAAuB,GAAG;EACvE,QAAQ;GACN,MAAM,IAAI,MAAM,qCAAqC;EACvD;CACF;CAEA,kBAAkB,SAAuB;EACvC,IAAI,KAAK,mBAAmB,KAAK,UAAU,GACzC,KAAK,6BAAa,IAAI,IAAI;EAE5B,KAAK,iBAAiB;EACtB,IAAI,YAAY,GACd,KAAK,aAAa,KAAA;CAEtB;CAEA,kBAAwB;EACtB,KAAK,YAAY,MAAM;CACzB;CAEA,CAAC,sBACC,MACA,gBACwC;EACxC,IAAI,aAAa;EACjB,IAAI,kBAAkB;EAEtB,OAAO,MAAM;GACX,MAAM,mBAAmB,KAAK,qBAC5B,MACA,gBACA,UACF;GACA,MAAM,wBAAwB,mBAAmB;GAEjD,MAAM,WAAW,yBAAyB,KAAK;GAE/C,MAAM,oBACJ,eAAe,KAAK,aAAa,KAAK,SAClC,OACA,KAAK,MAAM,YAAY,QAAQ;GAErC,KAAK,MAAM,CAAC,UAAU,kBAAkB,SAAS,KAAK,eAAe,GAAG;IACtE,MAAM,QAAQ,KAAK,qBAAqB,KAAK;IAC7C,IAAI,UAAU,KAAA,GAAW;KACvB,kBAAkB;KAClB,MAAM,CAAC,KAAK;KAEZ;IACF;IAEA,MAAM,SAAS,KAAK,eAAe,KAAK;IACxC,kBAAkB,OAAO;IACzB,MAAM;GACR;GAEA,IAAI,0BAA0B,KAAA,GAAW;IACvC,MAAM,eAAe,iBAAkB;IACvC,MAAM,oBAAoB,KAAK,qBAAqB,IAAI,YAAY;IACpE,IAAI,sBAAsB,KAAA,GACxB,MAAM,IAAI,MACR,kBAAkB,aAAa,uCACjC;IAEF,MAAM,CAAC,iBAAiB;IACxB,aAAa,wBAAwB,aAAa;IAClD,kBAAkB;GACpB,OACE;EAEJ;EAEA,OAAO;CACT;CAEA,aAAa,MAAc,gBAAwC;EACjE,IAAI,aAAa;EACjB,MAAM,cAAwB,CAAC;EAE/B,OAAO,MAAM;GACX,MAAM,mBAAmB,KAAK,qBAC5B,MACA,gBACA,UACF;GACA,MAAM,wBAAwB,mBAAmB;GAEjD,MAAM,WAAW,yBAAyB,KAAK;GAE/C,MAAM,oBACJ,eAAe,KAAK,aAAa,KAAK,SAClC,OACA,KAAK,MAAM,YAAY,QAAQ;GAErC,KAAK,MAAM,CAAC,UAAU,kBAAkB,SAAS,KAAK,eAAe,GAAG;IACtE,MAAM,QAAQ,KAAK,qBAAqB,KAAK;IAC7C,IAAI,UAAU,KAAA,GAAW;KACvB,YAAY,KAAK,KAAK;KAEtB;IACF;IAEA,MAAM,SAAS,KAAK,eAAe,KAAK;IACxC,YAAY,KAAK,GAAG,MAAM;GAC5B;GAEA,IAAI,0BAA0B,KAAA,GAAW;IACvC,MAAM,eAAe,iBAAkB;IACvC,MAAM,oBAAoB,KAAK,qBAAqB,IAAI,YAAY;IACpE,IAAI,sBAAsB,KAAA,GACxB,MAAM,IAAI,MACR,kBAAkB,aAAa,uCACjC;IAEF,YAAY,KAAK,iBAAiB;IAClC,aAAa,wBAAwB,aAAa;GACpD,OACE;EAEJ;EAEA,OAAO;CACT;CAEA,YAAY,MAAc,gBAAsC;EAC9D,IAAI,aAAa;EACjB,IAAI,cAAc;EAElB,OAAO,MAAM;GACX,MAAM,mBAAmB,KAAK,qBAC5B,MACA,gBACA,UACF;GACA,MAAM,wBAAwB,mBAAmB;GAEjD,MAAM,WAAW,yBAAyB,KAAK;GAE/C,MAAM,oBACJ,eAAe,KAAK,aAAa,KAAK,SAClC,OACA,KAAK,MAAM,YAAY,QAAQ;GAErC,KAAK,MAAM,CAAC,UAAU,kBAAkB,SAAS,KAAK,eAAe,GAAG;IAEtE,IADc,KAAK,qBAAqB,KAChC,MAAM,KAAA,GAAW;KACvB;KAEA;IACF;IAEA,MAAM,SAAS,KAAK,eAAe,KAAK;IACxC,eAAe,OAAO;GACxB;GAEA,IAAI,0BAA0B,KAAA,GAAW;IACvC,MAAM,eAAe,iBAAkB;IAEvC,IAD0B,KAAK,qBAAqB,IAAI,YACpC,MAAM,KAAA,GACxB,MAAM,IAAI,MACR,kBAAkB,aAAa,uCACjC;IAEF;IACA,aAAa,wBAAwB,aAAa;GACpD,OACE;EAEJ;EAEA,OAAO;CACT;CAEA,CAAC,sBACC,QAC4C;EAC5C,KAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,aAAa,KAAK,eAAe,KAAK;GAC5C,IAAI,YACF,MAAM;EAEV;CACF;CAEA,aAAa,QAAkC;EAC7C,IAAI,UAAU;EACd,IAAI,YAAY;EAEhB,KAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,aAAa,KAAK,eAAe,KAAK;GAC5C,IAAI,eAAe,KAAA,GACjB,MAAM,IAAI,MAAM,SAAS,MAAM,kCAAkC;GAEnE,IAAI,OAAO,eAAe,UAAU;IAClC,IAAI,cAAc,aAAa;KAC7B,WAAW,QAAQ,OAAO,WAAW,EAAE,QAAQ,KAAK,CAAC;KACrD,YAAY;IACd;IACA,WAAW;GACb,OAAO;IACL,MAAM,YAAY,IAAI,WAAW,UAAU,SAAS,WAAW,MAAM;IACrE,UAAU,IAAI,SAAS;IACvB,UAAU,IAAI,YAAY,UAAU,MAAM;IAC1C,YAAY;GACd;EACF;EAEA,IAAI,cAAc,aAChB,WAAW,QAAQ,OAAO,WAAW,EAAE,QAAQ,KAAK,CAAC;EAEvD,OAAO;CACT;CAEA,OAAO,0BACL,QACqC;EACrC,WAAW,MAAM,SAAS,QAAQ;GAChC,MAAM,qBAAqB,KAAK,eAAe,KAAK;GACpD,IAAI,oBACF,MAAM;EAEV;CACF;CAEA,qBAA6B,KAAiC;EAC5D,OAAO,KAAK,0BAA0B,IAAI,GAAG;CAC/C;CAEA,4BAAoC,KAAqB;EACvD,MAAM,QAAQ,KAAK,qBAAqB,GAAG;EAC3C,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,MACR,wDAAwD,KAC1D;EAEF,OAAO;CACT;CAEA,oBAA4B,KAAqC;EAC/D,MAAM,cAAc,mBAAmB,GAAG;EAE1C,IAAI,gBAAgB,KAAA,GAClB,OAAO,KAAK,qBAAqB,WAAW;EAI9C,MAAM,QAAQ,KAAK,aAAa,GAAG;EACnC,IAAI,UAAU,IACZ,OAAO,KAAK,4BAA4B,MAAM,CAAE;CAIpD;CAEA,2BAAmC,KAAyB;EAC1D,MAAM,QAAQ,KAAK,oBAAoB,GAAG;EAC1C,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,MACR,wDAAwD,IAAI,SAAS,GACvE;EAEF,OAAO;CACT;CAGA,aAAqB,KAAyB;EAC5C,IAAI,MAAM;EACV,IAAI,OAAO,KAAK,4BAA4B,SAAS;EAErD,OAAO,OAAO,MAAM;GAClB,MAAM,MAAO,MAAM,SAAU;GAC7B,MAAM,SAAS,KAAK,4BAA4B,IAAI,CAAE;GACtD,IAAI,MAAM;GACV,MAAM,YAAY,KAAK,IAAI,OAAO,QAAQ,IAAI,MAAM;GACpD,KAAK,IAAI,IAAI,GAAG,IAAI,WAAW,KAAK;IAClC,MAAM,OAAO,KAAM,IAAI;IACvB,IAAI,QAAQ,GAAG;GACjB;GACA,IAAI,QAAQ,GACV,MAAM,OAAO,SAAS,IAAI;GAE5B,IAAI,QAAQ,GACV,OAAO;GAET,IAAI,MAAM,GACR,MAAM,MAAM;QAEZ,OAAO,MAAM;EAEjB;EACA,OAAO;CACT;CAEA,qBACE,MACA,gBACA,YACiD;EACjD,IAAI,cAAc;EAElB,OAAO,MAAM;GACX,KAAK,yBAAyB,YAAY;GAC1C,MAAM,mBAAmB,KAAK,yBAAyB,KAAK,IAAI;GAEhE,IAAI,CAAC,kBACH;GAGF,MAAM,eAAe,iBAAiB;GAEtC,IAAI,gBAAgB,IAAI,YAAY,GAElC,OAAO,CADwB,iBAAiB,QAAQ,aACxB,YAAY;GAG9C,cAAc,iBAAiB,QAAQ,cAAc;EACvD;CACF;CAEA,eAAuB,WAAoD;EACzE,MAAM,QAAQ,KAAK,oBAAoB;EACvC,IAAI,OAAO,UAAU,UACnB,OAAO;EAET,IAAI,OAAO,UAAU,UAAU;GAC7B,MAAM,aAAa,KAAK,0BAA0B,IAAI,SAAS;GAC/D,IAAI,YACF,OAAO;EAEX;EACA,OAAO,KAAK,qBAAqB,IAAI,SAAS;CAChD;CAEA,gBAAwB,KAAa,OAAuB;EAC1D,IAAI,CAAC,KAAK,YAAY;EAEtB,IAAI,KAAK,WAAW,QAAQ,KAAK,gBAAgB;GAE/C,MAAM,WAAW,KAAK,WAAW,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC;GAC/C,KAAK,WAAW,OAAO,QAAQ;EACjC;EACA,KAAK,WAAW,IAAI,KAAK,KAAK;CAChC;CAEA,eAAuB,OAAyB;EAC9C,IAAI,MAAM,WAAW,KAAK,QAAQ,MAAM,YAAY,CAAC,CAAE,GACrD,OAAO,CAAC,KAAK,4BAA4B,KAAK,CAAC;EAGjD,IAAI,KAAK,YAAY,IAAI,KAAK,GAAG;GAC/B,MAAM,SAAS,KAAK,WAAW,IAAI,KAAK;GAExC,KAAK,WAAW,OAAO,KAAK;GAC5B,KAAK,WAAW,IAAI,OAAO,MAAM;GACjC,OAAO;EACT;EAEA,MAAM,aAAa,KAAK,YAAY,OAAO,KAAK;EAChD,MAAM,SAAS,KAAK,cAAc,UAAU;EAC5C,KAAK,gBAAgB,OAAO,MAAM;EAClC,OAAO;CACT;CAEA,cAEE,OACU;EAEV,MAAM,SAAmB,CAAC;EAE1B,MAAM,QAAkB,CAAC;EAGzB,MAAM,WACJ,YACA,YAAY,OAAO,aACnB,UAAU,OAAO,aAAa,OACnB;GACX,IAAI,YAAY,KAAA,GAEd,OAAO,OAAO;GAIhB,MAAM,MAAM,MAAM,SAAS,WAAW,OAAO;GAI7C,OADa,KAAK,oBAAoB,GAC5B,KAAK,OAAO;EACxB;EAGA,KAAK,IAAI,IAAI,GAAG,KAAK,MAAM,QAAQ,KAAK;GACtC,OAAO,KAAK,CAAC;GACb,IAAI,IAAI,MAAM,SAAS,GAErB,MAAM,KAAK,QAAQ,GAAG,GAAG,IAAI,CAAC,CAAC;QAG/B,MAAM,KAAK,OAAO,iBAAiB;EAEvC;EAGA,OAAO,OAAO,SAAS,GAAG;GACxB,IAAI,aAAa,OAAO;GACxB,IAAI,uBAAuB;GAG3B,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,SAAS,GAAG,KAAK;IACzC,MAAM,OAAO,MAAM;IACnB,IAAI,OAAO,YAAY;KACrB,aAAa;KACb,uBAAuB;IACzB;GACF;GAGA,IACE,eAAe,OAAO,qBACtB,yBAAyB,IAEzB;GAIF,OAAO,OAAO,uBAAuB,GAAG,CAAC;GAEzC,MAAM,OAAO,sBAAsB,CAAC;GAGpC,MAAM,wBAAwB,QAAQ,oBAAoB;GAG1D,IAAI,uBAAuB,GACzB,MAAM,uBAAuB,KAAK,QAAQ,uBAAuB,CAAC;EAEtE;EAGA,MAAM,SAAmB,CAAC;EAC1B,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,SAAS,GAAG,KAAK;GAC1C,MAAM,YAAY,OAAO;GACzB,MAAM,UAAU,OAAO,IAAI;GAC3B,MAAM,WAAW,KAAK,2BACpB,MAAM,SAAS,WAAW,OAAO,CACnC;GACA,OAAO,KAAK,QAAQ;EACtB;EACA,OAAO;CACT;AACF"}
+{"version":3,"file":"BytePairEncodingCore.js","names":[],"sources":["../src/BytePairEncodingCore.ts"],"sourcesContent":["import { DEFAULT_MERGE_CACHE_SIZE } from './constants.js'\nimport {\n  compareUint8Arrays,\n  isAscii,\n  startsWithByteOrderMark,\n  tryConvertToString,\n} from './utfUtil.js'\nimport { escapeRegExp } from './util.js'\n\nexport type RawBytePairRanks = readonly (string | readonly number[])[]\n\nexport interface BytePairEncodingConfig {\n  bytePairRankDecoder: RawBytePairRanks\n  specialTokensEncoder?: Map<string, number>\n  tokenSplitRegex: RegExp\n  /**\n   * LRU cache for merged tokens pairs.\n   * Increasing this value should make encoding similar strings faster,\n   * but will consume more memory.\n   * @default 100000\n   */\n  mergeCacheSize?: number\n}\n\nconst emptyBuffer = new Uint8Array(0)\n\n// `ignoreBOM: true` keeps a leading U+FEFF in the decoded output.\nexport const decoder = new TextDecoder('utf8', { ignoreBOM: true })\n\nexport class BytePairEncodingCore {\n  readonly mergeableBytePairRankCount: number\n  /**\n   * an array where the index is the BPE rank,\n   * and the value is the string or the array of bytes that it decodes to\n   * it may contain holes if token is unused\n   */\n  private bytePairRankDecoder: RawBytePairRanks\n  private bytePairNonUtfRankDecoder = new Map<number, Uint8Array>()\n  private bytePairNonUtfSortedEncoder: readonly [Uint8Array, number][]\n  /**\n   * a reverse map of the bytePairRankDecoder,\n   * where the key is the string and the value is the rank\n   * values that cannot be represented as a string are present in `bytePairNonUtfSortedEncoder`\n   */\n  private bytePairStringRankEncoder: Map<string, number>\n  private tokenSplitRegex: RegExp\n  private specialTokensEncoder: Map<string, number>\n  private specialTokensDecoder: Map<number, string>\n  private specialTokenPatternRegex: RegExp\n  private textEncoder = new TextEncoder()\n  private mergeCache?: Map<string, number[]>\n  private mergeCacheSize: number\n\n  constructor({\n    bytePairRankDecoder,\n    specialTokensEncoder,\n    tokenSplitRegex,\n    mergeCacheSize = DEFAULT_MERGE_CACHE_SIZE,\n  }: BytePairEncodingConfig) {\n    this.bytePairRankDecoder = bytePairRankDecoder\n    this.bytePairStringRankEncoder = new Map<string, number>()\n    this.mergeCacheSize = mergeCacheSize\n    if (mergeCacheSize > 0) {\n      this.mergeCache = new Map()\n    }\n\n    // size without array holes (which may be present in the encoder)\n    this.mergeableBytePairRankCount = Object.keys(bytePairRankDecoder).length\n    const binaryLookup: [Uint8Array, number][] = []\n    // forEach skips array holes:\n    bytePairRankDecoder.forEach((value, rank) => {\n      if (typeof value === 'string') {\n        this.bytePairStringRankEncoder.set(value, rank)\n        return\n      }\n      const byteArray = new Uint8Array(value)\n      binaryLookup.push([byteArray, rank])\n      this.bytePairNonUtfRankDecoder.set(rank, byteArray)\n      if (startsWithByteOrderMark(byteArray)) {\n        // The rank tables keep tokens that start with a UTF-8 byte order mark\n        // (EF BB BF) as raw bytes, because a plain decode would strip the mark.\n        // They are still valid UTF-8, so they must also be reachable through\n        // the string lookup that getBpeRankFromBytes uses for valid UTF-8.\n        const asString = tryConvertToString(byteArray)\n        if (asString !== undefined) {\n          this.bytePairStringRankEncoder.set(asString, rank)\n        }\n      }\n    })\n    this.bytePairNonUtfSortedEncoder = binaryLookup.sort((a, b) =>\n      compareUint8Arrays(a[0], b[0]),\n    )\n    this.specialTokensEncoder =\n      specialTokensEncoder ?? new Map<string, number>()\n    this.specialTokensDecoder = specialTokensEncoder\n      ? new Map([...specialTokensEncoder].map(([key, value]) => [value, key]))\n      : new Map<number, string>()\n    this.tokenSplitRegex = tokenSplitRegex\n\n    const escapedSpecialTokens = [...this.specialTokensEncoder.keys()].map(\n      escapeRegExp,\n    )\n    const allSpecialTokensRegex = escapedSpecialTokens.join('|')\n    try {\n      this.specialTokenPatternRegex = new RegExp(allSpecialTokensRegex, 'y')\n    } catch {\n      throw new Error('Invalid regular expression pattern.')\n    }\n  }\n\n  setMergeCacheSize(newSize: number): void {\n    if (this.mergeCacheSize === 0 && newSize > 0) {\n      this.mergeCache = new Map()\n    }\n    this.mergeCacheSize = newSize\n    if (newSize === 0) {\n      this.mergeCache = undefined\n    }\n  }\n\n  clearMergeCache(): void {\n    this.mergeCache?.clear()\n  }\n\n  *encodeNativeGenerator(\n    text: string,\n    allowedSpecial?: Set<string>,\n  ): Generator<number[], number, undefined> {\n    let startIndex = 0\n    let lastTokenLength = 0\n\n    while (true) {\n      const nextSpecialMatch = this.findNextSpecialToken(\n        text,\n        allowedSpecial,\n        startIndex,\n      )\n      const nextSpecialStartIndex = nextSpecialMatch?.[0]\n\n      const endIndex = nextSpecialStartIndex ?? text.length\n\n      const textBeforeSpecial =\n        startIndex === 0 && endIndex === text.length\n          ? text\n          : text.slice(startIndex, endIndex)\n\n      for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {\n        const token = this.getBpeRankFromString(match)\n        if (token !== undefined) {\n          lastTokenLength = 1\n          yield [token]\n\n          continue\n        }\n\n        const tokens = this.bytePairEncode(match)\n        lastTokenLength = tokens.length\n        yield tokens\n      }\n\n      if (nextSpecialStartIndex !== undefined) {\n        const specialToken = nextSpecialMatch![1]\n        const specialTokenValue = this.specialTokensEncoder.get(specialToken)\n        if (specialTokenValue === undefined) {\n          throw new Error(\n            `Special token \"${specialToken}\" is not in the special token encoder.`,\n          )\n        }\n        yield [specialTokenValue]\n        startIndex = nextSpecialStartIndex + specialToken.length\n        lastTokenLength = 1\n      } else {\n        break\n      }\n    }\n\n    return lastTokenLength\n  }\n\n  encodeNative(text: string, allowedSpecial?: Set<string>): number[] {\n    let startIndex = 0\n    const tokensArray: number[] = [] // Flat list to collect the tokens\n\n    while (true) {\n      const nextSpecialMatch = this.findNextSpecialToken(\n        text,\n        allowedSpecial,\n        startIndex,\n      )\n      const nextSpecialStartIndex = nextSpecialMatch?.[0]\n\n      const endIndex = nextSpecialStartIndex ?? text.length\n\n      const textBeforeSpecial =\n        startIndex === 0 && endIndex === text.length\n          ? text\n          : text.slice(startIndex, endIndex)\n\n      for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {\n        const token = this.getBpeRankFromString(match)\n        if (token !== undefined) {\n          tokensArray.push(token)\n\n          continue\n        }\n\n        const tokens = this.bytePairEncode(match)\n        tokensArray.push(...tokens)\n      }\n\n      if (nextSpecialStartIndex !== undefined) {\n        const specialToken = nextSpecialMatch![1]\n        const specialTokenValue = this.specialTokensEncoder.get(specialToken)\n        if (specialTokenValue === undefined) {\n          throw new Error(\n            `Special token \"${specialToken}\" is not in the special token encoder.`,\n          )\n        }\n        tokensArray.push(specialTokenValue)\n        startIndex = nextSpecialStartIndex + specialToken.length\n      } else {\n        break\n      }\n    }\n\n    return tokensArray\n  }\n\n  countNative(text: string, allowedSpecial?: Set<string>): number {\n    let startIndex = 0\n    let tokensCount = 0\n\n    while (true) {\n      const nextSpecialMatch = this.findNextSpecialToken(\n        text,\n        allowedSpecial,\n        startIndex,\n      )\n      const nextSpecialStartIndex = nextSpecialMatch?.[0]\n\n      const endIndex = nextSpecialStartIndex ?? text.length\n\n      const textBeforeSpecial =\n        startIndex === 0 && endIndex === text.length\n          ? text\n          : text.slice(startIndex, endIndex)\n\n      for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {\n        const token = this.getBpeRankFromString(match)\n        if (token !== undefined) {\n          tokensCount++\n\n          continue\n        }\n\n        const tokens = this.bytePairEncode(match)\n        tokensCount += tokens.length\n      }\n\n      if (nextSpecialStartIndex !== undefined) {\n        const specialToken = nextSpecialMatch![1]\n        const specialTokenValue = this.specialTokensEncoder.get(specialToken)\n        if (specialTokenValue === undefined) {\n          throw new Error(\n            `Special token \"${specialToken}\" is not in the special token encoder.`,\n          )\n        }\n        tokensCount++\n        startIndex = nextSpecialStartIndex + specialToken.length\n      } else {\n        break\n      }\n    }\n\n    return tokensCount\n  }\n\n  *decodeNativeGenerator(\n    tokens: Iterable<number>,\n  ): Generator<Uint8Array | string, void, void> {\n    for (const token of tokens) {\n      const tokenBytes = this.tryDecodeToken(token)\n      if (tokenBytes) {\n        yield tokenBytes\n      }\n    }\n  }\n\n  decodeNative(tokens: Iterable<number>): string {\n    let decoded = ''\n    let intBuffer = emptyBuffer\n\n    for (const token of tokens) {\n      const tokenBytes = this.tryDecodeToken(token)\n      if (tokenBytes === undefined) {\n        throw new Error(`Token ${token} is not in the byte pair encoder.`)\n      }\n      if (typeof tokenBytes === 'string') {\n        if (intBuffer !== emptyBuffer) {\n          decoded += decoder.decode(intBuffer, { stream: true })\n          intBuffer = emptyBuffer\n        }\n        decoded += tokenBytes\n      } else {\n        const newBuffer = new Uint8Array(intBuffer.length + tokenBytes.length)\n        newBuffer.set(intBuffer)\n        newBuffer.set(tokenBytes, intBuffer.length)\n        intBuffer = newBuffer\n      }\n    }\n\n    if (intBuffer !== emptyBuffer) {\n      decoded += decoder.decode(intBuffer, { stream: true })\n    }\n    return decoded\n  }\n\n  async *decodeNativeAsyncIterable(\n    tokens: AsyncIterable<number>,\n  ): AsyncGenerator<Uint8Array | string> {\n    for await (const token of tokens) {\n      const tokenBytesOrString = this.tryDecodeToken(token)\n      if (tokenBytesOrString) {\n        yield tokenBytesOrString\n      }\n    }\n  }\n\n  private getBpeRankFromString(key: string): number | undefined {\n    return this.bytePairStringRankEncoder.get(key)\n  }\n\n  private getBpeRankFromStringOrThrow(key: string): number {\n    const value = this.getBpeRankFromString(key)\n    if (value === undefined) {\n      throw new Error(\n        `The byte-pair encoding does not contain a value for: ${key}`,\n      )\n    }\n    return value\n  }\n\n  private getBpeRankFromBytes(key: Uint8Array): number | undefined {\n    const keyAsString = tryConvertToString(key)\n\n    if (keyAsString !== undefined) {\n      return this.getBpeRankFromString(keyAsString)\n    }\n\n    // Perform binary search on the binary keys\n    const index = this.binarySearch(key)\n    if (index !== -1) {\n      return this.bytePairNonUtfSortedEncoder[index]![1]\n    }\n\n    return undefined\n  }\n\n  private getBpeRankFromBytesOrThrow(key: Uint8Array): number {\n    const value = this.getBpeRankFromBytes(key)\n    if (value === undefined) {\n      throw new Error(\n        `The byte-pair encoding does not contain a value for: ${key.toString()}`,\n      )\n    }\n    return value\n  }\n\n  // Binary search on the binary keys\n  private binarySearch(key: Uint8Array): number {\n    let low = 0\n    let high = this.bytePairNonUtfSortedEncoder.length - 1\n\n    while (low <= high) {\n      const mid = (low + high) >>> 1\n      const midKey = this.bytePairNonUtfSortedEncoder[mid]![0]\n      let cmp = 0\n      const maxLength = Math.min(midKey.length, key.length)\n      for (let i = 0; i < maxLength; i++) {\n        cmp = midKey[i]! - key[i]!\n        if (cmp !== 0) break\n      }\n      if (cmp === 0) {\n        cmp = midKey.length - key.length\n      }\n      if (cmp === 0) {\n        return mid\n      }\n      if (cmp < 0) {\n        low = mid + 1\n      } else {\n        high = mid - 1\n      }\n    }\n    return -1\n  }\n\n  private findNextSpecialToken(\n    text: string,\n    allowedSpecial: Set<string> | undefined,\n    startIndex: number,\n  ): [startIndex: number, token: string] | undefined {\n    let searchIndex = startIndex\n\n    while (true) {\n      this.specialTokenPatternRegex.lastIndex = searchIndex\n      const nextSpecialMatch = this.specialTokenPatternRegex.exec(text)\n\n      if (!nextSpecialMatch) {\n        return undefined\n      }\n\n      const specialToken = nextSpecialMatch[0]\n\n      if (allowedSpecial?.has(specialToken)) {\n        const specialTokenStartIndex = nextSpecialMatch.index + searchIndex\n        return [specialTokenStartIndex, specialToken]\n      }\n\n      searchIndex = nextSpecialMatch.index + searchIndex + 1\n    }\n  }\n\n  private tryDecodeToken(tokenRank: number): Uint8Array | string | undefined {\n    const value = this.bytePairRankDecoder[tokenRank]\n    if (typeof value === 'string') {\n      return value\n    }\n    if (typeof value === 'object') {\n      const fromBinary = this.bytePairNonUtfRankDecoder.get(tokenRank)\n      if (fromBinary) {\n        return fromBinary\n      }\n    }\n    return this.specialTokensDecoder.get(tokenRank)\n  }\n\n  private addToMergeCache(key: string, value: number[]): void {\n    if (!this.mergeCache) return\n\n    if (this.mergeCache.size >= this.mergeCacheSize) {\n      // Remove least recently used item (first item)\n      const firstKey = this.mergeCache.keys().next().value!\n      this.mergeCache.delete(firstKey)\n    }\n    this.mergeCache.set(key, value)\n  }\n\n  private bytePairEncode(input: string): number[] {\n    if (input.length === 1 && isAscii(input.codePointAt(0)!)) {\n      return [this.getBpeRankFromStringOrThrow(input)]\n    }\n\n    if (this.mergeCache?.has(input)) {\n      const result = this.mergeCache.get(input)!\n      // Move to end to mark as recently used\n      this.mergeCache.delete(input)\n      this.mergeCache.set(input, result)\n      return result\n    }\n\n    const inputBytes = this.textEncoder.encode(input)\n    const result = this.bytePairMerge(inputBytes)\n    this.addToMergeCache(input, result)\n    return result\n  }\n\n  private bytePairMerge(\n    // Input array of bytes to process\n    piece: Uint8Array,\n  ): number[] {\n    const { length } = piece\n    // Partitions form a linked list over byte offsets: a live partition starting at\n    // offset 'i' spans [i, next[i]). 'pairRanks[i]' is the rank of merging it with\n    // the partition after it. Merges come off a min-heap ordered by (rank, offset),\n    // which picks the same pair as scanning for the lowest rank, leftmost first,\n    // without rescanning the piece after every merge.\n    const next = new Int32Array(length + 1)\n    const previous = new Int32Array(length + 1)\n    const pairRanks = new Float64Array(length + 1)\n    const live = new Uint8Array(length + 1)\n\n    const getPairRank = (startIndex: number): number => {\n      const middle = next[startIndex]!\n      if (middle >= length) return Number.POSITIVE_INFINITY\n      const rank = this.getBpeRankFromBytes(\n        piece.subarray(startIndex, next[middle]!),\n      )\n      return rank ?? Number.POSITIVE_INFINITY\n    }\n\n    // Heap entries are packed as [rank, startIndex] pairs in a flat array\n    const heap: number[] = []\n    const isLower = (a: number, b: number) =>\n      heap[a]! < heap[b]! ||\n      (heap[a] === heap[b] && heap[a + 1]! < heap[b + 1]!)\n    const swap = (a: number, b: number) => {\n      const rank = heap[a]!\n      const startIndex = heap[a + 1]!\n      heap[a] = heap[b]!\n      heap[a + 1] = heap[b + 1]!\n      heap[b] = rank\n      heap[b + 1] = startIndex\n    }\n    const push = (rank: number, startIndex: number) => {\n      heap.push(rank, startIndex)\n      let index = heap.length - 2\n      while (index > 0) {\n        const parent = (((index >> 1) - 1) >> 1) << 1\n        if (!isLower(index, parent)) break\n        swap(index, parent)\n        index = parent\n      }\n    }\n    const pop = () => {\n      const last = heap.length - 2\n      swap(0, last)\n      const rank = heap[last]!\n      const startIndex = heap[last + 1]!\n      heap.length = last\n      let index = 0\n      while (true) {\n        const left = index * 2 + 2\n        const right = left + 2\n        let lowest = index\n        if (left < heap.length && isLower(left, lowest)) lowest = left\n        if (right < heap.length && isLower(right, lowest)) lowest = right\n        if (lowest === index) break\n        swap(index, lowest)\n        index = lowest\n      }\n      return [rank, startIndex] as const\n    }\n\n    for (let i = 0; i <= length; i++) {\n      next[i] = i + 1\n      previous[i] = i - 1\n      live[i] = 1\n    }\n    for (let i = 0; i < length - 1; i++) {\n      pairRanks[i] = getPairRank(i)\n      if (pairRanks[i] !== Number.POSITIVE_INFINITY) push(pairRanks[i]!, i)\n    }\n\n    // Iteratively merge byte pairs until no more useful merges can be done\n    while (heap.length > 0) {\n      const [rank, startIndex] = pop()\n      // Skip entries made stale by an earlier merge\n      if (!live[startIndex] || pairRanks[startIndex] !== rank) continue\n\n      // Merge the partition at 'startIndex' with the one after it\n      const merged = next[startIndex]!\n      live[merged] = 0\n      next[startIndex] = next[merged]!\n      previous[next[merged]!] = startIndex\n\n      pairRanks[startIndex] = getPairRank(startIndex)\n      if (pairRanks[startIndex] !== Number.POSITIVE_INFINITY) {\n        push(pairRanks[startIndex]!, startIndex)\n      }\n\n      const before = previous[startIndex]!\n      if (before >= 0) {\n        pairRanks[before] = getPairRank(before)\n        if (pairRanks[before] !== Number.POSITIVE_INFINITY) {\n          push(pairRanks[before]!, before)\n        }\n      }\n    }\n\n    // Create the final output by applying the transform function to each partitioned range\n    const output: number[] = []\n    for (let i = 0; i < length; i = next[i]!) {\n      output.push(this.getBpeRankFromBytesOrThrow(piece.subarray(i, next[i]!)))\n    }\n    return output\n  }\n}\n"],"mappings":";;;;AAwBA,MAAM,8BAAc,IAAI,WAAW,CAAC;AAGpC,MAAa,UAAU,IAAI,YAAY,QAAQ,EAAE,WAAW,KAAK,CAAC;AAElE,IAAa,uBAAb,MAAkC;CAChC;;;;;;CAMA;CACA,4CAAoC,IAAI,IAAwB;CAChE;;;;;;CAMA;CACA;CACA;CACA;CACA;CACA,cAAsB,IAAI,YAAY;CACtC;CACA;CAEA,YAAY,EACV,qBACA,sBACA,iBACA,iBAAiB,4BACQ;EACzB,KAAK,sBAAsB;EAC3B,KAAK,4CAA4B,IAAI,IAAoB;EACzD,KAAK,iBAAiB;EACtB,IAAI,iBAAiB,GACnB,KAAK,6BAAa,IAAI,IAAI;EAI5B,KAAK,6BAA6B,OAAO,KAAK,mBAAmB,CAAC,CAAC;EACnE,MAAM,eAAuC,CAAC;EAE9C,oBAAoB,SAAS,OAAO,SAAS;GAC3C,IAAI,OAAO,UAAU,UAAU;IAC7B,KAAK,0BAA0B,IAAI,OAAO,IAAI;IAC9C;GACF;GACA,MAAM,YAAY,IAAI,WAAW,KAAK;GACtC,aAAa,KAAK,CAAC,WAAW,IAAI,CAAC;GACnC,KAAK,0BAA0B,IAAI,MAAM,SAAS;GAClD,IAAI,wBAAwB,SAAS,GAAG;IAKtC,MAAM,WAAW,mBAAmB,SAAS;IAC7C,IAAI,aAAa,KAAA,GACf,KAAK,0BAA0B,IAAI,UAAU,IAAI;GAErD;EACF,CAAC;EACD,KAAK,8BAA8B,aAAa,MAAM,GAAG,MACvD,mBAAmB,EAAE,IAAI,EAAE,EAAE,CAC/B;EACA,KAAK,uBACH,wCAAwB,IAAI,IAAoB;EAClD,KAAK,uBAAuB,uBACxB,IAAI,IAAI,CAAC,GAAG,oBAAoB,CAAC,CAAC,KAAK,CAAC,KAAK,WAAW,CAAC,OAAO,GAAG,CAAC,CAAC,oBACrE,IAAI,IAAoB;EAC5B,KAAK,kBAAkB;EAKvB,MAAM,wBAHuB,CAAC,GAAG,KAAK,qBAAqB,KAAK,CAAC,CAAC,CAAC,IACjE,YAE+C,CAAC,CAAC,KAAK,GAAG;EAC3D,IAAI;GACF,KAAK,2BAA2B,IAAI,OAAO,uBAAuB,GAAG;EACvE,QAAQ;GACN,MAAM,IAAI,MAAM,qCAAqC;EACvD;CACF;CAEA,kBAAkB,SAAuB;EACvC,IAAI,KAAK,mBAAmB,KAAK,UAAU,GACzC,KAAK,6BAAa,IAAI,IAAI;EAE5B,KAAK,iBAAiB;EACtB,IAAI,YAAY,GACd,KAAK,aAAa,KAAA;CAEtB;CAEA,kBAAwB;EACtB,KAAK,YAAY,MAAM;CACzB;CAEA,CAAC,sBACC,MACA,gBACwC;EACxC,IAAI,aAAa;EACjB,IAAI,kBAAkB;EAEtB,OAAO,MAAM;GACX,MAAM,mBAAmB,KAAK,qBAC5B,MACA,gBACA,UACF;GACA,MAAM,wBAAwB,mBAAmB;GAEjD,MAAM,WAAW,yBAAyB,KAAK;GAE/C,MAAM,oBACJ,eAAe,KAAK,aAAa,KAAK,SAClC,OACA,KAAK,MAAM,YAAY,QAAQ;GAErC,KAAK,MAAM,CAAC,UAAU,kBAAkB,SAAS,KAAK,eAAe,GAAG;IACtE,MAAM,QAAQ,KAAK,qBAAqB,KAAK;IAC7C,IAAI,UAAU,KAAA,GAAW;KACvB,kBAAkB;KAClB,MAAM,CAAC,KAAK;KAEZ;IACF;IAEA,MAAM,SAAS,KAAK,eAAe,KAAK;IACxC,kBAAkB,OAAO;IACzB,MAAM;GACR;GAEA,IAAI,0BAA0B,KAAA,GAAW;IACvC,MAAM,eAAe,iBAAkB;IACvC,MAAM,oBAAoB,KAAK,qBAAqB,IAAI,YAAY;IACpE,IAAI,sBAAsB,KAAA,GACxB,MAAM,IAAI,MACR,kBAAkB,aAAa,uCACjC;IAEF,MAAM,CAAC,iBAAiB;IACxB,aAAa,wBAAwB,aAAa;IAClD,kBAAkB;GACpB,OACE;EAEJ;EAEA,OAAO;CACT;CAEA,aAAa,MAAc,gBAAwC;EACjE,IAAI,aAAa;EACjB,MAAM,cAAwB,CAAC;EAE/B,OAAO,MAAM;GACX,MAAM,mBAAmB,KAAK,qBAC5B,MACA,gBACA,UACF;GACA,MAAM,wBAAwB,mBAAmB;GAEjD,MAAM,WAAW,yBAAyB,KAAK;GAE/C,MAAM,oBACJ,eAAe,KAAK,aAAa,KAAK,SAClC,OACA,KAAK,MAAM,YAAY,QAAQ;GAErC,KAAK,MAAM,CAAC,UAAU,kBAAkB,SAAS,KAAK,eAAe,GAAG;IACtE,MAAM,QAAQ,KAAK,qBAAqB,KAAK;IAC7C,IAAI,UAAU,KAAA,GAAW;KACvB,YAAY,KAAK,KAAK;KAEtB;IACF;IAEA,MAAM,SAAS,KAAK,eAAe,KAAK;IACxC,YAAY,KAAK,GAAG,MAAM;GAC5B;GAEA,IAAI,0BAA0B,KAAA,GAAW;IACvC,MAAM,eAAe,iBAAkB;IACvC,MAAM,oBAAoB,KAAK,qBAAqB,IAAI,YAAY;IACpE,IAAI,sBAAsB,KAAA,GACxB,MAAM,IAAI,MACR,kBAAkB,aAAa,uCACjC;IAEF,YAAY,KAAK,iBAAiB;IAClC,aAAa,wBAAwB,aAAa;GACpD,OACE;EAEJ;EAEA,OAAO;CACT;CAEA,YAAY,MAAc,gBAAsC;EAC9D,IAAI,aAAa;EACjB,IAAI,cAAc;EAElB,OAAO,MAAM;GACX,MAAM,mBAAmB,KAAK,qBAC5B,MACA,gBACA,UACF;GACA,MAAM,wBAAwB,mBAAmB;GAEjD,MAAM,WAAW,yBAAyB,KAAK;GAE/C,MAAM,oBACJ,eAAe,KAAK,aAAa,KAAK,SAClC,OACA,KAAK,MAAM,YAAY,QAAQ;GAErC,KAAK,MAAM,CAAC,UAAU,kBAAkB,SAAS,KAAK,eAAe,GAAG;IAEtE,IADc,KAAK,qBAAqB,KAChC,MAAM,KAAA,GAAW;KACvB;KAEA;IACF;IAEA,MAAM,SAAS,KAAK,eAAe,KAAK;IACxC,eAAe,OAAO;GACxB;GAEA,IAAI,0BAA0B,KAAA,GAAW;IACvC,MAAM,eAAe,iBAAkB;IAEvC,IAD0B,KAAK,qBAAqB,IAAI,YACpC,MAAM,KAAA,GACxB,MAAM,IAAI,MACR,kBAAkB,aAAa,uCACjC;IAEF;IACA,aAAa,wBAAwB,aAAa;GACpD,OACE;EAEJ;EAEA,OAAO;CACT;CAEA,CAAC,sBACC,QAC4C;EAC5C,KAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,aAAa,KAAK,eAAe,KAAK;GAC5C,IAAI,YACF,MAAM;EAEV;CACF;CAEA,aAAa,QAAkC;EAC7C,IAAI,UAAU;EACd,IAAI,YAAY;EAEhB,KAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,aAAa,KAAK,eAAe,KAAK;GAC5C,IAAI,eAAe,KAAA,GACjB,MAAM,IAAI,MAAM,SAAS,MAAM,kCAAkC;GAEnE,IAAI,OAAO,eAAe,UAAU;IAClC,IAAI,cAAc,aAAa;KAC7B,WAAW,QAAQ,OAAO,WAAW,EAAE,QAAQ,KAAK,CAAC;KACrD,YAAY;IACd;IACA,WAAW;GACb,OAAO;IACL,MAAM,YAAY,IAAI,WAAW,UAAU,SAAS,WAAW,MAAM;IACrE,UAAU,IAAI,SAAS;IACvB,UAAU,IAAI,YAAY,UAAU,MAAM;IAC1C,YAAY;GACd;EACF;EAEA,IAAI,cAAc,aAChB,WAAW,QAAQ,OAAO,WAAW,EAAE,QAAQ,KAAK,CAAC;EAEvD,OAAO;CACT;CAEA,OAAO,0BACL,QACqC;EACrC,WAAW,MAAM,SAAS,QAAQ;GAChC,MAAM,qBAAqB,KAAK,eAAe,KAAK;GACpD,IAAI,oBACF,MAAM;EAEV;CACF;CAEA,qBAA6B,KAAiC;EAC5D,OAAO,KAAK,0BAA0B,IAAI,GAAG;CAC/C;CAEA,4BAAoC,KAAqB;EACvD,MAAM,QAAQ,KAAK,qBAAqB,GAAG;EAC3C,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,MACR,wDAAwD,KAC1D;EAEF,OAAO;CACT;CAEA,oBAA4B,KAAqC;EAC/D,MAAM,cAAc,mBAAmB,GAAG;EAE1C,IAAI,gBAAgB,KAAA,GAClB,OAAO,KAAK,qBAAqB,WAAW;EAI9C,MAAM,QAAQ,KAAK,aAAa,GAAG;EACnC,IAAI,UAAU,IACZ,OAAO,KAAK,4BAA4B,MAAM,CAAE;CAIpD;CAEA,2BAAmC,KAAyB;EAC1D,MAAM,QAAQ,KAAK,oBAAoB,GAAG;EAC1C,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,MACR,wDAAwD,IAAI,SAAS,GACvE;EAEF,OAAO;CACT;CAGA,aAAqB,KAAyB;EAC5C,IAAI,MAAM;EACV,IAAI,OAAO,KAAK,4BAA4B,SAAS;EAErD,OAAO,OAAO,MAAM;GAClB,MAAM,MAAO,MAAM,SAAU;GAC7B,MAAM,SAAS,KAAK,4BAA4B,IAAI,CAAE;GACtD,IAAI,MAAM;GACV,MAAM,YAAY,KAAK,IAAI,OAAO,QAAQ,IAAI,MAAM;GACpD,KAAK,IAAI,IAAI,GAAG,IAAI,WAAW,KAAK;IAClC,MAAM,OAAO,KAAM,IAAI;IACvB,IAAI,QAAQ,GAAG;GACjB;GACA,IAAI,QAAQ,GACV,MAAM,OAAO,SAAS,IAAI;GAE5B,IAAI,QAAQ,GACV,OAAO;GAET,IAAI,MAAM,GACR,MAAM,MAAM;QAEZ,OAAO,MAAM;EAEjB;EACA,OAAO;CACT;CAEA,qBACE,MACA,gBACA,YACiD;EACjD,IAAI,cAAc;EAElB,OAAO,MAAM;GACX,KAAK,yBAAyB,YAAY;GAC1C,MAAM,mBAAmB,KAAK,yBAAyB,KAAK,IAAI;GAEhE,IAAI,CAAC,kBACH;GAGF,MAAM,eAAe,iBAAiB;GAEtC,IAAI,gBAAgB,IAAI,YAAY,GAElC,OAAO,CADwB,iBAAiB,QAAQ,aACxB,YAAY;GAG9C,cAAc,iBAAiB,QAAQ,cAAc;EACvD;CACF;CAEA,eAAuB,WAAoD;EACzE,MAAM,QAAQ,KAAK,oBAAoB;EACvC,IAAI,OAAO,UAAU,UACnB,OAAO;EAET,IAAI,OAAO,UAAU,UAAU;GAC7B,MAAM,aAAa,KAAK,0BAA0B,IAAI,SAAS;GAC/D,IAAI,YACF,OAAO;EAEX;EACA,OAAO,KAAK,qBAAqB,IAAI,SAAS;CAChD;CAEA,gBAAwB,KAAa,OAAuB;EAC1D,IAAI,CAAC,KAAK,YAAY;EAEtB,IAAI,KAAK,WAAW,QAAQ,KAAK,gBAAgB;GAE/C,MAAM,WAAW,KAAK,WAAW,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC;GAC/C,KAAK,WAAW,OAAO,QAAQ;EACjC;EACA,KAAK,WAAW,IAAI,KAAK,KAAK;CAChC;CAEA,eAAuB,OAAyB;EAC9C,IAAI,MAAM,WAAW,KAAK,QAAQ,MAAM,YAAY,CAAC,CAAE,GACrD,OAAO,CAAC,KAAK,4BAA4B,KAAK,CAAC;EAGjD,IAAI,KAAK,YAAY,IAAI,KAAK,GAAG;GAC/B,MAAM,SAAS,KAAK,WAAW,IAAI,KAAK;GAExC,KAAK,WAAW,OAAO,KAAK;GAC5B,KAAK,WAAW,IAAI,OAAO,MAAM;GACjC,OAAO;EACT;EAEA,MAAM,aAAa,KAAK,YAAY,OAAO,KAAK;EAChD,MAAM,SAAS,KAAK,cAAc,UAAU;EAC5C,KAAK,gBAAgB,OAAO,MAAM;EAClC,OAAO;CACT;CAEA,cAEE,OACU;EACV,MAAM,EAAE,WAAW;EAMnB,MAAM,OAAO,IAAI,WAAW,SAAS,CAAC;EACtC,MAAM,WAAW,IAAI,WAAW,SAAS,CAAC;EAC1C,MAAM,YAAY,IAAI,aAAa,SAAS,CAAC;EAC7C,MAAM,OAAO,IAAI,WAAW,SAAS,CAAC;EAEtC,MAAM,eAAe,eAA+B;GAClD,MAAM,SAAS,KAAK;GACpB,IAAI,UAAU,QAAQ,OAAO,OAAO;GAIpC,OAHa,KAAK,oBAChB,MAAM,SAAS,YAAY,KAAK,OAAQ,CAEhC,KAAK,OAAO;EACxB;EAGA,MAAM,OAAiB,CAAC;EACxB,MAAM,WAAW,GAAW,MAC1B,KAAK,KAAM,KAAK,MACf,KAAK,OAAO,KAAK,MAAM,KAAK,IAAI,KAAM,KAAK,IAAI;EAClD,MAAM,QAAQ,GAAW,MAAc;GACrC,MAAM,OAAO,KAAK;GAClB,MAAM,aAAa,KAAK,IAAI;GAC5B,KAAK,KAAK,KAAK;GACf,KAAK,IAAI,KAAK,KAAK,IAAI;GACvB,KAAK,KAAK;GACV,KAAK,IAAI,KAAK;EAChB;EACA,MAAM,QAAQ,MAAc,eAAuB;GACjD,KAAK,KAAK,MAAM,UAAU;GAC1B,IAAI,QAAQ,KAAK,SAAS;GAC1B,OAAO,QAAQ,GAAG;IAChB,MAAM,UAAY,SAAS,KAAK,KAAM,KAAM;IAC5C,IAAI,CAAC,QAAQ,OAAO,MAAM,GAAG;IAC7B,KAAK,OAAO,MAAM;IAClB,QAAQ;GACV;EACF;EACA,MAAM,YAAY;GAChB,MAAM,OAAO,KAAK,SAAS;GAC3B,KAAK,GAAG,IAAI;GACZ,MAAM,OAAO,KAAK;GAClB,MAAM,aAAa,KAAK,OAAO;GAC/B,KAAK,SAAS;GACd,IAAI,QAAQ;GACZ,OAAO,MAAM;IACX,MAAM,OAAO,QAAQ,IAAI;IACzB,MAAM,QAAQ,OAAO;IACrB,IAAI,SAAS;IACb,IAAI,OAAO,KAAK,UAAU,QAAQ,MAAM,MAAM,GAAG,SAAS;IAC1D,IAAI,QAAQ,KAAK,UAAU,QAAQ,OAAO,MAAM,GAAG,SAAS;IAC5D,IAAI,WAAW,OAAO;IACtB,KAAK,OAAO,MAAM;IAClB,QAAQ;GACV;GACA,OAAO,CAAC,MAAM,UAAU;EAC1B;EAEA,KAAK,IAAI,IAAI,GAAG,KAAK,QAAQ,KAAK;GAChC,KAAK,KAAK,IAAI;GACd,SAAS,KAAK,IAAI;GAClB,KAAK,KAAK;EACZ;EACA,KAAK,IAAI,IAAI,GAAG,IAAI,SAAS,GAAG,KAAK;GACnC,UAAU,KAAK,YAAY,CAAC;GAC5B,IAAI,UAAU,OAAO,OAAO,mBAAmB,KAAK,UAAU,IAAK,CAAC;EACtE;EAGA,OAAO,KAAK,SAAS,GAAG;GACtB,MAAM,CAAC,MAAM,cAAc,IAAI;GAE/B,IAAI,CAAC,KAAK,eAAe,UAAU,gBAAgB,MAAM;GAGzD,MAAM,SAAS,KAAK;GACpB,KAAK,UAAU;GACf,KAAK,cAAc,KAAK;GACxB,SAAS,KAAK,WAAY;GAE1B,UAAU,cAAc,YAAY,UAAU;GAC9C,IAAI,UAAU,gBAAgB,OAAO,mBACnC,KAAK,UAAU,aAAc,UAAU;GAGzC,MAAM,SAAS,SAAS;GACxB,IAAI,UAAU,GAAG;IACf,UAAU,UAAU,YAAY,MAAM;IACtC,IAAI,UAAU,YAAY,OAAO,mBAC/B,KAAK,UAAU,SAAU,MAAM;GAEnC;EACF;EAGA,MAAM,SAAmB,CAAC;EAC1B,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,IAAI,KAAK,IACnC,OAAO,KAAK,KAAK,2BAA2B,MAAM,SAAS,GAAG,KAAK,EAAG,CAAC,CAAC;EAE1E,OAAO;CACT;AACF"}

package/esm/encodingParams/constants.js

@@ -1,10 +1,10 @@
 //#region src/encodingParams/constants.ts
-const R50K_TOKEN_SPLIT_REGEX = /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu;
+const R50K_TOKEN_SPLIT_REGEX = /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\p{White_Space}\p{L}\p{N}]+|\p{White_Space}+(?!\P{White_Space})|\p{White_Space}+/gu;
 const CONTRACTION_SUFFIX_PATTERN = String.raw`'(?:[sS]|[dD]|[mM]|[tT]|[lL][lL]|[vV][eE]|[rR][eE])`;
 const OPTIONAL_CONTRACTION_SUFFIX = String.raw`(?:${CONTRACTION_SUFFIX_PATTERN})?`;
-const CL100K_TOKEN_SPLIT_PATTERN = String.raw`${CONTRACTION_SUFFIX_PATTERN}|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s+$|\s*[\r\n]|\s+(?!\S)|\s`;
+const CL100K_TOKEN_SPLIT_PATTERN = String.raw`${CONTRACTION_SUFFIX_PATTERN}|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\p{White_Space}\p{L}\p{N}]+[\r\n]*|\p{White_Space}+$|\p{White_Space}*[\r\n]|\p{White_Space}+(?!\P{White_Space})|\p{White_Space}`;
 const CL100K_TOKEN_SPLIT_REGEX = new RegExp(CL100K_TOKEN_SPLIT_PATTERN, "gu");
-const O200K_TOKEN_SPLIT_PATTERN = String.raw`[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+${OPTIONAL_CONTRACTION_SUFFIX}|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*${OPTIONAL_CONTRACTION_SUFFIX}|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n/]*|\s*[\r\n]+|\s+(?!\S)|\s+`;
+const O200K_TOKEN_SPLIT_PATTERN = String.raw`[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+${OPTIONAL_CONTRACTION_SUFFIX}|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*${OPTIONAL_CONTRACTION_SUFFIX}|\p{N}{1,3}| ?[^\p{White_Space}\p{L}\p{N}]+[\r\n/]*|\p{White_Space}*[\r\n]+|\p{White_Space}+(?!\P{White_Space})|\p{White_Space}+`;
 const O200K_TOKEN_SPLIT_REGEX = new RegExp(O200K_TOKEN_SPLIT_PATTERN, "gu");
 //#endregion
 export { CL100K_TOKEN_SPLIT_REGEX, O200K_TOKEN_SPLIT_REGEX, R50K_TOKEN_SPLIT_REGEX };

package/esm/encodingParams/constants.js.map

@@ -1 +1 @@
-{"version":3,"file":"constants.js","names":[],"sources":["../../src/encodingParams/constants.ts"],"sourcesContent":["export const R50K_TOKEN_SPLIT_REGEX =\n  /'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+/gu\n\nconst CONTRACTION_SUFFIX_PATTERN = String.raw`'(?:[sS]|[dD]|[mM]|[tT]|[lL][lL]|[vV][eE]|[rR][eE])`\n\nconst OPTIONAL_CONTRACTION_SUFFIX = String.raw`(?:${CONTRACTION_SUFFIX_PATTERN})?`\n\nconst CL100K_TOKEN_SPLIT_PATTERN = String.raw`${CONTRACTION_SUFFIX_PATTERN}|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s+$|\\s*[\\r\\n]|\\s+(?!\\S)|\\s`\n\nexport const CL100K_TOKEN_SPLIT_REGEX = new RegExp(\n  CL100K_TOKEN_SPLIT_PATTERN,\n  'gu',\n)\n\nconst O200K_TOKEN_SPLIT_PATTERN = String.raw`[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+${OPTIONAL_CONTRACTION_SUFFIX}|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*${OPTIONAL_CONTRACTION_SUFFIX}|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+`\n\nexport const O200K_TOKEN_SPLIT_REGEX = new RegExp(\n  O200K_TOKEN_SPLIT_PATTERN,\n  'gu',\n)\n"],"mappings":";AAAA,MAAa,yBACX;AAEF,MAAM,6BAA6B,OAAO,GAAG;AAE7C,MAAM,8BAA8B,OAAO,GAAG,MAAM,2BAA2B;AAE/E,MAAM,6BAA6B,OAAO,GAAG,GAAG,2BAA2B;AAE3E,MAAa,2BAA2B,IAAI,OAC1C,4BACA,IACF;AAEA,MAAM,4BAA4B,OAAO,GAAG,+EAA+E,4BAA4B,+EAA+E,4BAA4B;AAElQ,MAAa,0BAA0B,IAAI,OACzC,2BACA,IACF"}
+{"version":3,"file":"constants.js","names":[],"sources":["../../src/encodingParams/constants.ts"],"sourcesContent":["// The split patterns below use `\\p{White_Space}` / `\\P{White_Space}` where\n// tiktoken's patterns use `\\s` / `\\S`: the regex engine behind tiktoken treats\n// `\\s` as the Unicode White_Space property, whereas JavaScript's `\\s` differs\n// from it on exactly two code points (U+0085 and U+FEFF).\nexport const R50K_TOKEN_SPLIT_REGEX =\n  /'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\p{White_Space}\\p{L}\\p{N}]+|\\p{White_Space}+(?!\\P{White_Space})|\\p{White_Space}+/gu\n\nconst CONTRACTION_SUFFIX_PATTERN = String.raw`'(?:[sS]|[dD]|[mM]|[tT]|[lL][lL]|[vV][eE]|[rR][eE])`\n\nconst OPTIONAL_CONTRACTION_SUFFIX = String.raw`(?:${CONTRACTION_SUFFIX_PATTERN})?`\n\nconst CL100K_TOKEN_SPLIT_PATTERN = String.raw`${CONTRACTION_SUFFIX_PATTERN}|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\p{White_Space}\\p{L}\\p{N}]+[\\r\\n]*|\\p{White_Space}+$|\\p{White_Space}*[\\r\\n]|\\p{White_Space}+(?!\\P{White_Space})|\\p{White_Space}`\n\nexport const CL100K_TOKEN_SPLIT_REGEX = new RegExp(\n  CL100K_TOKEN_SPLIT_PATTERN,\n  'gu',\n)\n\nconst O200K_TOKEN_SPLIT_PATTERN = String.raw`[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+${OPTIONAL_CONTRACTION_SUFFIX}|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*${OPTIONAL_CONTRACTION_SUFFIX}|\\p{N}{1,3}| ?[^\\p{White_Space}\\p{L}\\p{N}]+[\\r\\n/]*|\\p{White_Space}*[\\r\\n]+|\\p{White_Space}+(?!\\P{White_Space})|\\p{White_Space}+`\n\nexport const O200K_TOKEN_SPLIT_REGEX = new RegExp(\n  O200K_TOKEN_SPLIT_PATTERN,\n  'gu',\n)\n"],"mappings":";AAIA,MAAa,yBACX;AAEF,MAAM,6BAA6B,OAAO,GAAG;AAE7C,MAAM,8BAA8B,OAAO,GAAG,MAAM,2BAA2B;AAE/E,MAAM,6BAA6B,OAAO,GAAG,GAAG,2BAA2B;AAE3E,MAAa,2BAA2B,IAAI,OAC1C,4BACA,IACF;AAEA,MAAM,4BAA4B,OAAO,GAAG,+EAA+E,4BAA4B,+EAA+E,4BAA4B;AAElQ,MAAa,0BAA0B,IAAI,OACzC,2BACA,IACF"}

package/esm/utfUtil.d.ts

@@ -1,8 +1,9 @@
 //#region src/utfUtil.d.ts
 declare const isAscii: (codePoint: number) => boolean;
 declare function endsWithIncompleteUtfPairSurrogate(string: string): boolean;
+declare function startsWithByteOrderMark(bytes: Uint8Array): boolean;
 declare function tryConvertToString(arr: Uint8Array): string | undefined;
 declare function compareUint8Arrays(a: Uint8Array, b: Uint8Array): number;
 //#endregion
-export { compareUint8Arrays, endsWithIncompleteUtfPairSurrogate, isAscii, tryConvertToString };
+export { compareUint8Arrays, endsWithIncompleteUtfPairSurrogate, isAscii, startsWithByteOrderMark, tryConvertToString };
 //# sourceMappingURL=utfUtil.d.ts.map

package/esm/utfUtil.js

@@ -43,7 +43,13 @@ function isValidUTF8(bytes) {
 	}
 	return true;
 }
-const textDecoder = new TextDecoder("utf8", { fatal: false });
+function startsWithByteOrderMark(bytes) {
+	return bytes.length >= 3 && bytes[0] === 239 && bytes[1] === 187 && bytes[2] === 191;
+}
+const textDecoder = new TextDecoder("utf8", {
+	fatal: false,
+	ignoreBOM: true
+});
 function tryConvertToString(arr) {
 	if (!isValidUTF8(arr)) return;
 	return textDecoder.decode(arr);
@@ -54,6 +60,6 @@ function compareUint8Arrays(a, b) {
 	return a.length - b.length;
 }
 //#endregion
-export { compareUint8Arrays, endsWithIncompleteUtfPairSurrogate, isAscii, tryConvertToString };
+export { compareUint8Arrays, endsWithIncompleteUtfPairSurrogate, isAscii, startsWithByteOrderMark, tryConvertToString };
 
 //# sourceMappingURL=utfUtil.js.map

package/esm/utfUtil.js.map

@@ -1 +1 @@
-{"version":3,"file":"utfUtil.js","names":[],"sources":["../src/utfUtil.ts"],"sourcesContent":["export const isAscii = (codePoint: number) => codePoint <= 0x7f\n\nconst HIGH_SURROGATE_START = 55_296\nconst HIGH_SURROGATE_END = 56_319\n\nexport function endsWithIncompleteUtfPairSurrogate(string: string): boolean {\n  if (string.length === 0) return false\n  // Check if the last character is a high surrogate\n  const lastCharCode = string.charCodeAt(string.length - 1)\n  return (\n    lastCharCode >= HIGH_SURROGATE_START && lastCharCode <= HIGH_SURROGATE_END\n  )\n}\n\nfunction isValidUTF8(bytes: Uint8Array): boolean {\n  let i = 0\n  while (i < bytes.length) {\n    const byte1 = bytes[i]!\n\n    let numBytes = 0\n    let codePoint = 0\n\n    // Determine the number of bytes in the current UTF-8 character\n    if (byte1 <= 0x7f) {\n      // 1-byte character (ASCII)\n      numBytes = 1\n      codePoint = byte1\n    } else if ((byte1 & 0xe0) === 0xc0) {\n      // 2-byte character\n      numBytes = 2\n      codePoint = byte1 & 0x1f\n      if (byte1 <= 0xc1) return false // Overlong encoding not allowed\n    } else if ((byte1 & 0xf0) === 0xe0) {\n      // 3-byte character\n      numBytes = 3\n      codePoint = byte1 & 0x0f\n    } else if ((byte1 & 0xf8) === 0xf0) {\n      // 4-byte character\n      numBytes = 4\n      codePoint = byte1 & 0x07\n      if (byte1 > 0xf4) return false // Code points above U+10FFFF not allowed\n    } else {\n      // Invalid first byte of UTF-8 character\n      return false\n    }\n\n    // Ensure there are enough continuation bytes\n    if (i + numBytes > bytes.length) return false\n\n    // Process the continuation bytes\n    for (let j = 1; j < numBytes; j++) {\n      const byte = bytes[i + j]\n      if (byte === undefined || (byte & 0xc0) !== 0x80) return false // Continuation bytes must start with '10'\n      codePoint = (codePoint << 6) | (byte & 0x3f)\n    }\n\n    // Check for overlong encodings\n    if (numBytes === 2 && codePoint < 0x80) return false // Overlong 2-byte sequence\n    if (numBytes === 3 && codePoint < 2_048) return false // Overlong 3-byte sequence\n    if (numBytes === 4 && codePoint < 65_536) return false // Overlong 4-byte sequence\n\n    // Check for surrogate halves (U+D800 to U+DFFF)\n    if (codePoint >= 55_296 && codePoint <= 57_343) return false\n\n    // Check for code points above U+10FFFF\n    if (codePoint > 1_114_111) return false\n\n    // Move to the next character\n    i += numBytes\n  }\n  return true\n}\n\nconst textDecoder = new TextDecoder('utf8', { fatal: false })\n\nexport function tryConvertToString(arr: Uint8Array): string | undefined {\n  if (!isValidUTF8(arr)) {\n    return undefined\n  }\n  return textDecoder.decode(arr)\n}\n\n// Helper function to compare two Uint8Arrays lexicographically\nexport function compareUint8Arrays(a: Uint8Array, b: Uint8Array): number {\n  const len = Math.min(a.length, b.length)\n  for (let i = 0; i < len; i++) {\n    if (a[i] !== b[i]) {\n      return a[i]! - b[i]!\n    }\n  }\n  return a.length - b.length\n}\n"],"mappings":";AAAA,MAAa,WAAW,cAAsB,aAAa;AAE3D,MAAM,uBAAuB;AAC7B,MAAM,qBAAqB;AAE3B,SAAgB,mCAAmC,QAAyB;CAC1E,IAAI,OAAO,WAAW,GAAG,OAAO;CAEhC,MAAM,eAAe,OAAO,WAAW,OAAO,SAAS,CAAC;CACxD,OACE,gBAAgB,wBAAwB,gBAAgB;AAE5D;AAEA,SAAS,YAAY,OAA4B;CAC/C,IAAI,IAAI;CACR,OAAO,IAAI,MAAM,QAAQ;EACvB,MAAM,QAAQ,MAAM;EAEpB,IAAI,WAAW;EACf,IAAI,YAAY;EAGhB,IAAI,SAAS,KAAM;GAEjB,WAAW;GACX,YAAY;EACd,OAAO,KAAK,QAAQ,SAAU,KAAM;GAElC,WAAW;GACX,YAAY,QAAQ;GACpB,IAAI,SAAS,KAAM,OAAO;EAC5B,OAAO,KAAK,QAAQ,SAAU,KAAM;GAElC,WAAW;GACX,YAAY,QAAQ;EACtB,OAAO,KAAK,QAAQ,SAAU,KAAM;GAElC,WAAW;GACX,YAAY,QAAQ;GACpB,IAAI,QAAQ,KAAM,OAAO;EAC3B,OAEE,OAAO;EAIT,IAAI,IAAI,WAAW,MAAM,QAAQ,OAAO;EAGxC,KAAK,IAAI,IAAI,GAAG,IAAI,UAAU,KAAK;GACjC,MAAM,OAAO,MAAM,IAAI;GACvB,IAAI,SAAS,KAAA,MAAc,OAAO,SAAU,KAAM,OAAO;GACzD,YAAa,aAAa,IAAM,OAAO;EACzC;EAGA,IAAI,aAAa,KAAK,YAAY,KAAM,OAAO;EAC/C,IAAI,aAAa,KAAK,YAAY,MAAO,OAAO;EAChD,IAAI,aAAa,KAAK,YAAY,OAAQ,OAAO;EAGjD,IAAI,aAAa,SAAU,aAAa,OAAQ,OAAO;EAGvD,IAAI,YAAY,SAAW,OAAO;EAGlC,KAAK;CACP;CACA,OAAO;AACT;AAEA,MAAM,cAAc,IAAI,YAAY,QAAQ,EAAE,OAAO,MAAM,CAAC;AAE5D,SAAgB,mBAAmB,KAAqC;CACtE,IAAI,CAAC,YAAY,GAAG,GAClB;CAEF,OAAO,YAAY,OAAO,GAAG;AAC/B;AAGA,SAAgB,mBAAmB,GAAe,GAAuB;CACvE,MAAM,MAAM,KAAK,IAAI,EAAE,QAAQ,EAAE,MAAM;CACvC,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,KACvB,IAAI,EAAE,OAAO,EAAE,IACb,OAAO,EAAE,KAAM,EAAE;CAGrB,OAAO,EAAE,SAAS,EAAE;AACtB"}
+{"version":3,"file":"utfUtil.js","names":[],"sources":["../src/utfUtil.ts"],"sourcesContent":["export const isAscii = (codePoint: number) => codePoint <= 0x7f\n\nconst HIGH_SURROGATE_START = 55_296\nconst HIGH_SURROGATE_END = 56_319\n\nexport function endsWithIncompleteUtfPairSurrogate(string: string): boolean {\n  if (string.length === 0) return false\n  // Check if the last character is a high surrogate\n  const lastCharCode = string.charCodeAt(string.length - 1)\n  return (\n    lastCharCode >= HIGH_SURROGATE_START && lastCharCode <= HIGH_SURROGATE_END\n  )\n}\n\nfunction isValidUTF8(bytes: Uint8Array): boolean {\n  let i = 0\n  while (i < bytes.length) {\n    const byte1 = bytes[i]!\n\n    let numBytes = 0\n    let codePoint = 0\n\n    // Determine the number of bytes in the current UTF-8 character\n    if (byte1 <= 0x7f) {\n      // 1-byte character (ASCII)\n      numBytes = 1\n      codePoint = byte1\n    } else if ((byte1 & 0xe0) === 0xc0) {\n      // 2-byte character\n      numBytes = 2\n      codePoint = byte1 & 0x1f\n      if (byte1 <= 0xc1) return false // Overlong encoding not allowed\n    } else if ((byte1 & 0xf0) === 0xe0) {\n      // 3-byte character\n      numBytes = 3\n      codePoint = byte1 & 0x0f\n    } else if ((byte1 & 0xf8) === 0xf0) {\n      // 4-byte character\n      numBytes = 4\n      codePoint = byte1 & 0x07\n      if (byte1 > 0xf4) return false // Code points above U+10FFFF not allowed\n    } else {\n      // Invalid first byte of UTF-8 character\n      return false\n    }\n\n    // Ensure there are enough continuation bytes\n    if (i + numBytes > bytes.length) return false\n\n    // Process the continuation bytes\n    for (let j = 1; j < numBytes; j++) {\n      const byte = bytes[i + j]\n      if (byte === undefined || (byte & 0xc0) !== 0x80) return false // Continuation bytes must start with '10'\n      codePoint = (codePoint << 6) | (byte & 0x3f)\n    }\n\n    // Check for overlong encodings\n    if (numBytes === 2 && codePoint < 0x80) return false // Overlong 2-byte sequence\n    if (numBytes === 3 && codePoint < 2_048) return false // Overlong 3-byte sequence\n    if (numBytes === 4 && codePoint < 65_536) return false // Overlong 4-byte sequence\n\n    // Check for surrogate halves (U+D800 to U+DFFF)\n    if (codePoint >= 55_296 && codePoint <= 57_343) return false\n\n    // Check for code points above U+10FFFF\n    if (codePoint > 1_114_111) return false\n\n    // Move to the next character\n    i += numBytes\n  }\n  return true\n}\n\nexport function startsWithByteOrderMark(bytes: Uint8Array): boolean {\n  return (\n    bytes.length >= 3 &&\n    bytes[0] === 0xef &&\n    bytes[1] === 0xbb &&\n    bytes[2] === 0xbf\n  )\n}\n\n// `ignoreBOM: true` keeps a leading U+FEFF in the output instead of silently\n// stripping it, so the bytes EF BB BF map to U+FEFF rather than to an empty string.\nconst textDecoder = new TextDecoder('utf8', { fatal: false, ignoreBOM: true })\n\nexport function tryConvertToString(arr: Uint8Array): string | undefined {\n  if (!isValidUTF8(arr)) {\n    return undefined\n  }\n  return textDecoder.decode(arr)\n}\n\n// Helper function to compare two Uint8Arrays lexicographically\nexport function compareUint8Arrays(a: Uint8Array, b: Uint8Array): number {\n  const len = Math.min(a.length, b.length)\n  for (let i = 0; i < len; i++) {\n    if (a[i] !== b[i]) {\n      return a[i]! - b[i]!\n    }\n  }\n  return a.length - b.length\n}\n"],"mappings":";AAAA,MAAa,WAAW,cAAsB,aAAa;AAE3D,MAAM,uBAAuB;AAC7B,MAAM,qBAAqB;AAE3B,SAAgB,mCAAmC,QAAyB;CAC1E,IAAI,OAAO,WAAW,GAAG,OAAO;CAEhC,MAAM,eAAe,OAAO,WAAW,OAAO,SAAS,CAAC;CACxD,OACE,gBAAgB,wBAAwB,gBAAgB;AAE5D;AAEA,SAAS,YAAY,OAA4B;CAC/C,IAAI,IAAI;CACR,OAAO,IAAI,MAAM,QAAQ;EACvB,MAAM,QAAQ,MAAM;EAEpB,IAAI,WAAW;EACf,IAAI,YAAY;EAGhB,IAAI,SAAS,KAAM;GAEjB,WAAW;GACX,YAAY;EACd,OAAO,KAAK,QAAQ,SAAU,KAAM;GAElC,WAAW;GACX,YAAY,QAAQ;GACpB,IAAI,SAAS,KAAM,OAAO;EAC5B,OAAO,KAAK,QAAQ,SAAU,KAAM;GAElC,WAAW;GACX,YAAY,QAAQ;EACtB,OAAO,KAAK,QAAQ,SAAU,KAAM;GAElC,WAAW;GACX,YAAY,QAAQ;GACpB,IAAI,QAAQ,KAAM,OAAO;EAC3B,OAEE,OAAO;EAIT,IAAI,IAAI,WAAW,MAAM,QAAQ,OAAO;EAGxC,KAAK,IAAI,IAAI,GAAG,IAAI,UAAU,KAAK;GACjC,MAAM,OAAO,MAAM,IAAI;GACvB,IAAI,SAAS,KAAA,MAAc,OAAO,SAAU,KAAM,OAAO;GACzD,YAAa,aAAa,IAAM,OAAO;EACzC;EAGA,IAAI,aAAa,KAAK,YAAY,KAAM,OAAO;EAC/C,IAAI,aAAa,KAAK,YAAY,MAAO,OAAO;EAChD,IAAI,aAAa,KAAK,YAAY,OAAQ,OAAO;EAGjD,IAAI,aAAa,SAAU,aAAa,OAAQ,OAAO;EAGvD,IAAI,YAAY,SAAW,OAAO;EAGlC,KAAK;CACP;CACA,OAAO;AACT;AAEA,SAAgB,wBAAwB,OAA4B;CAClE,OACE,MAAM,UAAU,KAChB,MAAM,OAAO,OACb,MAAM,OAAO,OACb,MAAM,OAAO;AAEjB;AAIA,MAAM,cAAc,IAAI,YAAY,QAAQ;CAAE,OAAO;CAAO,WAAW;AAAK,CAAC;AAE7E,SAAgB,mBAAmB,KAAqC;CACtE,IAAI,CAAC,YAAY,GAAG,GAClB;CAEF,OAAO,YAAY,OAAO,GAAG;AAC/B;AAGA,SAAgB,mBAAmB,GAAe,GAAuB;CACvE,MAAM,MAAM,KAAK,IAAI,EAAE,QAAQ,EAAE,MAAM;CACvC,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,KACvB,IAAI,EAAE,OAAO,EAAE,IACb,OAAO,EAAE,KAAM,EAAE;CAGrB,OAAO,EAAE,SAAS,EAAE;AACtB"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gpt-tokenizer",
-  "version": "4.0.0",
+  "version": "4.0.2",
   "description": "A pure JavaScript BPE tokenizer with model metadata and pricing for GPT-5, GPT-4o, o-series, and other OpenAI models",
   "keywords": [
     "BPE",

package/src/BytePairEncodingCore.test.ts

@@ -0,0 +1,13 @@
+import { describe, expect, test } from 'vite-plus/test'
+import { encode as encodeCl100k } from './encoding/cl100k_base.js'
+import { encode as encodeO200k } from './encoding/o200k_base.js'
+
+// A long run without whitespace becomes a single piece for the merge step.
+// Rescanning the piece after every merge made this quadratic (minutes for a
+// few hundred thousand characters), so this guards the heap-based merge.
+describe('long runs without whitespace', () => {
+  test('encodes a 200,000-character run', () => {
+    expect(encodeCl100k('='.repeat(200_000))).toHaveLength(3125)
+    expect(encodeO200k('a'.repeat(200_000))).toHaveLength(25_000)
+  })
+})

package/src/BytePairEncodingCore.ts

@@ -1,5 +1,10 @@
 import { DEFAULT_MERGE_CACHE_SIZE } from './constants.js'
-import { compareUint8Arrays, isAscii, tryConvertToString } from './utfUtil.js'
+import {
+  compareUint8Arrays,
+  isAscii,
+  startsWithByteOrderMark,
+  tryConvertToString,
+} from './utfUtil.js'
 import { escapeRegExp } from './util.js'
 
 export type RawBytePairRanks = readonly (string | readonly number[])[]
@@ -19,7 +24,8 @@ export interface BytePairEncodingConfig {
 
 const emptyBuffer = new Uint8Array(0)
 
-export const decoder = new TextDecoder('utf8')
+// `ignoreBOM: true` keeps a leading U+FEFF in the decoded output.
+export const decoder = new TextDecoder('utf8', { ignoreBOM: true })
 
 export class BytePairEncodingCore {
   readonly mergeableBytePairRankCount: number
@@ -70,6 +76,16 @@ export class BytePairEncodingCore {
       const byteArray = new Uint8Array(value)
       binaryLookup.push([byteArray, rank])
       this.bytePairNonUtfRankDecoder.set(rank, byteArray)
+      if (startsWithByteOrderMark(byteArray)) {
+        // The rank tables keep tokens that start with a UTF-8 byte order mark
+        // (EF BB BF) as raw bytes, because a plain decode would strip the mark.
+        // They are still valid UTF-8, so they must also be reachable through
+        // the string lookup that getBpeRankFromBytes uses for valid UTF-8.
+        const asString = tryConvertToString(byteArray)
+        if (asString !== undefined) {
+          this.bytePairStringRankEncoder.set(asString, rank)
+        }
+      }
     })
     this.bytePairNonUtfSortedEncoder = binaryLookup.sort((a, b) =>
       compareUint8Arrays(a[0], b[0]),
@@ -453,87 +469,109 @@ export class BytePairEncodingCore {
     // Input array of bytes to process
     piece: Uint8Array,
   ): number[] {
-    // 'starts' holds the start indices of each partition
-    const starts: number[] = []
-    // 'ranks' holds the BPE ranks of each partition pair
-    const ranks: number[] = []
-
-    // Helper function to get the rank of a byte pair starting at 'startIndex'
-    const getRank = (
-      startIndex: number,
-      pairStart = starts[startIndex],
-      pairEnd = starts[startIndex + 2],
-    ): number => {
-      if (pairEnd === undefined) {
-        // No valid pair exists
-        return Number.POSITIVE_INFINITY
-      }
-
-      // Extract the byte pair
-      const key = piece.subarray(pairStart, pairEnd)
-
-      // Retrieve the BPE rank of this byte pair (if it exists)
-      const rank = this.getBpeRankFromBytes(key)
+    const { length } = piece
+    // Partitions form a linked list over byte offsets: a live partition starting at
+    // offset 'i' spans [i, next[i]). 'pairRanks[i]' is the rank of merging it with
+    // the partition after it. Merges come off a min-heap ordered by (rank, offset),
+    // which picks the same pair as scanning for the lowest rank, leftmost first,
+    // without rescanning the piece after every merge.
+    const next = new Int32Array(length + 1)
+    const previous = new Int32Array(length + 1)
+    const pairRanks = new Float64Array(length + 1)
+    const live = new Uint8Array(length + 1)
+
+    const getPairRank = (startIndex: number): number => {
+      const middle = next[startIndex]!
+      if (middle >= length) return Number.POSITIVE_INFINITY
+      const rank = this.getBpeRankFromBytes(
+        piece.subarray(startIndex, next[middle]!),
+      )
       return rank ?? Number.POSITIVE_INFINITY
     }
 
-    // Initialize the 'starts' array with all possible start indices
-    for (let i = 0; i <= piece.length; i++) {
-      starts.push(i)
-      if (i < piece.length - 1) {
-        // Initialize the BPE values for all adjacent pairs
-        ranks.push(getRank(i, i, i + 2))
-      } else {
-        // Initialize BPE values to infinity for the last pair
-        ranks.push(Number.POSITIVE_INFINITY)
-      }
+    // Heap entries are packed as [rank, startIndex] pairs in a flat array
+    const heap: number[] = []
+    const isLower = (a: number, b: number) =>
+      heap[a]! < heap[b]! ||
+      (heap[a] === heap[b] && heap[a + 1]! < heap[b + 1]!)
+    const swap = (a: number, b: number) => {
+      const rank = heap[a]!
+      const startIndex = heap[a + 1]!
+      heap[a] = heap[b]!
+      heap[a + 1] = heap[b + 1]!
+      heap[b] = rank
+      heap[b + 1] = startIndex
     }
-
-    // Iteratively merge byte pairs until no more useful merges can be done
-    while (starts.length > 1) {
-      let lowestRank = Number.POSITIVE_INFINITY
-      let lowestPartitionIndex = -1
-
-      // Find the partition with the minimum rank
-      for (let i = 0; i < ranks.length - 1; i++) {
-        const rank = ranks[i]!
-        if (rank < lowestRank) {
-          lowestRank = rank
-          lowestPartitionIndex = i
-        }
+    const push = (rank: number, startIndex: number) => {
+      heap.push(rank, startIndex)
+      let index = heap.length - 2
+      while (index > 0) {
+        const parent = (((index >> 1) - 1) >> 1) << 1
+        if (!isLower(index, parent)) break
+        swap(index, parent)
+        index = parent
       }
-
-      // If no valid pair is left to merge, exit the loop
-      if (
-        lowestRank === Number.POSITIVE_INFINITY ||
-        lowestPartitionIndex === -1
-      ) {
-        break
+    }
+    const pop = () => {
+      const last = heap.length - 2
+      swap(0, last)
+      const rank = heap[last]!
+      const startIndex = heap[last + 1]!
+      heap.length = last
+      let index = 0
+      while (true) {
+        const left = index * 2 + 2
+        const right = left + 2
+        let lowest = index
+        if (left < heap.length && isLower(left, lowest)) lowest = left
+        if (right < heap.length && isLower(right, lowest)) lowest = right
+        if (lowest === index) break
+        swap(index, lowest)
+        index = lowest
       }
+      return [rank, startIndex] as const
+    }
 
-      // Merge the pair at 'lowestPartitionIndex' by removing the next start index
-      starts.splice(lowestPartitionIndex + 1, 1)
-      // Remove the BPE value of the merged pair
-      ranks.splice(lowestPartitionIndex, 1)
+    for (let i = 0; i <= length; i++) {
+      next[i] = i + 1
+      previous[i] = i - 1
+      live[i] = 1
+    }
+    for (let i = 0; i < length - 1; i++) {
+      pairRanks[i] = getPairRank(i)
+      if (pairRanks[i] !== Number.POSITIVE_INFINITY) push(pairRanks[i]!, i)
+    }
 
-      // Update the current merged pair's rank
-      ranks[lowestPartitionIndex] = getRank(lowestPartitionIndex)
+    // Iteratively merge byte pairs until no more useful merges can be done
+    while (heap.length > 0) {
+      const [rank, startIndex] = pop()
+      // Skip entries made stale by an earlier merge
+      if (!live[startIndex] || pairRanks[startIndex] !== rank) continue
+
+      // Merge the partition at 'startIndex' with the one after it
+      const merged = next[startIndex]!
+      live[merged] = 0
+      next[startIndex] = next[merged]!
+      previous[next[merged]!] = startIndex
+
+      pairRanks[startIndex] = getPairRank(startIndex)
+      if (pairRanks[startIndex] !== Number.POSITIVE_INFINITY) {
+        push(pairRanks[startIndex]!, startIndex)
+      }
 
-      // Update the rank of the previous pair, if it exists
-      if (lowestPartitionIndex > 0) {
-        ranks[lowestPartitionIndex - 1] = getRank(lowestPartitionIndex - 1)
+      const before = previous[startIndex]!
+      if (before >= 0) {
+        pairRanks[before] = getPairRank(before)
+        if (pairRanks[before] !== Number.POSITIVE_INFINITY) {
+          push(pairRanks[before]!, before)
+        }
       }
     }
 
     // Create the final output by applying the transform function to each partitioned range
     const output: number[] = []
-    for (let i = 0; i < starts.length - 1; i++) {
-      const pairStart = starts[i]
-      const pairEnd = starts[i + 1]
-      const bpeValue = this.getBpeRankFromBytesOrThrow(
-        piece.subarray(pairStart, pairEnd),
-      )
-      output.push(bpeValue)
+    for (let i = 0; i < length; i = next[i]!) {
+      output.push(this.getBpeRankFromBytesOrThrow(piece.subarray(i, next[i]!)))
     }
     return output
   }