gpt-tokenizer 4.0.1 → 4.0.2

package/cjs/BytePairEncodingCore.js

@@ -4,7 +4,7 @@ const require_utfUtil = require("./utfUtil.js");
 const require_util = require("./util.js");
 //#region src/BytePairEncodingCore.ts
 const emptyBuffer = /* @__PURE__ */ new Uint8Array(0);
-const decoder = new TextDecoder("utf8");
+const decoder = new TextDecoder("utf8", { ignoreBOM: true });
 var BytePairEncodingCore = class {
 	mergeableBytePairRankCount;
 	/**
@@ -43,6 +43,10 @@ var BytePairEncodingCore = class {
 			const byteArray = new Uint8Array(value);
 			binaryLookup.push([byteArray, rank]);
 			this.bytePairNonUtfRankDecoder.set(rank, byteArray);
+			if (require_utfUtil.startsWithByteOrderMark(byteArray)) {
+				const asString = require_utfUtil.tryConvertToString(byteArray);
+				if (asString !== void 0) this.bytePairStringRankEncoder.set(asString, rank);
+			}
 		});
 		this.bytePairNonUtfSortedEncoder = binaryLookup.sort((a, b) => require_utfUtil.compareUint8Arrays(a[0], b[0]));
 		this.specialTokensEncoder = specialTokensEncoder ?? /* @__PURE__ */ new Map();

package/cjs/BytePairEncodingCore.js.map

@@ -1 +1 @@
-{"version":3,"file":"BytePairEncodingCore.js","names":["DEFAULT_MERGE_CACHE_SIZE","compareUint8Arrays","escapeRegExp","tryConvertToString","isAscii"],"sources":["../src/BytePairEncodingCore.ts"],"sourcesContent":["import { DEFAULT_MERGE_CACHE_SIZE } from './constants.js'\nimport { compareUint8Arrays, isAscii, tryConvertToString } from './utfUtil.js'\nimport { escapeRegExp } from './util.js'\n\nexport type RawBytePairRanks = readonly (string | readonly number[])[]\n\nexport interface BytePairEncodingConfig {\n  bytePairRankDecoder: RawBytePairRanks\n  specialTokensEncoder?: Map<string, number>\n  tokenSplitRegex: RegExp\n  /**\n   * LRU cache for merged tokens pairs.\n   * Increasing this value should make encoding similar strings faster,\n   * but will consume more memory.\n   * @default 100000\n   */\n  mergeCacheSize?: number\n}\n\nconst emptyBuffer = new Uint8Array(0)\n\nexport const decoder = new TextDecoder('utf8')\n\nexport class BytePairEncodingCore {\n  readonly mergeableBytePairRankCount: number\n  /**\n   * an array where the index is the BPE rank,\n   * and the value is the string or the array of bytes that it decodes to\n   * it may contain holes if token is unused\n   */\n  private bytePairRankDecoder: RawBytePairRanks\n  private bytePairNonUtfRankDecoder = new Map<number, Uint8Array>()\n  private bytePairNonUtfSortedEncoder: readonly [Uint8Array, number][]\n  /**\n   * a reverse map of the bytePairRankDecoder,\n   * where the key is the string and the value is the rank\n   * values that cannot be represented as a string are present in `bytePairNonUtfSortedEncoder`\n   */\n  private bytePairStringRankEncoder: Map<string, number>\n  private tokenSplitRegex: RegExp\n  private specialTokensEncoder: Map<string, number>\n  private specialTokensDecoder: Map<number, string>\n  private specialTokenPatternRegex: RegExp\n  private textEncoder = new TextEncoder()\n  private mergeCache?: Map<string, number[]>\n  private mergeCacheSize: number\n\n  constructor({\n    bytePairRankDecoder,\n    specialTokensEncoder,\n    tokenSplitRegex,\n    mergeCacheSize = DEFAULT_MERGE_CACHE_SIZE,\n  }: BytePairEncodingConfig) {\n    this.bytePairRankDecoder = bytePairRankDecoder\n    this.bytePairStringRankEncoder = new Map<string, number>()\n    this.mergeCacheSize = mergeCacheSize\n    if (mergeCacheSize > 0) {\n      this.mergeCache = new Map()\n    }\n\n    // size without array holes (which may be present in the encoder)\n    this.mergeableBytePairRankCount = Object.keys(bytePairRankDecoder).length\n    const binaryLookup: [Uint8Array, number][] = []\n    // forEach skips array holes:\n    bytePairRankDecoder.forEach((value, rank) => {\n      if (typeof value === 'string') {\n        this.bytePairStringRankEncoder.set(value, rank)\n        return\n      }\n      const byteArray = new Uint8Array(value)\n      binaryLookup.push([byteArray, rank])\n      this.bytePairNonUtfRankDecoder.set(rank, byteArray)\n    })\n    this.bytePairNonUtfSortedEncoder = binaryLookup.sort((a, b) =>\n      compareUint8Arrays(a[0], b[0]),\n    )\n    this.specialTokensEncoder =\n      specialTokensEncoder ?? new Map<string, number>()\n    this.specialTokensDecoder = specialTokensEncoder\n      ? new Map([...specialTokensEncoder].map(([key, value]) => [value, key]))\n      : new Map<number, string>()\n    this.tokenSplitRegex = tokenSplitRegex\n\n    const escapedSpecialTokens = [...this.specialTokensEncoder.keys()].map(\n      escapeRegExp,\n    )\n    const allSpecialTokensRegex = escapedSpecialTokens.join('|')\n    try {\n      this.specialTokenPatternRegex = new RegExp(allSpecialTokensRegex, 'y')\n    } catch {\n      throw new Error('Invalid regular expression pattern.')\n    }\n  }\n\n  setMergeCacheSize(newSize: number): void {\n    if (this.mergeCacheSize === 0 && newSize > 0) {\n      this.mergeCache = new Map()\n    }\n    this.mergeCacheSize = newSize\n    if (newSize === 0) {\n      this.mergeCache = undefined\n    }\n  }\n\n  clearMergeCache(): void {\n    this.mergeCache?.clear()\n  }\n\n  *encodeNativeGenerator(\n    text: string,\n    allowedSpecial?: Set<string>,\n  ): Generator<number[], number, undefined> {\n    let startIndex = 0\n    let lastTokenLength = 0\n\n    while (true) {\n      const nextSpecialMatch = this.findNextSpecialToken(\n        text,\n        allowedSpecial,\n        startIndex,\n      )\n      const nextSpecialStartIndex = nextSpecialMatch?.[0]\n\n      const endIndex = nextSpecialStartIndex ?? text.length\n\n      const textBeforeSpecial =\n        startIndex === 0 && endIndex === text.length\n          ? text\n          : text.slice(startIndex, endIndex)\n\n      for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {\n        const token = this.getBpeRankFromString(match)\n        if (token !== undefined) {\n          lastTokenLength = 1\n          yield [token]\n\n          continue\n        }\n\n        const tokens = this.bytePairEncode(match)\n        lastTokenLength = tokens.length\n        yield tokens\n      }\n\n      if (nextSpecialStartIndex !== undefined) {\n        const specialToken = nextSpecialMatch![1]\n        const specialTokenValue = this.specialTokensEncoder.get(specialToken)\n        if (specialTokenValue === undefined) {\n          throw new Error(\n            `Special token \"${specialToken}\" is not in the special token encoder.`,\n          )\n        }\n        yield [specialTokenValue]\n        startIndex = nextSpecialStartIndex + specialToken.length\n        lastTokenLength = 1\n      } else {\n        break\n      }\n    }\n\n    return lastTokenLength\n  }\n\n  encodeNative(text: string, allowedSpecial?: Set<string>): number[] {\n    let startIndex = 0\n    const tokensArray: number[] = [] // Flat list to collect the tokens\n\n    while (true) {\n      const nextSpecialMatch = this.findNextSpecialToken(\n        text,\n        allowedSpecial,\n        startIndex,\n      )\n      const nextSpecialStartIndex = nextSpecialMatch?.[0]\n\n      const endIndex = nextSpecialStartIndex ?? text.length\n\n      const textBeforeSpecial =\n        startIndex === 0 && endIndex === text.length\n          ? text\n          : text.slice(startIndex, endIndex)\n\n      for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {\n        const token = this.getBpeRankFromString(match)\n        if (token !== undefined) {\n          tokensArray.push(token)\n\n          continue\n        }\n\n        const tokens = this.bytePairEncode(match)\n        tokensArray.push(...tokens)\n      }\n\n      if (nextSpecialStartIndex !== undefined) {\n        const specialToken = nextSpecialMatch![1]\n        const specialTokenValue = this.specialTokensEncoder.get(specialToken)\n        if (specialTokenValue === undefined) {\n          throw new Error(\n            `Special token \"${specialToken}\" is not in the special token encoder.`,\n          )\n        }\n        tokensArray.push(specialTokenValue)\n        startIndex = nextSpecialStartIndex + specialToken.length\n      } else {\n        break\n      }\n    }\n\n    return tokensArray\n  }\n\n  countNative(text: string, allowedSpecial?: Set<string>): number {\n    let startIndex = 0\n    let tokensCount = 0\n\n    while (true) {\n      const nextSpecialMatch = this.findNextSpecialToken(\n        text,\n        allowedSpecial,\n        startIndex,\n      )\n      const nextSpecialStartIndex = nextSpecialMatch?.[0]\n\n      const endIndex = nextSpecialStartIndex ?? text.length\n\n      const textBeforeSpecial =\n        startIndex === 0 && endIndex === text.length\n          ? text\n          : text.slice(startIndex, endIndex)\n\n      for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {\n        const token = this.getBpeRankFromString(match)\n        if (token !== undefined) {\n          tokensCount++\n\n          continue\n        }\n\n        const tokens = this.bytePairEncode(match)\n        tokensCount += tokens.length\n      }\n\n      if (nextSpecialStartIndex !== undefined) {\n        const specialToken = nextSpecialMatch![1]\n        const specialTokenValue = this.specialTokensEncoder.get(specialToken)\n        if (specialTokenValue === undefined) {\n          throw new Error(\n            `Special token \"${specialToken}\" is not in the special token encoder.`,\n          )\n        }\n        tokensCount++\n        startIndex = nextSpecialStartIndex + specialToken.length\n      } else {\n        break\n      }\n    }\n\n    return tokensCount\n  }\n\n  *decodeNativeGenerator(\n    tokens: Iterable<number>,\n  ): Generator<Uint8Array | string, void, void> {\n    for (const token of tokens) {\n      const tokenBytes = this.tryDecodeToken(token)\n      if (tokenBytes) {\n        yield tokenBytes\n      }\n    }\n  }\n\n  decodeNative(tokens: Iterable<number>): string {\n    let decoded = ''\n    let intBuffer = emptyBuffer\n\n    for (const token of tokens) {\n      const tokenBytes = this.tryDecodeToken(token)\n      if (tokenBytes === undefined) {\n        throw new Error(`Token ${token} is not in the byte pair encoder.`)\n      }\n      if (typeof tokenBytes === 'string') {\n        if (intBuffer !== emptyBuffer) {\n          decoded += decoder.decode(intBuffer, { stream: true })\n          intBuffer = emptyBuffer\n        }\n        decoded += tokenBytes\n      } else {\n        const newBuffer = new Uint8Array(intBuffer.length + tokenBytes.length)\n        newBuffer.set(intBuffer)\n        newBuffer.set(tokenBytes, intBuffer.length)\n        intBuffer = newBuffer\n      }\n    }\n\n    if (intBuffer !== emptyBuffer) {\n      decoded += decoder.decode(intBuffer, { stream: true })\n    }\n    return decoded\n  }\n\n  async *decodeNativeAsyncIterable(\n    tokens: AsyncIterable<number>,\n  ): AsyncGenerator<Uint8Array | string> {\n    for await (const token of tokens) {\n      const tokenBytesOrString = this.tryDecodeToken(token)\n      if (tokenBytesOrString) {\n        yield tokenBytesOrString\n      }\n    }\n  }\n\n  private getBpeRankFromString(key: string): number | undefined {\n    return this.bytePairStringRankEncoder.get(key)\n  }\n\n  private getBpeRankFromStringOrThrow(key: string): number {\n    const value = this.getBpeRankFromString(key)\n    if (value === undefined) {\n      throw new Error(\n        `The byte-pair encoding does not contain a value for: ${key}`,\n      )\n    }\n    return value\n  }\n\n  private getBpeRankFromBytes(key: Uint8Array): number | undefined {\n    const keyAsString = tryConvertToString(key)\n\n    if (keyAsString !== undefined) {\n      return this.getBpeRankFromString(keyAsString)\n    }\n\n    // Perform binary search on the binary keys\n    const index = this.binarySearch(key)\n    if (index !== -1) {\n      return this.bytePairNonUtfSortedEncoder[index]![1]\n    }\n\n    return undefined\n  }\n\n  private getBpeRankFromBytesOrThrow(key: Uint8Array): number {\n    const value = this.getBpeRankFromBytes(key)\n    if (value === undefined) {\n      throw new Error(\n        `The byte-pair encoding does not contain a value for: ${key.toString()}`,\n      )\n    }\n    return value\n  }\n\n  // Binary search on the binary keys\n  private binarySearch(key: Uint8Array): number {\n    let low = 0\n    let high = this.bytePairNonUtfSortedEncoder.length - 1\n\n    while (low <= high) {\n      const mid = (low + high) >>> 1\n      const midKey = this.bytePairNonUtfSortedEncoder[mid]![0]\n      let cmp = 0\n      const maxLength = Math.min(midKey.length, key.length)\n      for (let i = 0; i < maxLength; i++) {\n        cmp = midKey[i]! - key[i]!\n        if (cmp !== 0) break\n      }\n      if (cmp === 0) {\n        cmp = midKey.length - key.length\n      }\n      if (cmp === 0) {\n        return mid\n      }\n      if (cmp < 0) {\n        low = mid + 1\n      } else {\n        high = mid - 1\n      }\n    }\n    return -1\n  }\n\n  private findNextSpecialToken(\n    text: string,\n    allowedSpecial: Set<string> | undefined,\n    startIndex: number,\n  ): [startIndex: number, token: string] | undefined {\n    let searchIndex = startIndex\n\n    while (true) {\n      this.specialTokenPatternRegex.lastIndex = searchIndex\n      const nextSpecialMatch = this.specialTokenPatternRegex.exec(text)\n\n      if (!nextSpecialMatch) {\n        return undefined\n      }\n\n      const specialToken = nextSpecialMatch[0]\n\n      if (allowedSpecial?.has(specialToken)) {\n        const specialTokenStartIndex = nextSpecialMatch.index + searchIndex\n        return [specialTokenStartIndex, specialToken]\n      }\n\n      searchIndex = nextSpecialMatch.index + searchIndex + 1\n    }\n  }\n\n  private tryDecodeToken(tokenRank: number): Uint8Array | string | undefined {\n    const value = this.bytePairRankDecoder[tokenRank]\n    if (typeof value === 'string') {\n      return value\n    }\n    if (typeof value === 'object') {\n      const fromBinary = this.bytePairNonUtfRankDecoder.get(tokenRank)\n      if (fromBinary) {\n        return fromBinary\n      }\n    }\n    return this.specialTokensDecoder.get(tokenRank)\n  }\n\n  private addToMergeCache(key: string, value: number[]): void {\n    if (!this.mergeCache) return\n\n    if (this.mergeCache.size >= this.mergeCacheSize) {\n      // Remove least recently used item (first item)\n      const firstKey = this.mergeCache.keys().next().value!\n      this.mergeCache.delete(firstKey)\n    }\n    this.mergeCache.set(key, value)\n  }\n\n  private bytePairEncode(input: string): number[] {\n    if (input.length === 1 && isAscii(input.codePointAt(0)!)) {\n      return [this.getBpeRankFromStringOrThrow(input)]\n    }\n\n    if (this.mergeCache?.has(input)) {\n      const result = this.mergeCache.get(input)!\n      // Move to end to mark as recently used\n      this.mergeCache.delete(input)\n      this.mergeCache.set(input, result)\n      return result\n    }\n\n    const inputBytes = this.textEncoder.encode(input)\n    const result = this.bytePairMerge(inputBytes)\n    this.addToMergeCache(input, result)\n    return result\n  }\n\n  private bytePairMerge(\n    // Input array of bytes to process\n    piece: Uint8Array,\n  ): number[] {\n    const { length } = piece\n    // Partitions form a linked list over byte offsets: a live partition starting at\n    // offset 'i' spans [i, next[i]). 'pairRanks[i]' is the rank of merging it with\n    // the partition after it. Merges come off a min-heap ordered by (rank, offset),\n    // which picks the same pair as scanning for the lowest rank, leftmost first,\n    // without rescanning the piece after every merge.\n    const next = new Int32Array(length + 1)\n    const previous = new Int32Array(length + 1)\n    const pairRanks = new Float64Array(length + 1)\n    const live = new Uint8Array(length + 1)\n\n    const getPairRank = (startIndex: number): number => {\n      const middle = next[startIndex]!\n      if (middle >= length) return Number.POSITIVE_INFINITY\n      const rank = this.getBpeRankFromBytes(\n        piece.subarray(startIndex, next[middle]!),\n      )\n      return rank ?? Number.POSITIVE_INFINITY\n    }\n\n    // Heap entries are packed as [rank, startIndex] pairs in a flat array\n    const heap: number[] = []\n    const isLower = (a: number, b: number) =>\n      heap[a]! < heap[b]! ||\n      (heap[a] === heap[b] && heap[a + 1]! < heap[b + 1]!)\n    const swap = (a: number, b: number) => {\n      const rank = heap[a]!\n      const startIndex = heap[a + 1]!\n      heap[a] = heap[b]!\n      heap[a + 1] = heap[b + 1]!\n      heap[b] = rank\n      heap[b + 1] = startIndex\n    }\n    const push = (rank: number, startIndex: number) => {\n      heap.push(rank, startIndex)\n      let index = heap.length - 2\n      while (index > 0) {\n        const parent = (((index >> 1) - 1) >> 1) << 1\n        if (!isLower(index, parent)) break\n        swap(index, parent)\n        index = parent\n      }\n    }\n    const pop = () => {\n      const last = heap.length - 2\n      swap(0, last)\n      const rank = heap[last]!\n      const startIndex = heap[last + 1]!\n      heap.length = last\n      let index = 0\n      while (true) {\n        const left = index * 2 + 2\n        const right = left + 2\n        let lowest = index\n        if (left < heap.length && isLower(left, lowest)) lowest = left\n        if (right < heap.length && isLower(right, lowest)) lowest = right\n        if (lowest === index) break\n        swap(index, lowest)\n        index = lowest\n      }\n      return [rank, startIndex] as const\n    }\n\n    for (let i = 0; i <= length; i++) {\n      next[i] = i + 1\n      previous[i] = i - 1\n      live[i] = 1\n    }\n    for (let i = 0; i < length - 1; i++) {\n      pairRanks[i] = getPairRank(i)\n      if (pairRanks[i] !== Number.POSITIVE_INFINITY) push(pairRanks[i]!, i)\n    }\n\n    // Iteratively merge byte pairs until no more useful merges can be done\n    while (heap.length > 0) {\n      const [rank, startIndex] = pop()\n      // Skip entries made stale by an earlier merge\n      if (!live[startIndex] || pairRanks[startIndex] !== rank) continue\n\n      // Merge the partition at 'startIndex' with the one after it\n      const merged = next[startIndex]!\n      live[merged] = 0\n      next[startIndex] = next[merged]!\n      previous[next[merged]!] = startIndex\n\n      pairRanks[startIndex] = getPairRank(startIndex)\n      if (pairRanks[startIndex] !== Number.POSITIVE_INFINITY) {\n        push(pairRanks[startIndex]!, startIndex)\n      }\n\n      const before = previous[startIndex]!\n      if (before >= 0) {\n        pairRanks[before] = getPairRank(before)\n        if (pairRanks[before] !== Number.POSITIVE_INFINITY) {\n          push(pairRanks[before]!, before)\n        }\n      }\n    }\n\n    // Create the final output by applying the transform function to each partitioned range\n    const output: number[] = []\n    for (let i = 0; i < length; i = next[i]!) {\n      output.push(this.getBpeRankFromBytesOrThrow(piece.subarray(i, next[i]!)))\n    }\n    return output\n  }\n}\n"],"mappings":";;;;;AAmBA,MAAM,8BAAc,IAAI,WAAW,CAAC;AAEpC,MAAa,UAAU,IAAI,YAAY,MAAM;AAE7C,IAAa,uBAAb,MAAkC;CAChC;;;;;;CAMA;CACA,4CAAoC,IAAI,IAAwB;CAChE;;;;;;CAMA;CACA;CACA;CACA;CACA;CACA,cAAsB,IAAI,YAAY;CACtC;CACA;CAEA,YAAY,EACV,qBACA,sBACA,iBACA,iBAAiBA,kBAAAA,4BACQ;EACzB,KAAK,sBAAsB;EAC3B,KAAK,4CAA4B,IAAI,IAAoB;EACzD,KAAK,iBAAiB;EACtB,IAAI,iBAAiB,GACnB,KAAK,6BAAa,IAAI,IAAI;EAI5B,KAAK,6BAA6B,OAAO,KAAK,mBAAmB,CAAC,CAAC;EACnE,MAAM,eAAuC,CAAC;EAE9C,oBAAoB,SAAS,OAAO,SAAS;GAC3C,IAAI,OAAO,UAAU,UAAU;IAC7B,KAAK,0BAA0B,IAAI,OAAO,IAAI;IAC9C;GACF;GACA,MAAM,YAAY,IAAI,WAAW,KAAK;GACtC,aAAa,KAAK,CAAC,WAAW,IAAI,CAAC;GACnC,KAAK,0BAA0B,IAAI,MAAM,SAAS;EACpD,CAAC;EACD,KAAK,8BAA8B,aAAa,MAAM,GAAG,MACvDC,gBAAAA,mBAAmB,EAAE,IAAI,EAAE,EAAE,CAC/B;EACA,KAAK,uBACH,wCAAwB,IAAI,IAAoB;EAClD,KAAK,uBAAuB,uBACxB,IAAI,IAAI,CAAC,GAAG,oBAAoB,CAAC,CAAC,KAAK,CAAC,KAAK,WAAW,CAAC,OAAO,GAAG,CAAC,CAAC,oBACrE,IAAI,IAAoB;EAC5B,KAAK,kBAAkB;EAKvB,MAAM,wBAHuB,CAAC,GAAG,KAAK,qBAAqB,KAAK,CAAC,CAAC,CAAC,IACjEC,aAAAA,YAE+C,CAAC,CAAC,KAAK,GAAG;EAC3D,IAAI;GACF,KAAK,2BAA2B,IAAI,OAAO,uBAAuB,GAAG;EACvE,QAAQ;GACN,MAAM,IAAI,MAAM,qCAAqC;EACvD;CACF;CAEA,kBAAkB,SAAuB;EACvC,IAAI,KAAK,mBAAmB,KAAK,UAAU,GACzC,KAAK,6BAAa,IAAI,IAAI;EAE5B,KAAK,iBAAiB;EACtB,IAAI,YAAY,GACd,KAAK,aAAa,KAAA;CAEtB;CAEA,kBAAwB;EACtB,KAAK,YAAY,MAAM;CACzB;CAEA,CAAC,sBACC,MACA,gBACwC;EACxC,IAAI,aAAa;EACjB,IAAI,kBAAkB;EAEtB,OAAO,MAAM;GACX,MAAM,mBAAmB,KAAK,qBAC5B,MACA,gBACA,UACF;GACA,MAAM,wBAAwB,mBAAmB;GAEjD,MAAM,WAAW,yBAAyB,KAAK;GAE/C,MAAM,oBACJ,eAAe,KAAK,aAAa,KAAK,SAClC,OACA,KAAK,MAAM,YAAY,QAAQ;GAErC,KAAK,MAAM,CAAC,UAAU,kBAAkB,SAAS,KAAK,eAAe,GAAG;IACtE,MAAM,QAAQ,KAAK,qBAAqB,KAAK;IAC7C,IAAI,UAAU,KAAA,GAAW;KACvB,kBAAkB;KAClB,MAAM,CAAC,KAAK;KAEZ;IACF;IAEA,MAAM,SAAS,KAAK,eAAe,KAAK;IACxC,kBAAkB,OAAO;IACzB,MAAM;GACR;GAEA,IAAI,0BAA0B,KAAA,GAAW;IACvC,MAAM,eAAe,iBAAkB;IACvC,MAAM,oBAAoB,KAAK,qBAAqB,IAAI,YAAY;IACpE,IAAI,sBAAsB,KAAA,GACxB,MAAM,IAAI,MACR,kBAAkB,aAAa,uCACjC;IAEF,MAAM,CAAC,iBAAiB;IACxB,aAAa,wBAAwB,aAAa;IAClD,kBAAkB;GACpB,OACE;EAEJ;EAEA,OAAO;CACT;CAEA,aAAa,MAAc,gBAAwC;EACjE,IAAI,aAAa;EACjB,MAAM,cAAwB,CAAC;EAE/B,OAAO,MAAM;GACX,MAAM,mBAAmB,KAAK,qBAC5B,MACA,gBACA,UACF;GACA,MAAM,wBAAwB,mBAAmB;GAEjD,MAAM,WAAW,yBAAyB,KAAK;GAE/C,MAAM,oBACJ,eAAe,KAAK,aAAa,KAAK,SAClC,OACA,KAAK,MAAM,YAAY,QAAQ;GAErC,KAAK,MAAM,CAAC,UAAU,kBAAkB,SAAS,KAAK,eAAe,GAAG;IACtE,MAAM,QAAQ,KAAK,qBAAqB,KAAK;IAC7C,IAAI,UAAU,KAAA,GAAW;KACvB,YAAY,KAAK,KAAK;KAEtB;IACF;IAEA,MAAM,SAAS,KAAK,eAAe,KAAK;IACxC,YAAY,KAAK,GAAG,MAAM;GAC5B;GAEA,IAAI,0BAA0B,KAAA,GAAW;IACvC,MAAM,eAAe,iBAAkB;IACvC,MAAM,oBAAoB,KAAK,qBAAqB,IAAI,YAAY;IACpE,IAAI,sBAAsB,KAAA,GACxB,MAAM,IAAI,MACR,kBAAkB,aAAa,uCACjC;IAEF,YAAY,KAAK,iBAAiB;IAClC,aAAa,wBAAwB,aAAa;GACpD,OACE;EAEJ;EAEA,OAAO;CACT;CAEA,YAAY,MAAc,gBAAsC;EAC9D,IAAI,aAAa;EACjB,IAAI,cAAc;EAElB,OAAO,MAAM;GACX,MAAM,mBAAmB,KAAK,qBAC5B,MACA,gBACA,UACF;GACA,MAAM,wBAAwB,mBAAmB;GAEjD,MAAM,WAAW,yBAAyB,KAAK;GAE/C,MAAM,oBACJ,eAAe,KAAK,aAAa,KAAK,SAClC,OACA,KAAK,MAAM,YAAY,QAAQ;GAErC,KAAK,MAAM,CAAC,UAAU,kBAAkB,SAAS,KAAK,eAAe,GAAG;IAEtE,IADc,KAAK,qBAAqB,KAChC,MAAM,KAAA,GAAW;KACvB;KAEA;IACF;IAEA,MAAM,SAAS,KAAK,eAAe,KAAK;IACxC,eAAe,OAAO;GACxB;GAEA,IAAI,0BAA0B,KAAA,GAAW;IACvC,MAAM,eAAe,iBAAkB;IAEvC,IAD0B,KAAK,qBAAqB,IAAI,YACpC,MAAM,KAAA,GACxB,MAAM,IAAI,MACR,kBAAkB,aAAa,uCACjC;IAEF;IACA,aAAa,wBAAwB,aAAa;GACpD,OACE;EAEJ;EAEA,OAAO;CACT;CAEA,CAAC,sBACC,QAC4C;EAC5C,KAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,aAAa,KAAK,eAAe,KAAK;GAC5C,IAAI,YACF,MAAM;EAEV;CACF;CAEA,aAAa,QAAkC;EAC7C,IAAI,UAAU;EACd,IAAI,YAAY;EAEhB,KAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,aAAa,KAAK,eAAe,KAAK;GAC5C,IAAI,eAAe,KAAA,GACjB,MAAM,IAAI,MAAM,SAAS,MAAM,kCAAkC;GAEnE,IAAI,OAAO,eAAe,UAAU;IAClC,IAAI,cAAc,aAAa;KAC7B,WAAW,QAAQ,OAAO,WAAW,EAAE,QAAQ,KAAK,CAAC;KACrD,YAAY;IACd;IACA,WAAW;GACb,OAAO;IACL,MAAM,YAAY,IAAI,WAAW,UAAU,SAAS,WAAW,MAAM;IACrE,UAAU,IAAI,SAAS;IACvB,UAAU,IAAI,YAAY,UAAU,MAAM;IAC1C,YAAY;GACd;EACF;EAEA,IAAI,cAAc,aAChB,WAAW,QAAQ,OAAO,WAAW,EAAE,QAAQ,KAAK,CAAC;EAEvD,OAAO;CACT;CAEA,OAAO,0BACL,QACqC;EACrC,WAAW,MAAM,SAAS,QAAQ;GAChC,MAAM,qBAAqB,KAAK,eAAe,KAAK;GACpD,IAAI,oBACF,MAAM;EAEV;CACF;CAEA,qBAA6B,KAAiC;EAC5D,OAAO,KAAK,0BAA0B,IAAI,GAAG;CAC/C;CAEA,4BAAoC,KAAqB;EACvD,MAAM,QAAQ,KAAK,qBAAqB,GAAG;EAC3C,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,MACR,wDAAwD,KAC1D;EAEF,OAAO;CACT;CAEA,oBAA4B,KAAqC;EAC/D,MAAM,cAAcC,gBAAAA,mBAAmB,GAAG;EAE1C,IAAI,gBAAgB,KAAA,GAClB,OAAO,KAAK,qBAAqB,WAAW;EAI9C,MAAM,QAAQ,KAAK,aAAa,GAAG;EACnC,IAAI,UAAU,IACZ,OAAO,KAAK,4BAA4B,MAAM,CAAE;CAIpD;CAEA,2BAAmC,KAAyB;EAC1D,MAAM,QAAQ,KAAK,oBAAoB,GAAG;EAC1C,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,MACR,wDAAwD,IAAI,SAAS,GACvE;EAEF,OAAO;CACT;CAGA,aAAqB,KAAyB;EAC5C,IAAI,MAAM;EACV,IAAI,OAAO,KAAK,4BAA4B,SAAS;EAErD,OAAO,OAAO,MAAM;GAClB,MAAM,MAAO,MAAM,SAAU;GAC7B,MAAM,SAAS,KAAK,4BAA4B,IAAI,CAAE;GACtD,IAAI,MAAM;GACV,MAAM,YAAY,KAAK,IAAI,OAAO,QAAQ,IAAI,MAAM;GACpD,KAAK,IAAI,IAAI,GAAG,IAAI,WAAW,KAAK;IAClC,MAAM,OAAO,KAAM,IAAI;IACvB,IAAI,QAAQ,GAAG;GACjB;GACA,IAAI,QAAQ,GACV,MAAM,OAAO,SAAS,IAAI;GAE5B,IAAI,QAAQ,GACV,OAAO;GAET,IAAI,MAAM,GACR,MAAM,MAAM;QAEZ,OAAO,MAAM;EAEjB;EACA,OAAO;CACT;CAEA,qBACE,MACA,gBACA,YACiD;EACjD,IAAI,cAAc;EAElB,OAAO,MAAM;GACX,KAAK,yBAAyB,YAAY;GAC1C,MAAM,mBAAmB,KAAK,yBAAyB,KAAK,IAAI;GAEhE,IAAI,CAAC,kBACH;GAGF,MAAM,eAAe,iBAAiB;GAEtC,IAAI,gBAAgB,IAAI,YAAY,GAElC,OAAO,CADwB,iBAAiB,QAAQ,aACxB,YAAY;GAG9C,cAAc,iBAAiB,QAAQ,cAAc;EACvD;CACF;CAEA,eAAuB,WAAoD;EACzE,MAAM,QAAQ,KAAK,oBAAoB;EACvC,IAAI,OAAO,UAAU,UACnB,OAAO;EAET,IAAI,OAAO,UAAU,UAAU;GAC7B,MAAM,aAAa,KAAK,0BAA0B,IAAI,SAAS;GAC/D,IAAI,YACF,OAAO;EAEX;EACA,OAAO,KAAK,qBAAqB,IAAI,SAAS;CAChD;CAEA,gBAAwB,KAAa,OAAuB;EAC1D,IAAI,CAAC,KAAK,YAAY;EAEtB,IAAI,KAAK,WAAW,QAAQ,KAAK,gBAAgB;GAE/C,MAAM,WAAW,KAAK,WAAW,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC;GAC/C,KAAK,WAAW,OAAO,QAAQ;EACjC;EACA,KAAK,WAAW,IAAI,KAAK,KAAK;CAChC;CAEA,eAAuB,OAAyB;EAC9C,IAAI,MAAM,WAAW,KAAKC,gBAAAA,QAAQ,MAAM,YAAY,CAAC,CAAE,GACrD,OAAO,CAAC,KAAK,4BAA4B,KAAK,CAAC;EAGjD,IAAI,KAAK,YAAY,IAAI,KAAK,GAAG;GAC/B,MAAM,SAAS,KAAK,WAAW,IAAI,KAAK;GAExC,KAAK,WAAW,OAAO,KAAK;GAC5B,KAAK,WAAW,IAAI,OAAO,MAAM;GACjC,OAAO;EACT;EAEA,MAAM,aAAa,KAAK,YAAY,OAAO,KAAK;EAChD,MAAM,SAAS,KAAK,cAAc,UAAU;EAC5C,KAAK,gBAAgB,OAAO,MAAM;EAClC,OAAO;CACT;CAEA,cAEE,OACU;EACV,MAAM,EAAE,WAAW;EAMnB,MAAM,OAAO,IAAI,WAAW,SAAS,CAAC;EACtC,MAAM,WAAW,IAAI,WAAW,SAAS,CAAC;EAC1C,MAAM,YAAY,IAAI,aAAa,SAAS,CAAC;EAC7C,MAAM,OAAO,IAAI,WAAW,SAAS,CAAC;EAEtC,MAAM,eAAe,eAA+B;GAClD,MAAM,SAAS,KAAK;GACpB,IAAI,UAAU,QAAQ,OAAO,OAAO;GAIpC,OAHa,KAAK,oBAChB,MAAM,SAAS,YAAY,KAAK,OAAQ,CAEhC,KAAK,OAAO;EACxB;EAGA,MAAM,OAAiB,CAAC;EACxB,MAAM,WAAW,GAAW,MAC1B,KAAK,KAAM,KAAK,MACf,KAAK,OAAO,KAAK,MAAM,KAAK,IAAI,KAAM,KAAK,IAAI;EAClD,MAAM,QAAQ,GAAW,MAAc;GACrC,MAAM,OAAO,KAAK;GAClB,MAAM,aAAa,KAAK,IAAI;GAC5B,KAAK,KAAK,KAAK;GACf,KAAK,IAAI,KAAK,KAAK,IAAI;GACvB,KAAK,KAAK;GACV,KAAK,IAAI,KAAK;EAChB;EACA,MAAM,QAAQ,MAAc,eAAuB;GACjD,KAAK,KAAK,MAAM,UAAU;GAC1B,IAAI,QAAQ,KAAK,SAAS;GAC1B,OAAO,QAAQ,GAAG;IAChB,MAAM,UAAY,SAAS,KAAK,KAAM,KAAM;IAC5C,IAAI,CAAC,QAAQ,OAAO,MAAM,GAAG;IAC7B,KAAK,OAAO,MAAM;IAClB,QAAQ;GACV;EACF;EACA,MAAM,YAAY;GAChB,MAAM,OAAO,KAAK,SAAS;GAC3B,KAAK,GAAG,IAAI;GACZ,MAAM,OAAO,KAAK;GAClB,MAAM,aAAa,KAAK,OAAO;GAC/B,KAAK,SAAS;GACd,IAAI,QAAQ;GACZ,OAAO,MAAM;IACX,MAAM,OAAO,QAAQ,IAAI;IACzB,MAAM,QAAQ,OAAO;IACrB,IAAI,SAAS;IACb,IAAI,OAAO,KAAK,UAAU,QAAQ,MAAM,MAAM,GAAG,SAAS;IAC1D,IAAI,QAAQ,KAAK,UAAU,QAAQ,OAAO,MAAM,GAAG,SAAS;IAC5D,IAAI,WAAW,OAAO;IACtB,KAAK,OAAO,MAAM;IAClB,QAAQ;GACV;GACA,OAAO,CAAC,MAAM,UAAU;EAC1B;EAEA,KAAK,IAAI,IAAI,GAAG,KAAK,QAAQ,KAAK;GAChC,KAAK,KAAK,IAAI;GACd,SAAS,KAAK,IAAI;GAClB,KAAK,KAAK;EACZ;EACA,KAAK,IAAI,IAAI,GAAG,IAAI,SAAS,GAAG,KAAK;GACnC,UAAU,KAAK,YAAY,CAAC;GAC5B,IAAI,UAAU,OAAO,OAAO,mBAAmB,KAAK,UAAU,IAAK,CAAC;EACtE;EAGA,OAAO,KAAK,SAAS,GAAG;GACtB,MAAM,CAAC,MAAM,cAAc,IAAI;GAE/B,IAAI,CAAC,KAAK,eAAe,UAAU,gBAAgB,MAAM;GAGzD,MAAM,SAAS,KAAK;GACpB,KAAK,UAAU;GACf,KAAK,cAAc,KAAK;GACxB,SAAS,KAAK,WAAY;GAE1B,UAAU,cAAc,YAAY,UAAU;GAC9C,IAAI,UAAU,gBAAgB,OAAO,mBACnC,KAAK,UAAU,aAAc,UAAU;GAGzC,MAAM,SAAS,SAAS;GACxB,IAAI,UAAU,GAAG;IACf,UAAU,UAAU,YAAY,MAAM;IACtC,IAAI,UAAU,YAAY,OAAO,mBAC/B,KAAK,UAAU,SAAU,MAAM;GAEnC;EACF;EAGA,MAAM,SAAmB,CAAC;EAC1B,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,IAAI,KAAK,IACnC,OAAO,KAAK,KAAK,2BAA2B,MAAM,SAAS,GAAG,KAAK,EAAG,CAAC,CAAC;EAE1E,OAAO;CACT;AACF"}
+{"version":3,"file":"BytePairEncodingCore.js","names":["DEFAULT_MERGE_CACHE_SIZE","startsWithByteOrderMark","tryConvertToString","compareUint8Arrays","escapeRegExp","isAscii"],"sources":["../src/BytePairEncodingCore.ts"],"sourcesContent":["import { DEFAULT_MERGE_CACHE_SIZE } from './constants.js'\nimport {\n  compareUint8Arrays,\n  isAscii,\n  startsWithByteOrderMark,\n  tryConvertToString,\n} from './utfUtil.js'\nimport { escapeRegExp } from './util.js'\n\nexport type RawBytePairRanks = readonly (string | readonly number[])[]\n\nexport interface BytePairEncodingConfig {\n  bytePairRankDecoder: RawBytePairRanks\n  specialTokensEncoder?: Map<string, number>\n  tokenSplitRegex: RegExp\n  /**\n   * LRU cache for merged tokens pairs.\n   * Increasing this value should make encoding similar strings faster,\n   * but will consume more memory.\n   * @default 100000\n   */\n  mergeCacheSize?: number\n}\n\nconst emptyBuffer = new Uint8Array(0)\n\n// `ignoreBOM: true` keeps a leading U+FEFF in the decoded output.\nexport const decoder = new TextDecoder('utf8', { ignoreBOM: true })\n\nexport class BytePairEncodingCore {\n  readonly mergeableBytePairRankCount: number\n  /**\n   * an array where the index is the BPE rank,\n   * and the value is the string or the array of bytes that it decodes to\n   * it may contain holes if token is unused\n   */\n  private bytePairRankDecoder: RawBytePairRanks\n  private bytePairNonUtfRankDecoder = new Map<number, Uint8Array>()\n  private bytePairNonUtfSortedEncoder: readonly [Uint8Array, number][]\n  /**\n   * a reverse map of the bytePairRankDecoder,\n   * where the key is the string and the value is the rank\n   * values that cannot be represented as a string are present in `bytePairNonUtfSortedEncoder`\n   */\n  private bytePairStringRankEncoder: Map<string, number>\n  private tokenSplitRegex: RegExp\n  private specialTokensEncoder: Map<string, number>\n  private specialTokensDecoder: Map<number, string>\n  private specialTokenPatternRegex: RegExp\n  private textEncoder = new TextEncoder()\n  private mergeCache?: Map<string, number[]>\n  private mergeCacheSize: number\n\n  constructor({\n    bytePairRankDecoder,\n    specialTokensEncoder,\n    tokenSplitRegex,\n    mergeCacheSize = DEFAULT_MERGE_CACHE_SIZE,\n  }: BytePairEncodingConfig) {\n    this.bytePairRankDecoder = bytePairRankDecoder\n    this.bytePairStringRankEncoder = new Map<string, number>()\n    this.mergeCacheSize = mergeCacheSize\n    if (mergeCacheSize > 0) {\n      this.mergeCache = new Map()\n    }\n\n    // size without array holes (which may be present in the encoder)\n    this.mergeableBytePairRankCount = Object.keys(bytePairRankDecoder).length\n    const binaryLookup: [Uint8Array, number][] = []\n    // forEach skips array holes:\n    bytePairRankDecoder.forEach((value, rank) => {\n      if (typeof value === 'string') {\n        this.bytePairStringRankEncoder.set(value, rank)\n        return\n      }\n      const byteArray = new Uint8Array(value)\n      binaryLookup.push([byteArray, rank])\n      this.bytePairNonUtfRankDecoder.set(rank, byteArray)\n      if (startsWithByteOrderMark(byteArray)) {\n        // The rank tables keep tokens that start with a UTF-8 byte order mark\n        // (EF BB BF) as raw bytes, because a plain decode would strip the mark.\n        // They are still valid UTF-8, so they must also be reachable through\n        // the string lookup that getBpeRankFromBytes uses for valid UTF-8.\n        const asString = tryConvertToString(byteArray)\n        if (asString !== undefined) {\n          this.bytePairStringRankEncoder.set(asString, rank)\n        }\n      }\n    })\n    this.bytePairNonUtfSortedEncoder = binaryLookup.sort((a, b) =>\n      compareUint8Arrays(a[0], b[0]),\n    )\n    this.specialTokensEncoder =\n      specialTokensEncoder ?? new Map<string, number>()\n    this.specialTokensDecoder = specialTokensEncoder\n      ? new Map([...specialTokensEncoder].map(([key, value]) => [value, key]))\n      : new Map<number, string>()\n    this.tokenSplitRegex = tokenSplitRegex\n\n    const escapedSpecialTokens = [...this.specialTokensEncoder.keys()].map(\n      escapeRegExp,\n    )\n    const allSpecialTokensRegex = escapedSpecialTokens.join('|')\n    try {\n      this.specialTokenPatternRegex = new RegExp(allSpecialTokensRegex, 'y')\n    } catch {\n      throw new Error('Invalid regular expression pattern.')\n    }\n  }\n\n  setMergeCacheSize(newSize: number): void {\n    if (this.mergeCacheSize === 0 && newSize > 0) {\n      this.mergeCache = new Map()\n    }\n    this.mergeCacheSize = newSize\n    if (newSize === 0) {\n      this.mergeCache = undefined\n    }\n  }\n\n  clearMergeCache(): void {\n    this.mergeCache?.clear()\n  }\n\n  *encodeNativeGenerator(\n    text: string,\n    allowedSpecial?: Set<string>,\n  ): Generator<number[], number, undefined> {\n    let startIndex = 0\n    let lastTokenLength = 0\n\n    while (true) {\n      const nextSpecialMatch = this.findNextSpecialToken(\n        text,\n        allowedSpecial,\n        startIndex,\n      )\n      const nextSpecialStartIndex = nextSpecialMatch?.[0]\n\n      const endIndex = nextSpecialStartIndex ?? text.length\n\n      const textBeforeSpecial =\n        startIndex === 0 && endIndex === text.length\n          ? text\n          : text.slice(startIndex, endIndex)\n\n      for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {\n        const token = this.getBpeRankFromString(match)\n        if (token !== undefined) {\n          lastTokenLength = 1\n          yield [token]\n\n          continue\n        }\n\n        const tokens = this.bytePairEncode(match)\n        lastTokenLength = tokens.length\n        yield tokens\n      }\n\n      if (nextSpecialStartIndex !== undefined) {\n        const specialToken = nextSpecialMatch![1]\n        const specialTokenValue = this.specialTokensEncoder.get(specialToken)\n        if (specialTokenValue === undefined) {\n          throw new Error(\n            `Special token \"${specialToken}\" is not in the special token encoder.`,\n          )\n        }\n        yield [specialTokenValue]\n        startIndex = nextSpecialStartIndex + specialToken.length\n        lastTokenLength = 1\n      } else {\n        break\n      }\n    }\n\n    return lastTokenLength\n  }\n\n  encodeNative(text: string, allowedSpecial?: Set<string>): number[] {\n    let startIndex = 0\n    const tokensArray: number[] = [] // Flat list to collect the tokens\n\n    while (true) {\n      const nextSpecialMatch = this.findNextSpecialToken(\n        text,\n        allowedSpecial,\n        startIndex,\n      )\n      const nextSpecialStartIndex = nextSpecialMatch?.[0]\n\n      const endIndex = nextSpecialStartIndex ?? text.length\n\n      const textBeforeSpecial =\n        startIndex === 0 && endIndex === text.length\n          ? text\n          : text.slice(startIndex, endIndex)\n\n      for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {\n        const token = this.getBpeRankFromString(match)\n        if (token !== undefined) {\n          tokensArray.push(token)\n\n          continue\n        }\n\n        const tokens = this.bytePairEncode(match)\n        tokensArray.push(...tokens)\n      }\n\n      if (nextSpecialStartIndex !== undefined) {\n        const specialToken = nextSpecialMatch![1]\n        const specialTokenValue = this.specialTokensEncoder.get(specialToken)\n        if (specialTokenValue === undefined) {\n          throw new Error(\n            `Special token \"${specialToken}\" is not in the special token encoder.`,\n          )\n        }\n        tokensArray.push(specialTokenValue)\n        startIndex = nextSpecialStartIndex + specialToken.length\n      } else {\n        break\n      }\n    }\n\n    return tokensArray\n  }\n\n  countNative(text: string, allowedSpecial?: Set<string>): number {\n    let startIndex = 0\n    let tokensCount = 0\n\n    while (true) {\n      const nextSpecialMatch = this.findNextSpecialToken(\n        text,\n        allowedSpecial,\n        startIndex,\n      )\n      const nextSpecialStartIndex = nextSpecialMatch?.[0]\n\n      const endIndex = nextSpecialStartIndex ?? text.length\n\n      const textBeforeSpecial =\n        startIndex === 0 && endIndex === text.length\n          ? text\n          : text.slice(startIndex, endIndex)\n\n      for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {\n        const token = this.getBpeRankFromString(match)\n        if (token !== undefined) {\n          tokensCount++\n\n          continue\n        }\n\n        const tokens = this.bytePairEncode(match)\n        tokensCount += tokens.length\n      }\n\n      if (nextSpecialStartIndex !== undefined) {\n        const specialToken = nextSpecialMatch![1]\n        const specialTokenValue = this.specialTokensEncoder.get(specialToken)\n        if (specialTokenValue === undefined) {\n          throw new Error(\n            `Special token \"${specialToken}\" is not in the special token encoder.`,\n          )\n        }\n        tokensCount++\n        startIndex = nextSpecialStartIndex + specialToken.length\n      } else {\n        break\n      }\n    }\n\n    return tokensCount\n  }\n\n  *decodeNativeGenerator(\n    tokens: Iterable<number>,\n  ): Generator<Uint8Array | string, void, void> {\n    for (const token of tokens) {\n      const tokenBytes = this.tryDecodeToken(token)\n      if (tokenBytes) {\n        yield tokenBytes\n      }\n    }\n  }\n\n  decodeNative(tokens: Iterable<number>): string {\n    let decoded = ''\n    let intBuffer = emptyBuffer\n\n    for (const token of tokens) {\n      const tokenBytes = this.tryDecodeToken(token)\n      if (tokenBytes === undefined) {\n        throw new Error(`Token ${token} is not in the byte pair encoder.`)\n      }\n      if (typeof tokenBytes === 'string') {\n        if (intBuffer !== emptyBuffer) {\n          decoded += decoder.decode(intBuffer, { stream: true })\n          intBuffer = emptyBuffer\n        }\n        decoded += tokenBytes\n      } else {\n        const newBuffer = new Uint8Array(intBuffer.length + tokenBytes.length)\n        newBuffer.set(intBuffer)\n        newBuffer.set(tokenBytes, intBuffer.length)\n        intBuffer = newBuffer\n      }\n    }\n\n    if (intBuffer !== emptyBuffer) {\n      decoded += decoder.decode(intBuffer, { stream: true })\n    }\n    return decoded\n  }\n\n  async *decodeNativeAsyncIterable(\n    tokens: AsyncIterable<number>,\n  ): AsyncGenerator<Uint8Array | string> {\n    for await (const token of tokens) {\n      const tokenBytesOrString = this.tryDecodeToken(token)\n      if (tokenBytesOrString) {\n        yield tokenBytesOrString\n      }\n    }\n  }\n\n  private getBpeRankFromString(key: string): number | undefined {\n    return this.bytePairStringRankEncoder.get(key)\n  }\n\n  private getBpeRankFromStringOrThrow(key: string): number {\n    const value = this.getBpeRankFromString(key)\n    if (value === undefined) {\n      throw new Error(\n        `The byte-pair encoding does not contain a value for: ${key}`,\n      )\n    }\n    return value\n  }\n\n  private getBpeRankFromBytes(key: Uint8Array): number | undefined {\n    const keyAsString = tryConvertToString(key)\n\n    if (keyAsString !== undefined) {\n      return this.getBpeRankFromString(keyAsString)\n    }\n\n    // Perform binary search on the binary keys\n    const index = this.binarySearch(key)\n    if (index !== -1) {\n      return this.bytePairNonUtfSortedEncoder[index]![1]\n    }\n\n    return undefined\n  }\n\n  private getBpeRankFromBytesOrThrow(key: Uint8Array): number {\n    const value = this.getBpeRankFromBytes(key)\n    if (value === undefined) {\n      throw new Error(\n        `The byte-pair encoding does not contain a value for: ${key.toString()}`,\n      )\n    }\n    return value\n  }\n\n  // Binary search on the binary keys\n  private binarySearch(key: Uint8Array): number {\n    let low = 0\n    let high = this.bytePairNonUtfSortedEncoder.length - 1\n\n    while (low <= high) {\n      const mid = (low + high) >>> 1\n      const midKey = this.bytePairNonUtfSortedEncoder[mid]![0]\n      let cmp = 0\n      const maxLength = Math.min(midKey.length, key.length)\n      for (let i = 0; i < maxLength; i++) {\n        cmp = midKey[i]! - key[i]!\n        if (cmp !== 0) break\n      }\n      if (cmp === 0) {\n        cmp = midKey.length - key.length\n      }\n      if (cmp === 0) {\n        return mid\n      }\n      if (cmp < 0) {\n        low = mid + 1\n      } else {\n        high = mid - 1\n      }\n    }\n    return -1\n  }\n\n  private findNextSpecialToken(\n    text: string,\n    allowedSpecial: Set<string> | undefined,\n    startIndex: number,\n  ): [startIndex: number, token: string] | undefined {\n    let searchIndex = startIndex\n\n    while (true) {\n      this.specialTokenPatternRegex.lastIndex = searchIndex\n      const nextSpecialMatch = this.specialTokenPatternRegex.exec(text)\n\n      if (!nextSpecialMatch) {\n        return undefined\n      }\n\n      const specialToken = nextSpecialMatch[0]\n\n      if (allowedSpecial?.has(specialToken)) {\n        const specialTokenStartIndex = nextSpecialMatch.index + searchIndex\n        return [specialTokenStartIndex, specialToken]\n      }\n\n      searchIndex = nextSpecialMatch.index + searchIndex + 1\n    }\n  }\n\n  private tryDecodeToken(tokenRank: number): Uint8Array | string | undefined {\n    const value = this.bytePairRankDecoder[tokenRank]\n    if (typeof value === 'string') {\n      return value\n    }\n    if (typeof value === 'object') {\n      const fromBinary = this.bytePairNonUtfRankDecoder.get(tokenRank)\n      if (fromBinary) {\n        return fromBinary\n      }\n    }\n    return this.specialTokensDecoder.get(tokenRank)\n  }\n\n  private addToMergeCache(key: string, value: number[]): void {\n    if (!this.mergeCache) return\n\n    if (this.mergeCache.size >= this.mergeCacheSize) {\n      // Remove least recently used item (first item)\n      const firstKey = this.mergeCache.keys().next().value!\n      this.mergeCache.delete(firstKey)\n    }\n    this.mergeCache.set(key, value)\n  }\n\n  private bytePairEncode(input: string): number[] {\n    if (input.length === 1 && isAscii(input.codePointAt(0)!)) {\n      return [this.getBpeRankFromStringOrThrow(input)]\n    }\n\n    if (this.mergeCache?.has(input)) {\n      const result = this.mergeCache.get(input)!\n      // Move to end to mark as recently used\n      this.mergeCache.delete(input)\n      this.mergeCache.set(input, result)\n      return result\n    }\n\n    const inputBytes = this.textEncoder.encode(input)\n    const result = this.bytePairMerge(inputBytes)\n    this.addToMergeCache(input, result)\n    return result\n  }\n\n  private bytePairMerge(\n    // Input array of bytes to process\n    piece: Uint8Array,\n  ): number[] {\n    const { length } = piece\n    // Partitions form a linked list over byte offsets: a live partition starting at\n    // offset 'i' spans [i, next[i]). 'pairRanks[i]' is the rank of merging it with\n    // the partition after it. Merges come off a min-heap ordered by (rank, offset),\n    // which picks the same pair as scanning for the lowest rank, leftmost first,\n    // without rescanning the piece after every merge.\n    const next = new Int32Array(length + 1)\n    const previous = new Int32Array(length + 1)\n    const pairRanks = new Float64Array(length + 1)\n    const live = new Uint8Array(length + 1)\n\n    const getPairRank = (startIndex: number): number => {\n      const middle = next[startIndex]!\n      if (middle >= length) return Number.POSITIVE_INFINITY\n      const rank = this.getBpeRankFromBytes(\n        piece.subarray(startIndex, next[middle]!),\n      )\n      return rank ?? Number.POSITIVE_INFINITY\n    }\n\n    // Heap entries are packed as [rank, startIndex] pairs in a flat array\n    const heap: number[] = []\n    const isLower = (a: number, b: number) =>\n      heap[a]! < heap[b]! ||\n      (heap[a] === heap[b] && heap[a + 1]! < heap[b + 1]!)\n    const swap = (a: number, b: number) => {\n      const rank = heap[a]!\n      const startIndex = heap[a + 1]!\n      heap[a] = heap[b]!\n      heap[a + 1] = heap[b + 1]!\n      heap[b] = rank\n      heap[b + 1] = startIndex\n    }\n    const push = (rank: number, startIndex: number) => {\n      heap.push(rank, startIndex)\n      let index = heap.length - 2\n      while (index > 0) {\n        const parent = (((index >> 1) - 1) >> 1) << 1\n        if (!isLower(index, parent)) break\n        swap(index, parent)\n        index = parent\n      }\n    }\n    const pop = () => {\n      const last = heap.length - 2\n      swap(0, last)\n      const rank = heap[last]!\n      const startIndex = heap[last + 1]!\n      heap.length = last\n      let index = 0\n      while (true) {\n        const left = index * 2 + 2\n        const right = left + 2\n        let lowest = index\n        if (left < heap.length && isLower(left, lowest)) lowest = left\n        if (right < heap.length && isLower(right, lowest)) lowest = right\n        if (lowest === index) break\n        swap(index, lowest)\n        index = lowest\n      }\n      return [rank, startIndex] as const\n    }\n\n    for (let i = 0; i <= length; i++) {\n      next[i] = i + 1\n      previous[i] = i - 1\n      live[i] = 1\n    }\n    for (let i = 0; i < length - 1; i++) {\n      pairRanks[i] = getPairRank(i)\n      if (pairRanks[i] !== Number.POSITIVE_INFINITY) push(pairRanks[i]!, i)\n    }\n\n    // Iteratively merge byte pairs until no more useful merges can be done\n    while (heap.length > 0) {\n      const [rank, startIndex] = pop()\n      // Skip entries made stale by an earlier merge\n      if (!live[startIndex] || pairRanks[startIndex] !== rank) continue\n\n      // Merge the partition at 'startIndex' with the one after it\n      const merged = next[startIndex]!\n      live[merged] = 0\n      next[startIndex] = next[merged]!\n      previous[next[merged]!] = startIndex\n\n      pairRanks[startIndex] = getPairRank(startIndex)\n      if (pairRanks[startIndex] !== Number.POSITIVE_INFINITY) {\n        push(pairRanks[startIndex]!, startIndex)\n      }\n\n      const before = previous[startIndex]!\n      if (before >= 0) {\n        pairRanks[before] = getPairRank(before)\n        if (pairRanks[before] !== Number.POSITIVE_INFINITY) {\n          push(pairRanks[before]!, before)\n        }\n      }\n    }\n\n    // Create the final output by applying the transform function to each partitioned range\n    const output: number[] = []\n    for (let i = 0; i < length; i = next[i]!) {\n      output.push(this.getBpeRankFromBytesOrThrow(piece.subarray(i, next[i]!)))\n    }\n    return output\n  }\n}\n"],"mappings":";;;;;AAwBA,MAAM,8BAAc,IAAI,WAAW,CAAC;AAGpC,MAAa,UAAU,IAAI,YAAY,QAAQ,EAAE,WAAW,KAAK,CAAC;AAElE,IAAa,uBAAb,MAAkC;CAChC;;;;;;CAMA;CACA,4CAAoC,IAAI,IAAwB;CAChE;;;;;;CAMA;CACA;CACA;CACA;CACA;CACA,cAAsB,IAAI,YAAY;CACtC;CACA;CAEA,YAAY,EACV,qBACA,sBACA,iBACA,iBAAiBA,kBAAAA,4BACQ;EACzB,KAAK,sBAAsB;EAC3B,KAAK,4CAA4B,IAAI,IAAoB;EACzD,KAAK,iBAAiB;EACtB,IAAI,iBAAiB,GACnB,KAAK,6BAAa,IAAI,IAAI;EAI5B,KAAK,6BAA6B,OAAO,KAAK,mBAAmB,CAAC,CAAC;EACnE,MAAM,eAAuC,CAAC;EAE9C,oBAAoB,SAAS,OAAO,SAAS;GAC3C,IAAI,OAAO,UAAU,UAAU;IAC7B,KAAK,0BAA0B,IAAI,OAAO,IAAI;IAC9C;GACF;GACA,MAAM,YAAY,IAAI,WAAW,KAAK;GACtC,aAAa,KAAK,CAAC,WAAW,IAAI,CAAC;GACnC,KAAK,0BAA0B,IAAI,MAAM,SAAS;GAClD,IAAIC,gBAAAA,wBAAwB,SAAS,GAAG;IAKtC,MAAM,WAAWC,gBAAAA,mBAAmB,SAAS;IAC7C,IAAI,aAAa,KAAA,GACf,KAAK,0BAA0B,IAAI,UAAU,IAAI;GAErD;EACF,CAAC;EACD,KAAK,8BAA8B,aAAa,MAAM,GAAG,MACvDC,gBAAAA,mBAAmB,EAAE,IAAI,EAAE,EAAE,CAC/B;EACA,KAAK,uBACH,wCAAwB,IAAI,IAAoB;EAClD,KAAK,uBAAuB,uBACxB,IAAI,IAAI,CAAC,GAAG,oBAAoB,CAAC,CAAC,KAAK,CAAC,KAAK,WAAW,CAAC,OAAO,GAAG,CAAC,CAAC,oBACrE,IAAI,IAAoB;EAC5B,KAAK,kBAAkB;EAKvB,MAAM,wBAHuB,CAAC,GAAG,KAAK,qBAAqB,KAAK,CAAC,CAAC,CAAC,IACjEC,aAAAA,YAE+C,CAAC,CAAC,KAAK,GAAG;EAC3D,IAAI;GACF,KAAK,2BAA2B,IAAI,OAAO,uBAAuB,GAAG;EACvE,QAAQ;GACN,MAAM,IAAI,MAAM,qCAAqC;EACvD;CACF;CAEA,kBAAkB,SAAuB;EACvC,IAAI,KAAK,mBAAmB,KAAK,UAAU,GACzC,KAAK,6BAAa,IAAI,IAAI;EAE5B,KAAK,iBAAiB;EACtB,IAAI,YAAY,GACd,KAAK,aAAa,KAAA;CAEtB;CAEA,kBAAwB;EACtB,KAAK,YAAY,MAAM;CACzB;CAEA,CAAC,sBACC,MACA,gBACwC;EACxC,IAAI,aAAa;EACjB,IAAI,kBAAkB;EAEtB,OAAO,MAAM;GACX,MAAM,mBAAmB,KAAK,qBAC5B,MACA,gBACA,UACF;GACA,MAAM,wBAAwB,mBAAmB;GAEjD,MAAM,WAAW,yBAAyB,KAAK;GAE/C,MAAM,oBACJ,eAAe,KAAK,aAAa,KAAK,SAClC,OACA,KAAK,MAAM,YAAY,QAAQ;GAErC,KAAK,MAAM,CAAC,UAAU,kBAAkB,SAAS,KAAK,eAAe,GAAG;IACtE,MAAM,QAAQ,KAAK,qBAAqB,KAAK;IAC7C,IAAI,UAAU,KAAA,GAAW;KACvB,kBAAkB;KAClB,MAAM,CAAC,KAAK;KAEZ;IACF;IAEA,MAAM,SAAS,KAAK,eAAe,KAAK;IACxC,kBAAkB,OAAO;IACzB,MAAM;GACR;GAEA,IAAI,0BAA0B,KAAA,GAAW;IACvC,MAAM,eAAe,iBAAkB;IACvC,MAAM,oBAAoB,KAAK,qBAAqB,IAAI,YAAY;IACpE,IAAI,sBAAsB,KAAA,GACxB,MAAM,IAAI,MACR,kBAAkB,aAAa,uCACjC;IAEF,MAAM,CAAC,iBAAiB;IACxB,aAAa,wBAAwB,aAAa;IAClD,kBAAkB;GACpB,OACE;EAEJ;EAEA,OAAO;CACT;CAEA,aAAa,MAAc,gBAAwC;EACjE,IAAI,aAAa;EACjB,MAAM,cAAwB,CAAC;EAE/B,OAAO,MAAM;GACX,MAAM,mBAAmB,KAAK,qBAC5B,MACA,gBACA,UACF;GACA,MAAM,wBAAwB,mBAAmB;GAEjD,MAAM,WAAW,yBAAyB,KAAK;GAE/C,MAAM,oBACJ,eAAe,KAAK,aAAa,KAAK,SAClC,OACA,KAAK,MAAM,YAAY,QAAQ;GAErC,KAAK,MAAM,CAAC,UAAU,kBAAkB,SAAS,KAAK,eAAe,GAAG;IACtE,MAAM,QAAQ,KAAK,qBAAqB,KAAK;IAC7C,IAAI,UAAU,KAAA,GAAW;KACvB,YAAY,KAAK,KAAK;KAEtB;IACF;IAEA,MAAM,SAAS,KAAK,eAAe,KAAK;IACxC,YAAY,KAAK,GAAG,MAAM;GAC5B;GAEA,IAAI,0BAA0B,KAAA,GAAW;IACvC,MAAM,eAAe,iBAAkB;IACvC,MAAM,oBAAoB,KAAK,qBAAqB,IAAI,YAAY;IACpE,IAAI,sBAAsB,KAAA,GACxB,MAAM,IAAI,MACR,kBAAkB,aAAa,uCACjC;IAEF,YAAY,KAAK,iBAAiB;IAClC,aAAa,wBAAwB,aAAa;GACpD,OACE;EAEJ;EAEA,OAAO;CACT;CAEA,YAAY,MAAc,gBAAsC;EAC9D,IAAI,aAAa;EACjB,IAAI,cAAc;EAElB,OAAO,MAAM;GACX,MAAM,mBAAmB,KAAK,qBAC5B,MACA,gBACA,UACF;GACA,MAAM,wBAAwB,mBAAmB;GAEjD,MAAM,WAAW,yBAAyB,KAAK;GAE/C,MAAM,oBACJ,eAAe,KAAK,aAAa,KAAK,SAClC,OACA,KAAK,MAAM,YAAY,QAAQ;GAErC,KAAK,MAAM,CAAC,UAAU,kBAAkB,SAAS,KAAK,eAAe,GAAG;IAEtE,IADc,KAAK,qBAAqB,KAChC,MAAM,KAAA,GAAW;KACvB;KAEA;IACF;IAEA,MAAM,SAAS,KAAK,eAAe,KAAK;IACxC,eAAe,OAAO;GACxB;GAEA,IAAI,0BAA0B,KAAA,GAAW;IACvC,MAAM,eAAe,iBAAkB;IAEvC,IAD0B,KAAK,qBAAqB,IAAI,YACpC,MAAM,KAAA,GACxB,MAAM,IAAI,MACR,kBAAkB,aAAa,uCACjC;IAEF;IACA,aAAa,wBAAwB,aAAa;GACpD,OACE;EAEJ;EAEA,OAAO;CACT;CAEA,CAAC,sBACC,QAC4C;EAC5C,KAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,aAAa,KAAK,eAAe,KAAK;GAC5C,IAAI,YACF,MAAM;EAEV;CACF;CAEA,aAAa,QAAkC;EAC7C,IAAI,UAAU;EACd,IAAI,YAAY;EAEhB,KAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,aAAa,KAAK,eAAe,KAAK;GAC5C,IAAI,eAAe,KAAA,GACjB,MAAM,IAAI,MAAM,SAAS,MAAM,kCAAkC;GAEnE,IAAI,OAAO,eAAe,UAAU;IAClC,IAAI,cAAc,aAAa;KAC7B,WAAW,QAAQ,OAAO,WAAW,EAAE,QAAQ,KAAK,CAAC;KACrD,YAAY;IACd;IACA,WAAW;GACb,OAAO;IACL,MAAM,YAAY,IAAI,WAAW,UAAU,SAAS,WAAW,MAAM;IACrE,UAAU,IAAI,SAAS;IACvB,UAAU,IAAI,YAAY,UAAU,MAAM;IAC1C,YAAY;GACd;EACF;EAEA,IAAI,cAAc,aAChB,WAAW,QAAQ,OAAO,WAAW,EAAE,QAAQ,KAAK,CAAC;EAEvD,OAAO;CACT;CAEA,OAAO,0BACL,QACqC;EACrC,WAAW,MAAM,SAAS,QAAQ;GAChC,MAAM,qBAAqB,KAAK,eAAe,KAAK;GACpD,IAAI,oBACF,MAAM;EAEV;CACF;CAEA,qBAA6B,KAAiC;EAC5D,OAAO,KAAK,0BAA0B,IAAI,GAAG;CAC/C;CAEA,4BAAoC,KAAqB;EACvD,MAAM,QAAQ,KAAK,qBAAqB,GAAG;EAC3C,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,MACR,wDAAwD,KAC1D;EAEF,OAAO;CACT;CAEA,oBAA4B,KAAqC;EAC/D,MAAM,cAAcF,gBAAAA,mBAAmB,GAAG;EAE1C,IAAI,gBAAgB,KAAA,GAClB,OAAO,KAAK,qBAAqB,WAAW;EAI9C,MAAM,QAAQ,KAAK,aAAa,GAAG;EACnC,IAAI,UAAU,IACZ,OAAO,KAAK,4BAA4B,MAAM,CAAE;CAIpD;CAEA,2BAAmC,KAAyB;EAC1D,MAAM,QAAQ,KAAK,oBAAoB,GAAG;EAC1C,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,MACR,wDAAwD,IAAI,SAAS,GACvE;EAEF,OAAO;CACT;CAGA,aAAqB,KAAyB;EAC5C,IAAI,MAAM;EACV,IAAI,OAAO,KAAK,4BAA4B,SAAS;EAErD,OAAO,OAAO,MAAM;GAClB,MAAM,MAAO,MAAM,SAAU;GAC7B,MAAM,SAAS,KAAK,4BAA4B,IAAI,CAAE;GACtD,IAAI,MAAM;GACV,MAAM,YAAY,KAAK,IAAI,OAAO,QAAQ,IAAI,MAAM;GACpD,KAAK,IAAI,IAAI,GAAG,IAAI,WAAW,KAAK;IAClC,MAAM,OAAO,KAAM,IAAI;IACvB,IAAI,QAAQ,GAAG;GACjB;GACA,IAAI,QAAQ,GACV,MAAM,OAAO,SAAS,IAAI;GAE5B,IAAI,QAAQ,GACV,OAAO;GAET,IAAI,MAAM,GACR,MAAM,MAAM;QAEZ,OAAO,MAAM;EAEjB;EACA,OAAO;CACT;CAEA,qBACE,MACA,gBACA,YACiD;EACjD,IAAI,cAAc;EAElB,OAAO,MAAM;GACX,KAAK,yBAAyB,YAAY;GAC1C,MAAM,mBAAmB,KAAK,yBAAyB,KAAK,IAAI;GAEhE,IAAI,CAAC,kBACH;GAGF,MAAM,eAAe,iBAAiB;GAEtC,IAAI,gBAAgB,IAAI,YAAY,GAElC,OAAO,CADwB,iBAAiB,QAAQ,aACxB,YAAY;GAG9C,cAAc,iBAAiB,QAAQ,cAAc;EACvD;CACF;CAEA,eAAuB,WAAoD;EACzE,MAAM,QAAQ,KAAK,oBAAoB;EACvC,IAAI,OAAO,UAAU,UACnB,OAAO;EAET,IAAI,OAAO,UAAU,UAAU;GAC7B,MAAM,aAAa,KAAK,0BAA0B,IAAI,SAAS;GAC/D,IAAI,YACF,OAAO;EAEX;EACA,OAAO,KAAK,qBAAqB,IAAI,SAAS;CAChD;CAEA,gBAAwB,KAAa,OAAuB;EAC1D,IAAI,CAAC,KAAK,YAAY;EAEtB,IAAI,KAAK,WAAW,QAAQ,KAAK,gBAAgB;GAE/C,MAAM,WAAW,KAAK,WAAW,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC;GAC/C,KAAK,WAAW,OAAO,QAAQ;EACjC;EACA,KAAK,WAAW,IAAI,KAAK,KAAK;CAChC;CAEA,eAAuB,OAAyB;EAC9C,IAAI,MAAM,WAAW,KAAKG,gBAAAA,QAAQ,MAAM,YAAY,CAAC,CAAE,GACrD,OAAO,CAAC,KAAK,4BAA4B,KAAK,CAAC;EAGjD,IAAI,KAAK,YAAY,IAAI,KAAK,GAAG;GAC/B,MAAM,SAAS,KAAK,WAAW,IAAI,KAAK;GAExC,KAAK,WAAW,OAAO,KAAK;GAC5B,KAAK,WAAW,IAAI,OAAO,MAAM;GACjC,OAAO;EACT;EAEA,MAAM,aAAa,KAAK,YAAY,OAAO,KAAK;EAChD,MAAM,SAAS,KAAK,cAAc,UAAU;EAC5C,KAAK,gBAAgB,OAAO,MAAM;EAClC,OAAO;CACT;CAEA,cAEE,OACU;EACV,MAAM,EAAE,WAAW;EAMnB,MAAM,OAAO,IAAI,WAAW,SAAS,CAAC;EACtC,MAAM,WAAW,IAAI,WAAW,SAAS,CAAC;EAC1C,MAAM,YAAY,IAAI,aAAa,SAAS,CAAC;EAC7C,MAAM,OAAO,IAAI,WAAW,SAAS,CAAC;EAEtC,MAAM,eAAe,eAA+B;GAClD,MAAM,SAAS,KAAK;GACpB,IAAI,UAAU,QAAQ,OAAO,OAAO;GAIpC,OAHa,KAAK,oBAChB,MAAM,SAAS,YAAY,KAAK,OAAQ,CAEhC,KAAK,OAAO;EACxB;EAGA,MAAM,OAAiB,CAAC;EACxB,MAAM,WAAW,GAAW,MAC1B,KAAK,KAAM,KAAK,MACf,KAAK,OAAO,KAAK,MAAM,KAAK,IAAI,KAAM,KAAK,IAAI;EAClD,MAAM,QAAQ,GAAW,MAAc;GACrC,MAAM,OAAO,KAAK;GAClB,MAAM,aAAa,KAAK,IAAI;GAC5B,KAAK,KAAK,KAAK;GACf,KAAK,IAAI,KAAK,KAAK,IAAI;GACvB,KAAK,KAAK;GACV,KAAK,IAAI,KAAK;EAChB;EACA,MAAM,QAAQ,MAAc,eAAuB;GACjD,KAAK,KAAK,MAAM,UAAU;GAC1B,IAAI,QAAQ,KAAK,SAAS;GAC1B,OAAO,QAAQ,GAAG;IAChB,MAAM,UAAY,SAAS,KAAK,KAAM,KAAM;IAC5C,IAAI,CAAC,QAAQ,OAAO,MAAM,GAAG;IAC7B,KAAK,OAAO,MAAM;IAClB,QAAQ;GACV;EACF;EACA,MAAM,YAAY;GAChB,MAAM,OAAO,KAAK,SAAS;GAC3B,KAAK,GAAG,IAAI;GACZ,MAAM,OAAO,KAAK;GAClB,MAAM,aAAa,KAAK,OAAO;GAC/B,KAAK,SAAS;GACd,IAAI,QAAQ;GACZ,OAAO,MAAM;IACX,MAAM,OAAO,QAAQ,IAAI;IACzB,MAAM,QAAQ,OAAO;IACrB,IAAI,SAAS;IACb,IAAI,OAAO,KAAK,UAAU,QAAQ,MAAM,MAAM,GAAG,SAAS;IAC1D,IAAI,QAAQ,KAAK,UAAU,QAAQ,OAAO,MAAM,GAAG,SAAS;IAC5D,IAAI,WAAW,OAAO;IACtB,KAAK,OAAO,MAAM;IAClB,QAAQ;GACV;GACA,OAAO,CAAC,MAAM,UAAU;EAC1B;EAEA,KAAK,IAAI,IAAI,GAAG,KAAK,QAAQ,KAAK;GAChC,KAAK,KAAK,IAAI;GACd,SAAS,KAAK,IAAI;GAClB,KAAK,KAAK;EACZ;EACA,KAAK,IAAI,IAAI,GAAG,IAAI,SAAS,GAAG,KAAK;GACnC,UAAU,KAAK,YAAY,CAAC;GAC5B,IAAI,UAAU,OAAO,OAAO,mBAAmB,KAAK,UAAU,IAAK,CAAC;EACtE;EAGA,OAAO,KAAK,SAAS,GAAG;GACtB,MAAM,CAAC,MAAM,cAAc,IAAI;GAE/B,IAAI,CAAC,KAAK,eAAe,UAAU,gBAAgB,MAAM;GAGzD,MAAM,SAAS,KAAK;GACpB,KAAK,UAAU;GACf,KAAK,cAAc,KAAK;GACxB,SAAS,KAAK,WAAY;GAE1B,UAAU,cAAc,YAAY,UAAU;GAC9C,IAAI,UAAU,gBAAgB,OAAO,mBACnC,KAAK,UAAU,aAAc,UAAU;GAGzC,MAAM,SAAS,SAAS;GACxB,IAAI,UAAU,GAAG;IACf,UAAU,UAAU,YAAY,MAAM;IACtC,IAAI,UAAU,YAAY,OAAO,mBAC/B,KAAK,UAAU,SAAU,MAAM;GAEnC;EACF;EAGA,MAAM,SAAmB,CAAC;EAC1B,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,IAAI,KAAK,IACnC,OAAO,KAAK,KAAK,2BAA2B,MAAM,SAAS,GAAG,KAAK,EAAG,CAAC,CAAC;EAE1E,OAAO;CACT;AACF"}

package/cjs/encodingParams/constants.js

@@ -1,11 +1,11 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
 //#region src/encodingParams/constants.ts
-const R50K_TOKEN_SPLIT_REGEX = /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu;
+const R50K_TOKEN_SPLIT_REGEX = /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\p{White_Space}\p{L}\p{N}]+|\p{White_Space}+(?!\P{White_Space})|\p{White_Space}+/gu;
 const CONTRACTION_SUFFIX_PATTERN = String.raw`'(?:[sS]|[dD]|[mM]|[tT]|[lL][lL]|[vV][eE]|[rR][eE])`;
 const OPTIONAL_CONTRACTION_SUFFIX = String.raw`(?:${CONTRACTION_SUFFIX_PATTERN})?`;
-const CL100K_TOKEN_SPLIT_PATTERN = String.raw`${CONTRACTION_SUFFIX_PATTERN}|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s+$|\s*[\r\n]|\s+(?!\S)|\s`;
+const CL100K_TOKEN_SPLIT_PATTERN = String.raw`${CONTRACTION_SUFFIX_PATTERN}|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\p{White_Space}\p{L}\p{N}]+[\r\n]*|\p{White_Space}+$|\p{White_Space}*[\r\n]|\p{White_Space}+(?!\P{White_Space})|\p{White_Space}`;
 const CL100K_TOKEN_SPLIT_REGEX = new RegExp(CL100K_TOKEN_SPLIT_PATTERN, "gu");
-const O200K_TOKEN_SPLIT_PATTERN = String.raw`[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+${OPTIONAL_CONTRACTION_SUFFIX}|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*${OPTIONAL_CONTRACTION_SUFFIX}|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n/]*|\s*[\r\n]+|\s+(?!\S)|\s+`;
+const O200K_TOKEN_SPLIT_PATTERN = String.raw`[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+${OPTIONAL_CONTRACTION_SUFFIX}|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*${OPTIONAL_CONTRACTION_SUFFIX}|\p{N}{1,3}| ?[^\p{White_Space}\p{L}\p{N}]+[\r\n/]*|\p{White_Space}*[\r\n]+|\p{White_Space}+(?!\P{White_Space})|\p{White_Space}+`;
 const O200K_TOKEN_SPLIT_REGEX = new RegExp(O200K_TOKEN_SPLIT_PATTERN, "gu");
 //#endregion
 exports.CL100K_TOKEN_SPLIT_REGEX = CL100K_TOKEN_SPLIT_REGEX;

package/cjs/encodingParams/constants.js.map

@@ -1 +1 @@
-{"version":3,"file":"constants.js","names":[],"sources":["../../src/encodingParams/constants.ts"],"sourcesContent":["export const R50K_TOKEN_SPLIT_REGEX =\n  /'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+/gu\n\nconst CONTRACTION_SUFFIX_PATTERN = String.raw`'(?:[sS]|[dD]|[mM]|[tT]|[lL][lL]|[vV][eE]|[rR][eE])`\n\nconst OPTIONAL_CONTRACTION_SUFFIX = String.raw`(?:${CONTRACTION_SUFFIX_PATTERN})?`\n\nconst CL100K_TOKEN_SPLIT_PATTERN = String.raw`${CONTRACTION_SUFFIX_PATTERN}|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s+$|\\s*[\\r\\n]|\\s+(?!\\S)|\\s`\n\nexport const CL100K_TOKEN_SPLIT_REGEX = new RegExp(\n  CL100K_TOKEN_SPLIT_PATTERN,\n  'gu',\n)\n\nconst O200K_TOKEN_SPLIT_PATTERN = String.raw`[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+${OPTIONAL_CONTRACTION_SUFFIX}|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*${OPTIONAL_CONTRACTION_SUFFIX}|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+`\n\nexport const O200K_TOKEN_SPLIT_REGEX = new RegExp(\n  O200K_TOKEN_SPLIT_PATTERN,\n  'gu',\n)\n"],"mappings":";;AAAA,MAAa,yBACX;AAEF,MAAM,6BAA6B,OAAO,GAAG;AAE7C,MAAM,8BAA8B,OAAO,GAAG,MAAM,2BAA2B;AAE/E,MAAM,6BAA6B,OAAO,GAAG,GAAG,2BAA2B;AAE3E,MAAa,2BAA2B,IAAI,OAC1C,4BACA,IACF;AAEA,MAAM,4BAA4B,OAAO,GAAG,+EAA+E,4BAA4B,+EAA+E,4BAA4B;AAElQ,MAAa,0BAA0B,IAAI,OACzC,2BACA,IACF"}
+{"version":3,"file":"constants.js","names":[],"sources":["../../src/encodingParams/constants.ts"],"sourcesContent":["// The split patterns below use `\\p{White_Space}` / `\\P{White_Space}` where\n// tiktoken's patterns use `\\s` / `\\S`: the regex engine behind tiktoken treats\n// `\\s` as the Unicode White_Space property, whereas JavaScript's `\\s` differs\n// from it on exactly two code points (U+0085 and U+FEFF).\nexport const R50K_TOKEN_SPLIT_REGEX =\n  /'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\p{White_Space}\\p{L}\\p{N}]+|\\p{White_Space}+(?!\\P{White_Space})|\\p{White_Space}+/gu\n\nconst CONTRACTION_SUFFIX_PATTERN = String.raw`'(?:[sS]|[dD]|[mM]|[tT]|[lL][lL]|[vV][eE]|[rR][eE])`\n\nconst OPTIONAL_CONTRACTION_SUFFIX = String.raw`(?:${CONTRACTION_SUFFIX_PATTERN})?`\n\nconst CL100K_TOKEN_SPLIT_PATTERN = String.raw`${CONTRACTION_SUFFIX_PATTERN}|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\p{White_Space}\\p{L}\\p{N}]+[\\r\\n]*|\\p{White_Space}+$|\\p{White_Space}*[\\r\\n]|\\p{White_Space}+(?!\\P{White_Space})|\\p{White_Space}`\n\nexport const CL100K_TOKEN_SPLIT_REGEX = new RegExp(\n  CL100K_TOKEN_SPLIT_PATTERN,\n  'gu',\n)\n\nconst O200K_TOKEN_SPLIT_PATTERN = String.raw`[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+${OPTIONAL_CONTRACTION_SUFFIX}|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*${OPTIONAL_CONTRACTION_SUFFIX}|\\p{N}{1,3}| ?[^\\p{White_Space}\\p{L}\\p{N}]+[\\r\\n/]*|\\p{White_Space}*[\\r\\n]+|\\p{White_Space}+(?!\\P{White_Space})|\\p{White_Space}+`\n\nexport const O200K_TOKEN_SPLIT_REGEX = new RegExp(\n  O200K_TOKEN_SPLIT_PATTERN,\n  'gu',\n)\n"],"mappings":";;AAIA,MAAa,yBACX;AAEF,MAAM,6BAA6B,OAAO,GAAG;AAE7C,MAAM,8BAA8B,OAAO,GAAG,MAAM,2BAA2B;AAE/E,MAAM,6BAA6B,OAAO,GAAG,GAAG,2BAA2B;AAE3E,MAAa,2BAA2B,IAAI,OAC1C,4BACA,IACF;AAEA,MAAM,4BAA4B,OAAO,GAAG,+EAA+E,4BAA4B,+EAA+E,4BAA4B;AAElQ,MAAa,0BAA0B,IAAI,OACzC,2BACA,IACF"}

package/cjs/utfUtil.d.ts

@@ -1,8 +1,9 @@
 //#region src/utfUtil.d.ts
 declare const isAscii: (codePoint: number) => boolean;
 declare function endsWithIncompleteUtfPairSurrogate(string: string): boolean;
+declare function startsWithByteOrderMark(bytes: Uint8Array): boolean;
 declare function tryConvertToString(arr: Uint8Array): string | undefined;
 declare function compareUint8Arrays(a: Uint8Array, b: Uint8Array): number;
 //#endregion
-export { compareUint8Arrays, endsWithIncompleteUtfPairSurrogate, isAscii, tryConvertToString };
+export { compareUint8Arrays, endsWithIncompleteUtfPairSurrogate, isAscii, startsWithByteOrderMark, tryConvertToString };
 //# sourceMappingURL=utfUtil.d.ts.map

package/cjs/utfUtil.js

@@ -44,7 +44,13 @@ function isValidUTF8(bytes) {
 	}
 	return true;
 }
-const textDecoder = new TextDecoder("utf8", { fatal: false });
+function startsWithByteOrderMark(bytes) {
+	return bytes.length >= 3 && bytes[0] === 239 && bytes[1] === 187 && bytes[2] === 191;
+}
+const textDecoder = new TextDecoder("utf8", {
+	fatal: false,
+	ignoreBOM: true
+});
 function tryConvertToString(arr) {
 	if (!isValidUTF8(arr)) return;
 	return textDecoder.decode(arr);
@@ -58,6 +64,7 @@ function compareUint8Arrays(a, b) {
 exports.compareUint8Arrays = compareUint8Arrays;
 exports.endsWithIncompleteUtfPairSurrogate = endsWithIncompleteUtfPairSurrogate;
 exports.isAscii = isAscii;
+exports.startsWithByteOrderMark = startsWithByteOrderMark;
 exports.tryConvertToString = tryConvertToString;
 
 //# sourceMappingURL=utfUtil.js.map

package/cjs/utfUtil.js.map

@@ -1 +1 @@
-{"version":3,"file":"utfUtil.js","names":[],"sources":["../src/utfUtil.ts"],"sourcesContent":["export const isAscii = (codePoint: number) => codePoint <= 0x7f\n\nconst HIGH_SURROGATE_START = 55_296\nconst HIGH_SURROGATE_END = 56_319\n\nexport function endsWithIncompleteUtfPairSurrogate(string: string): boolean {\n  if (string.length === 0) return false\n  // Check if the last character is a high surrogate\n  const lastCharCode = string.charCodeAt(string.length - 1)\n  return (\n    lastCharCode >= HIGH_SURROGATE_START && lastCharCode <= HIGH_SURROGATE_END\n  )\n}\n\nfunction isValidUTF8(bytes: Uint8Array): boolean {\n  let i = 0\n  while (i < bytes.length) {\n    const byte1 = bytes[i]!\n\n    let numBytes = 0\n    let codePoint = 0\n\n    // Determine the number of bytes in the current UTF-8 character\n    if (byte1 <= 0x7f) {\n      // 1-byte character (ASCII)\n      numBytes = 1\n      codePoint = byte1\n    } else if ((byte1 & 0xe0) === 0xc0) {\n      // 2-byte character\n      numBytes = 2\n      codePoint = byte1 & 0x1f\n      if (byte1 <= 0xc1) return false // Overlong encoding not allowed\n    } else if ((byte1 & 0xf0) === 0xe0) {\n      // 3-byte character\n      numBytes = 3\n      codePoint = byte1 & 0x0f\n    } else if ((byte1 & 0xf8) === 0xf0) {\n      // 4-byte character\n      numBytes = 4\n      codePoint = byte1 & 0x07\n      if (byte1 > 0xf4) return false // Code points above U+10FFFF not allowed\n    } else {\n      // Invalid first byte of UTF-8 character\n      return false\n    }\n\n    // Ensure there are enough continuation bytes\n    if (i + numBytes > bytes.length) return false\n\n    // Process the continuation bytes\n    for (let j = 1; j < numBytes; j++) {\n      const byte = bytes[i + j]\n      if (byte === undefined || (byte & 0xc0) !== 0x80) return false // Continuation bytes must start with '10'\n      codePoint = (codePoint << 6) | (byte & 0x3f)\n    }\n\n    // Check for overlong encodings\n    if (numBytes === 2 && codePoint < 0x80) return false // Overlong 2-byte sequence\n    if (numBytes === 3 && codePoint < 2_048) return false // Overlong 3-byte sequence\n    if (numBytes === 4 && codePoint < 65_536) return false // Overlong 4-byte sequence\n\n    // Check for surrogate halves (U+D800 to U+DFFF)\n    if (codePoint >= 55_296 && codePoint <= 57_343) return false\n\n    // Check for code points above U+10FFFF\n    if (codePoint > 1_114_111) return false\n\n    // Move to the next character\n    i += numBytes\n  }\n  return true\n}\n\nconst textDecoder = new TextDecoder('utf8', { fatal: false })\n\nexport function tryConvertToString(arr: Uint8Array): string | undefined {\n  if (!isValidUTF8(arr)) {\n    return undefined\n  }\n  return textDecoder.decode(arr)\n}\n\n// Helper function to compare two Uint8Arrays lexicographically\nexport function compareUint8Arrays(a: Uint8Array, b: Uint8Array): number {\n  const len = Math.min(a.length, b.length)\n  for (let i = 0; i < len; i++) {\n    if (a[i] !== b[i]) {\n      return a[i]! - b[i]!\n    }\n  }\n  return a.length - b.length\n}\n"],"mappings":";;AAAA,MAAa,WAAW,cAAsB,aAAa;AAE3D,MAAM,uBAAuB;AAC7B,MAAM,qBAAqB;AAE3B,SAAgB,mCAAmC,QAAyB;CAC1E,IAAI,OAAO,WAAW,GAAG,OAAO;CAEhC,MAAM,eAAe,OAAO,WAAW,OAAO,SAAS,CAAC;CACxD,OACE,gBAAgB,wBAAwB,gBAAgB;AAE5D;AAEA,SAAS,YAAY,OAA4B;CAC/C,IAAI,IAAI;CACR,OAAO,IAAI,MAAM,QAAQ;EACvB,MAAM,QAAQ,MAAM;EAEpB,IAAI,WAAW;EACf,IAAI,YAAY;EAGhB,IAAI,SAAS,KAAM;GAEjB,WAAW;GACX,YAAY;EACd,OAAO,KAAK,QAAQ,SAAU,KAAM;GAElC,WAAW;GACX,YAAY,QAAQ;GACpB,IAAI,SAAS,KAAM,OAAO;EAC5B,OAAO,KAAK,QAAQ,SAAU,KAAM;GAElC,WAAW;GACX,YAAY,QAAQ;EACtB,OAAO,KAAK,QAAQ,SAAU,KAAM;GAElC,WAAW;GACX,YAAY,QAAQ;GACpB,IAAI,QAAQ,KAAM,OAAO;EAC3B,OAEE,OAAO;EAIT,IAAI,IAAI,WAAW,MAAM,QAAQ,OAAO;EAGxC,KAAK,IAAI,IAAI,GAAG,IAAI,UAAU,KAAK;GACjC,MAAM,OAAO,MAAM,IAAI;GACvB,IAAI,SAAS,KAAA,MAAc,OAAO,SAAU,KAAM,OAAO;GACzD,YAAa,aAAa,IAAM,OAAO;EACzC;EAGA,IAAI,aAAa,KAAK,YAAY,KAAM,OAAO;EAC/C,IAAI,aAAa,KAAK,YAAY,MAAO,OAAO;EAChD,IAAI,aAAa,KAAK,YAAY,OAAQ,OAAO;EAGjD,IAAI,aAAa,SAAU,aAAa,OAAQ,OAAO;EAGvD,IAAI,YAAY,SAAW,OAAO;EAGlC,KAAK;CACP;CACA,OAAO;AACT;AAEA,MAAM,cAAc,IAAI,YAAY,QAAQ,EAAE,OAAO,MAAM,CAAC;AAE5D,SAAgB,mBAAmB,KAAqC;CACtE,IAAI,CAAC,YAAY,GAAG,GAClB;CAEF,OAAO,YAAY,OAAO,GAAG;AAC/B;AAGA,SAAgB,mBAAmB,GAAe,GAAuB;CACvE,MAAM,MAAM,KAAK,IAAI,EAAE,QAAQ,EAAE,MAAM;CACvC,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,KACvB,IAAI,EAAE,OAAO,EAAE,IACb,OAAO,EAAE,KAAM,EAAE;CAGrB,OAAO,EAAE,SAAS,EAAE;AACtB"}
+{"version":3,"file":"utfUtil.js","names":[],"sources":["../src/utfUtil.ts"],"sourcesContent":["export const isAscii = (codePoint: number) => codePoint <= 0x7f\n\nconst HIGH_SURROGATE_START = 55_296\nconst HIGH_SURROGATE_END = 56_319\n\nexport function endsWithIncompleteUtfPairSurrogate(string: string): boolean {\n  if (string.length === 0) return false\n  // Check if the last character is a high surrogate\n  const lastCharCode = string.charCodeAt(string.length - 1)\n  return (\n    lastCharCode >= HIGH_SURROGATE_START && lastCharCode <= HIGH_SURROGATE_END\n  )\n}\n\nfunction isValidUTF8(bytes: Uint8Array): boolean {\n  let i = 0\n  while (i < bytes.length) {\n    const byte1 = bytes[i]!\n\n    let numBytes = 0\n    let codePoint = 0\n\n    // Determine the number of bytes in the current UTF-8 character\n    if (byte1 <= 0x7f) {\n      // 1-byte character (ASCII)\n      numBytes = 1\n      codePoint = byte1\n    } else if ((byte1 & 0xe0) === 0xc0) {\n      // 2-byte character\n      numBytes = 2\n      codePoint = byte1 & 0x1f\n      if (byte1 <= 0xc1) return false // Overlong encoding not allowed\n    } else if ((byte1 & 0xf0) === 0xe0) {\n      // 3-byte character\n      numBytes = 3\n      codePoint = byte1 & 0x0f\n    } else if ((byte1 & 0xf8) === 0xf0) {\n      // 4-byte character\n      numBytes = 4\n      codePoint = byte1 & 0x07\n      if (byte1 > 0xf4) return false // Code points above U+10FFFF not allowed\n    } else {\n      // Invalid first byte of UTF-8 character\n      return false\n    }\n\n    // Ensure there are enough continuation bytes\n    if (i + numBytes > bytes.length) return false\n\n    // Process the continuation bytes\n    for (let j = 1; j < numBytes; j++) {\n      const byte = bytes[i + j]\n      if (byte === undefined || (byte & 0xc0) !== 0x80) return false // Continuation bytes must start with '10'\n      codePoint = (codePoint << 6) | (byte & 0x3f)\n    }\n\n    // Check for overlong encodings\n    if (numBytes === 2 && codePoint < 0x80) return false // Overlong 2-byte sequence\n    if (numBytes === 3 && codePoint < 2_048) return false // Overlong 3-byte sequence\n    if (numBytes === 4 && codePoint < 65_536) return false // Overlong 4-byte sequence\n\n    // Check for surrogate halves (U+D800 to U+DFFF)\n    if (codePoint >= 55_296 && codePoint <= 57_343) return false\n\n    // Check for code points above U+10FFFF\n    if (codePoint > 1_114_111) return false\n\n    // Move to the next character\n    i += numBytes\n  }\n  return true\n}\n\nexport function startsWithByteOrderMark(bytes: Uint8Array): boolean {\n  return (\n    bytes.length >= 3 &&\n    bytes[0] === 0xef &&\n    bytes[1] === 0xbb &&\n    bytes[2] === 0xbf\n  )\n}\n\n// `ignoreBOM: true` keeps a leading U+FEFF in the output instead of silently\n// stripping it, so the bytes EF BB BF map to U+FEFF rather than to an empty string.\nconst textDecoder = new TextDecoder('utf8', { fatal: false, ignoreBOM: true })\n\nexport function tryConvertToString(arr: Uint8Array): string | undefined {\n  if (!isValidUTF8(arr)) {\n    return undefined\n  }\n  return textDecoder.decode(arr)\n}\n\n// Helper function to compare two Uint8Arrays lexicographically\nexport function compareUint8Arrays(a: Uint8Array, b: Uint8Array): number {\n  const len = Math.min(a.length, b.length)\n  for (let i = 0; i < len; i++) {\n    if (a[i] !== b[i]) {\n      return a[i]! - b[i]!\n    }\n  }\n  return a.length - b.length\n}\n"],"mappings":";;AAAA,MAAa,WAAW,cAAsB,aAAa;AAE3D,MAAM,uBAAuB;AAC7B,MAAM,qBAAqB;AAE3B,SAAgB,mCAAmC,QAAyB;CAC1E,IAAI,OAAO,WAAW,GAAG,OAAO;CAEhC,MAAM,eAAe,OAAO,WAAW,OAAO,SAAS,CAAC;CACxD,OACE,gBAAgB,wBAAwB,gBAAgB;AAE5D;AAEA,SAAS,YAAY,OAA4B;CAC/C,IAAI,IAAI;CACR,OAAO,IAAI,MAAM,QAAQ;EACvB,MAAM,QAAQ,MAAM;EAEpB,IAAI,WAAW;EACf,IAAI,YAAY;EAGhB,IAAI,SAAS,KAAM;GAEjB,WAAW;GACX,YAAY;EACd,OAAO,KAAK,QAAQ,SAAU,KAAM;GAElC,WAAW;GACX,YAAY,QAAQ;GACpB,IAAI,SAAS,KAAM,OAAO;EAC5B,OAAO,KAAK,QAAQ,SAAU,KAAM;GAElC,WAAW;GACX,YAAY,QAAQ;EACtB,OAAO,KAAK,QAAQ,SAAU,KAAM;GAElC,WAAW;GACX,YAAY,QAAQ;GACpB,IAAI,QAAQ,KAAM,OAAO;EAC3B,OAEE,OAAO;EAIT,IAAI,IAAI,WAAW,MAAM,QAAQ,OAAO;EAGxC,KAAK,IAAI,IAAI,GAAG,IAAI,UAAU,KAAK;GACjC,MAAM,OAAO,MAAM,IAAI;GACvB,IAAI,SAAS,KAAA,MAAc,OAAO,SAAU,KAAM,OAAO;GACzD,YAAa,aAAa,IAAM,OAAO;EACzC;EAGA,IAAI,aAAa,KAAK,YAAY,KAAM,OAAO;EAC/C,IAAI,aAAa,KAAK,YAAY,MAAO,OAAO;EAChD,IAAI,aAAa,KAAK,YAAY,OAAQ,OAAO;EAGjD,IAAI,aAAa,SAAU,aAAa,OAAQ,OAAO;EAGvD,IAAI,YAAY,SAAW,OAAO;EAGlC,KAAK;CACP;CACA,OAAO;AACT;AAEA,SAAgB,wBAAwB,OAA4B;CAClE,OACE,MAAM,UAAU,KAChB,MAAM,OAAO,OACb,MAAM,OAAO,OACb,MAAM,OAAO;AAEjB;AAIA,MAAM,cAAc,IAAI,YAAY,QAAQ;CAAE,OAAO;CAAO,WAAW;AAAK,CAAC;AAE7E,SAAgB,mBAAmB,KAAqC;CACtE,IAAI,CAAC,YAAY,GAAG,GAClB;CAEF,OAAO,YAAY,OAAO,GAAG;AAC/B;AAGA,SAAgB,mBAAmB,GAAe,GAAuB;CACvE,MAAM,MAAM,KAAK,IAAI,EAAE,QAAQ,EAAE,MAAM;CACvC,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,KACvB,IAAI,EAAE,OAAO,EAAE,IACb,OAAO,EAAE,KAAM,EAAE;CAGrB,OAAO,EAAE,SAAS,EAAE;AACtB"}

package/esm/BytePairEncodingCore.js

@@ -1,9 +1,9 @@
 import { DEFAULT_MERGE_CACHE_SIZE } from "./constants.js";
-import { compareUint8Arrays, isAscii, tryConvertToString } from "./utfUtil.js";
+import { compareUint8Arrays, isAscii, startsWithByteOrderMark, tryConvertToString } from "./utfUtil.js";
 import { escapeRegExp } from "./util.js";
 //#region src/BytePairEncodingCore.ts
 const emptyBuffer = /* @__PURE__ */ new Uint8Array(0);
-const decoder = new TextDecoder("utf8");
+const decoder = new TextDecoder("utf8", { ignoreBOM: true });
 var BytePairEncodingCore = class {
 	mergeableBytePairRankCount;
 	/**
@@ -42,6 +42,10 @@ var BytePairEncodingCore = class {
 			const byteArray = new Uint8Array(value);
 			binaryLookup.push([byteArray, rank]);
 			this.bytePairNonUtfRankDecoder.set(rank, byteArray);
+			if (startsWithByteOrderMark(byteArray)) {
+				const asString = tryConvertToString(byteArray);
+				if (asString !== void 0) this.bytePairStringRankEncoder.set(asString, rank);
+			}
 		});
 		this.bytePairNonUtfSortedEncoder = binaryLookup.sort((a, b) => compareUint8Arrays(a[0], b[0]));
 		this.specialTokensEncoder = specialTokensEncoder ?? /* @__PURE__ */ new Map();

package/esm/BytePairEncodingCore.js.map

@@ -1 +1 @@
-{"version":3,"file":"BytePairEncodingCore.js","names":[],"sources":["../src/BytePairEncodingCore.ts"],"sourcesContent":["import { DEFAULT_MERGE_CACHE_SIZE } from './constants.js'\nimport { compareUint8Arrays, isAscii, tryConvertToString } from './utfUtil.js'\nimport { escapeRegExp } from './util.js'\n\nexport type RawBytePairRanks = readonly (string | readonly number[])[]\n\nexport interface BytePairEncodingConfig {\n  bytePairRankDecoder: RawBytePairRanks\n  specialTokensEncoder?: Map<string, number>\n  tokenSplitRegex: RegExp\n  /**\n   * LRU cache for merged tokens pairs.\n   * Increasing this value should make encoding similar strings faster,\n   * but will consume more memory.\n   * @default 100000\n   */\n  mergeCacheSize?: number\n}\n\nconst emptyBuffer = new Uint8Array(0)\n\nexport const decoder = new TextDecoder('utf8')\n\nexport class BytePairEncodingCore {\n  readonly mergeableBytePairRankCount: number\n  /**\n   * an array where the index is the BPE rank,\n   * and the value is the string or the array of bytes that it decodes to\n   * it may contain holes if token is unused\n   */\n  private bytePairRankDecoder: RawBytePairRanks\n  private bytePairNonUtfRankDecoder = new Map<number, Uint8Array>()\n  private bytePairNonUtfSortedEncoder: readonly [Uint8Array, number][]\n  /**\n   * a reverse map of the bytePairRankDecoder,\n   * where the key is the string and the value is the rank\n   * values that cannot be represented as a string are present in `bytePairNonUtfSortedEncoder`\n   */\n  private bytePairStringRankEncoder: Map<string, number>\n  private tokenSplitRegex: RegExp\n  private specialTokensEncoder: Map<string, number>\n  private specialTokensDecoder: Map<number, string>\n  private specialTokenPatternRegex: RegExp\n  private textEncoder = new TextEncoder()\n  private mergeCache?: Map<string, number[]>\n  private mergeCacheSize: number\n\n  constructor({\n    bytePairRankDecoder,\n    specialTokensEncoder,\n    tokenSplitRegex,\n    mergeCacheSize = DEFAULT_MERGE_CACHE_SIZE,\n  }: BytePairEncodingConfig) {\n    this.bytePairRankDecoder = bytePairRankDecoder\n    this.bytePairStringRankEncoder = new Map<string, number>()\n    this.mergeCacheSize = mergeCacheSize\n    if (mergeCacheSize > 0) {\n      this.mergeCache = new Map()\n    }\n\n    // size without array holes (which may be present in the encoder)\n    this.mergeableBytePairRankCount = Object.keys(bytePairRankDecoder).length\n    const binaryLookup: [Uint8Array, number][] = []\n    // forEach skips array holes:\n    bytePairRankDecoder.forEach((value, rank) => {\n      if (typeof value === 'string') {\n        this.bytePairStringRankEncoder.set(value, rank)\n        return\n      }\n      const byteArray = new Uint8Array(value)\n      binaryLookup.push([byteArray, rank])\n      this.bytePairNonUtfRankDecoder.set(rank, byteArray)\n    })\n    this.bytePairNonUtfSortedEncoder = binaryLookup.sort((a, b) =>\n      compareUint8Arrays(a[0], b[0]),\n    )\n    this.specialTokensEncoder =\n      specialTokensEncoder ?? new Map<string, number>()\n    this.specialTokensDecoder = specialTokensEncoder\n      ? new Map([...specialTokensEncoder].map(([key, value]) => [value, key]))\n      : new Map<number, string>()\n    this.tokenSplitRegex = tokenSplitRegex\n\n    const escapedSpecialTokens = [...this.specialTokensEncoder.keys()].map(\n      escapeRegExp,\n    )\n    const allSpecialTokensRegex = escapedSpecialTokens.join('|')\n    try {\n      this.specialTokenPatternRegex = new RegExp(allSpecialTokensRegex, 'y')\n    } catch {\n      throw new Error('Invalid regular expression pattern.')\n    }\n  }\n\n  setMergeCacheSize(newSize: number): void {\n    if (this.mergeCacheSize === 0 && newSize > 0) {\n      this.mergeCache = new Map()\n    }\n    this.mergeCacheSize = newSize\n    if (newSize === 0) {\n      this.mergeCache = undefined\n    }\n  }\n\n  clearMergeCache(): void {\n    this.mergeCache?.clear()\n  }\n\n  *encodeNativeGenerator(\n    text: string,\n    allowedSpecial?: Set<string>,\n  ): Generator<number[], number, undefined> {\n    let startIndex = 0\n    let lastTokenLength = 0\n\n    while (true) {\n      const nextSpecialMatch = this.findNextSpecialToken(\n        text,\n        allowedSpecial,\n        startIndex,\n      )\n      const nextSpecialStartIndex = nextSpecialMatch?.[0]\n\n      const endIndex = nextSpecialStartIndex ?? text.length\n\n      const textBeforeSpecial =\n        startIndex === 0 && endIndex === text.length\n          ? text\n          : text.slice(startIndex, endIndex)\n\n      for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {\n        const token = this.getBpeRankFromString(match)\n        if (token !== undefined) {\n          lastTokenLength = 1\n          yield [token]\n\n          continue\n        }\n\n        const tokens = this.bytePairEncode(match)\n        lastTokenLength = tokens.length\n        yield tokens\n      }\n\n      if (nextSpecialStartIndex !== undefined) {\n        const specialToken = nextSpecialMatch![1]\n        const specialTokenValue = this.specialTokensEncoder.get(specialToken)\n        if (specialTokenValue === undefined) {\n          throw new Error(\n            `Special token \"${specialToken}\" is not in the special token encoder.`,\n          )\n        }\n        yield [specialTokenValue]\n        startIndex = nextSpecialStartIndex + specialToken.length\n        lastTokenLength = 1\n      } else {\n        break\n      }\n    }\n\n    return lastTokenLength\n  }\n\n  encodeNative(text: string, allowedSpecial?: Set<string>): number[] {\n    let startIndex = 0\n    const tokensArray: number[] = [] // Flat list to collect the tokens\n\n    while (true) {\n      const nextSpecialMatch = this.findNextSpecialToken(\n        text,\n        allowedSpecial,\n        startIndex,\n      )\n      const nextSpecialStartIndex = nextSpecialMatch?.[0]\n\n      const endIndex = nextSpecialStartIndex ?? text.length\n\n      const textBeforeSpecial =\n        startIndex === 0 && endIndex === text.length\n          ? text\n          : text.slice(startIndex, endIndex)\n\n      for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {\n        const token = this.getBpeRankFromString(match)\n        if (token !== undefined) {\n          tokensArray.push(token)\n\n          continue\n        }\n\n        const tokens = this.bytePairEncode(match)\n        tokensArray.push(...tokens)\n      }\n\n      if (nextSpecialStartIndex !== undefined) {\n        const specialToken = nextSpecialMatch![1]\n        const specialTokenValue = this.specialTokensEncoder.get(specialToken)\n        if (specialTokenValue === undefined) {\n          throw new Error(\n            `Special token \"${specialToken}\" is not in the special token encoder.`,\n          )\n        }\n        tokensArray.push(specialTokenValue)\n        startIndex = nextSpecialStartIndex + specialToken.length\n      } else {\n        break\n      }\n    }\n\n    return tokensArray\n  }\n\n  countNative(text: string, allowedSpecial?: Set<string>): number {\n    let startIndex = 0\n    let tokensCount = 0\n\n    while (true) {\n      const nextSpecialMatch = this.findNextSpecialToken(\n        text,\n        allowedSpecial,\n        startIndex,\n      )\n      const nextSpecialStartIndex = nextSpecialMatch?.[0]\n\n      const endIndex = nextSpecialStartIndex ?? text.length\n\n      const textBeforeSpecial =\n        startIndex === 0 && endIndex === text.length\n          ? text\n          : text.slice(startIndex, endIndex)\n\n      for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {\n        const token = this.getBpeRankFromString(match)\n        if (token !== undefined) {\n          tokensCount++\n\n          continue\n        }\n\n        const tokens = this.bytePairEncode(match)\n        tokensCount += tokens.length\n      }\n\n      if (nextSpecialStartIndex !== undefined) {\n        const specialToken = nextSpecialMatch![1]\n        const specialTokenValue = this.specialTokensEncoder.get(specialToken)\n        if (specialTokenValue === undefined) {\n          throw new Error(\n            `Special token \"${specialToken}\" is not in the special token encoder.`,\n          )\n        }\n        tokensCount++\n        startIndex = nextSpecialStartIndex + specialToken.length\n      } else {\n        break\n      }\n    }\n\n    return tokensCount\n  }\n\n  *decodeNativeGenerator(\n    tokens: Iterable<number>,\n  ): Generator<Uint8Array | string, void, void> {\n    for (const token of tokens) {\n      const tokenBytes = this.tryDecodeToken(token)\n      if (tokenBytes) {\n        yield tokenBytes\n      }\n    }\n  }\n\n  decodeNative(tokens: Iterable<number>): string {\n    let decoded = ''\n    let intBuffer = emptyBuffer\n\n    for (const token of tokens) {\n      const tokenBytes = this.tryDecodeToken(token)\n      if (tokenBytes === undefined) {\n        throw new Error(`Token ${token} is not in the byte pair encoder.`)\n      }\n      if (typeof tokenBytes === 'string') {\n        if (intBuffer !== emptyBuffer) {\n          decoded += decoder.decode(intBuffer, { stream: true })\n          intBuffer = emptyBuffer\n        }\n        decoded += tokenBytes\n      } else {\n        const newBuffer = new Uint8Array(intBuffer.length + tokenBytes.length)\n        newBuffer.set(intBuffer)\n        newBuffer.set(tokenBytes, intBuffer.length)\n        intBuffer = newBuffer\n      }\n    }\n\n    if (intBuffer !== emptyBuffer) {\n      decoded += decoder.decode(intBuffer, { stream: true })\n    }\n    return decoded\n  }\n\n  async *decodeNativeAsyncIterable(\n    tokens: AsyncIterable<number>,\n  ): AsyncGenerator<Uint8Array | string> {\n    for await (const token of tokens) {\n      const tokenBytesOrString = this.tryDecodeToken(token)\n      if (tokenBytesOrString) {\n        yield tokenBytesOrString\n      }\n    }\n  }\n\n  private getBpeRankFromString(key: string): number | undefined {\n    return this.bytePairStringRankEncoder.get(key)\n  }\n\n  private getBpeRankFromStringOrThrow(key: string): number {\n    const value = this.getBpeRankFromString(key)\n    if (value === undefined) {\n      throw new Error(\n        `The byte-pair encoding does not contain a value for: ${key}`,\n      )\n    }\n    return value\n  }\n\n  private getBpeRankFromBytes(key: Uint8Array): number | undefined {\n    const keyAsString = tryConvertToString(key)\n\n    if (keyAsString !== undefined) {\n      return this.getBpeRankFromString(keyAsString)\n    }\n\n    // Perform binary search on the binary keys\n    const index = this.binarySearch(key)\n    if (index !== -1) {\n      return this.bytePairNonUtfSortedEncoder[index]![1]\n    }\n\n    return undefined\n  }\n\n  private getBpeRankFromBytesOrThrow(key: Uint8Array): number {\n    const value = this.getBpeRankFromBytes(key)\n    if (value === undefined) {\n      throw new Error(\n        `The byte-pair encoding does not contain a value for: ${key.toString()}`,\n      )\n    }\n    return value\n  }\n\n  // Binary search on the binary keys\n  private binarySearch(key: Uint8Array): number {\n    let low = 0\n    let high = this.bytePairNonUtfSortedEncoder.length - 1\n\n    while (low <= high) {\n      const mid = (low + high) >>> 1\n      const midKey = this.bytePairNonUtfSortedEncoder[mid]![0]\n      let cmp = 0\n      const maxLength = Math.min(midKey.length, key.length)\n      for (let i = 0; i < maxLength; i++) {\n        cmp = midKey[i]! - key[i]!\n        if (cmp !== 0) break\n      }\n      if (cmp === 0) {\n        cmp = midKey.length - key.length\n      }\n      if (cmp === 0) {\n        return mid\n      }\n      if (cmp < 0) {\n        low = mid + 1\n      } else {\n        high = mid - 1\n      }\n    }\n    return -1\n  }\n\n  private findNextSpecialToken(\n    text: string,\n    allowedSpecial: Set<string> | undefined,\n    startIndex: number,\n  ): [startIndex: number, token: string] | undefined {\n    let searchIndex = startIndex\n\n    while (true) {\n      this.specialTokenPatternRegex.lastIndex = searchIndex\n      const nextSpecialMatch = this.specialTokenPatternRegex.exec(text)\n\n      if (!nextSpecialMatch) {\n        return undefined\n      }\n\n      const specialToken = nextSpecialMatch[0]\n\n      if (allowedSpecial?.has(specialToken)) {\n        const specialTokenStartIndex = nextSpecialMatch.index + searchIndex\n        return [specialTokenStartIndex, specialToken]\n      }\n\n      searchIndex = nextSpecialMatch.index + searchIndex + 1\n    }\n  }\n\n  private tryDecodeToken(tokenRank: number): Uint8Array | string | undefined {\n    const value = this.bytePairRankDecoder[tokenRank]\n    if (typeof value === 'string') {\n      return value\n    }\n    if (typeof value === 'object') {\n      const fromBinary = this.bytePairNonUtfRankDecoder.get(tokenRank)\n      if (fromBinary) {\n        return fromBinary\n      }\n    }\n    return this.specialTokensDecoder.get(tokenRank)\n  }\n\n  private addToMergeCache(key: string, value: number[]): void {\n    if (!this.mergeCache) return\n\n    if (this.mergeCache.size >= this.mergeCacheSize) {\n      // Remove least recently used item (first item)\n      const firstKey = this.mergeCache.keys().next().value!\n      this.mergeCache.delete(firstKey)\n    }\n    this.mergeCache.set(key, value)\n  }\n\n  private bytePairEncode(input: string): number[] {\n    if (input.length === 1 && isAscii(input.codePointAt(0)!)) {\n      return [this.getBpeRankFromStringOrThrow(input)]\n    }\n\n    if (this.mergeCache?.has(input)) {\n      const result = this.mergeCache.get(input)!\n      // Move to end to mark as recently used\n      this.mergeCache.delete(input)\n      this.mergeCache.set(input, result)\n      return result\n    }\n\n    const inputBytes = this.textEncoder.encode(input)\n    const result = this.bytePairMerge(inputBytes)\n    this.addToMergeCache(input, result)\n    return result\n  }\n\n  private bytePairMerge(\n    // Input array of bytes to process\n    piece: Uint8Array,\n  ): number[] {\n    const { length } = piece\n    // Partitions form a linked list over byte offsets: a live partition starting at\n    // offset 'i' spans [i, next[i]). 'pairRanks[i]' is the rank of merging it with\n    // the partition after it. Merges come off a min-heap ordered by (rank, offset),\n    // which picks the same pair as scanning for the lowest rank, leftmost first,\n    // without rescanning the piece after every merge.\n    const next = new Int32Array(length + 1)\n    const previous = new Int32Array(length + 1)\n    const pairRanks = new Float64Array(length + 1)\n    const live = new Uint8Array(length + 1)\n\n    const getPairRank = (startIndex: number): number => {\n      const middle = next[startIndex]!\n      if (middle >= length) return Number.POSITIVE_INFINITY\n      const rank = this.getBpeRankFromBytes(\n        piece.subarray(startIndex, next[middle]!),\n      )\n      return rank ?? Number.POSITIVE_INFINITY\n    }\n\n    // Heap entries are packed as [rank, startIndex] pairs in a flat array\n    const heap: number[] = []\n    const isLower = (a: number, b: number) =>\n      heap[a]! < heap[b]! ||\n      (heap[a] === heap[b] && heap[a + 1]! < heap[b + 1]!)\n    const swap = (a: number, b: number) => {\n      const rank = heap[a]!\n      const startIndex = heap[a + 1]!\n      heap[a] = heap[b]!\n      heap[a + 1] = heap[b + 1]!\n      heap[b] = rank\n      heap[b + 1] = startIndex\n    }\n    const push = (rank: number, startIndex: number) => {\n      heap.push(rank, startIndex)\n      let index = heap.length - 2\n      while (index > 0) {\n        const parent = (((index >> 1) - 1) >> 1) << 1\n        if (!isLower(index, parent)) break\n        swap(index, parent)\n        index = parent\n      }\n    }\n    const pop = () => {\n      const last = heap.length - 2\n      swap(0, last)\n      const rank = heap[last]!\n      const startIndex = heap[last + 1]!\n      heap.length = last\n      let index = 0\n      while (true) {\n        const left = index * 2 + 2\n        const right = left + 2\n        let lowest = index\n        if (left < heap.length && isLower(left, lowest)) lowest = left\n        if (right < heap.length && isLower(right, lowest)) lowest = right\n        if (lowest === index) break\n        swap(index, lowest)\n        index = lowest\n      }\n      return [rank, startIndex] as const\n    }\n\n    for (let i = 0; i <= length; i++) {\n      next[i] = i + 1\n      previous[i] = i - 1\n      live[i] = 1\n    }\n    for (let i = 0; i < length - 1; i++) {\n      pairRanks[i] = getPairRank(i)\n      if (pairRanks[i] !== Number.POSITIVE_INFINITY) push(pairRanks[i]!, i)\n    }\n\n    // Iteratively merge byte pairs until no more useful merges can be done\n    while (heap.length > 0) {\n      const [rank, startIndex] = pop()\n      // Skip entries made stale by an earlier merge\n      if (!live[startIndex] || pairRanks[startIndex] !== rank) continue\n\n      // Merge the partition at 'startIndex' with the one after it\n      const merged = next[startIndex]!\n      live[merged] = 0\n      next[startIndex] = next[merged]!\n      previous[next[merged]!] = startIndex\n\n      pairRanks[startIndex] = getPairRank(startIndex)\n      if (pairRanks[startIndex] !== Number.POSITIVE_INFINITY) {\n        push(pairRanks[startIndex]!, startIndex)\n      }\n\n      const before = previous[startIndex]!\n      if (before >= 0) {\n        pairRanks[before] = getPairRank(before)\n        if (pairRanks[before] !== Number.POSITIVE_INFINITY) {\n          push(pairRanks[before]!, before)\n        }\n      }\n    }\n\n    // Create the final output by applying the transform function to each partitioned range\n    const output: number[] = []\n    for (let i = 0; i < length; i = next[i]!) {\n      output.push(this.getBpeRankFromBytesOrThrow(piece.subarray(i, next[i]!)))\n    }\n    return output\n  }\n}\n"],"mappings":";;;;AAmBA,MAAM,8BAAc,IAAI,WAAW,CAAC;AAEpC,MAAa,UAAU,IAAI,YAAY,MAAM;AAE7C,IAAa,uBAAb,MAAkC;CAChC;;;;;;CAMA;CACA,4CAAoC,IAAI,IAAwB;CAChE;;;;;;CAMA;CACA;CACA;CACA;CACA;CACA,cAAsB,IAAI,YAAY;CACtC;CACA;CAEA,YAAY,EACV,qBACA,sBACA,iBACA,iBAAiB,4BACQ;EACzB,KAAK,sBAAsB;EAC3B,KAAK,4CAA4B,IAAI,IAAoB;EACzD,KAAK,iBAAiB;EACtB,IAAI,iBAAiB,GACnB,KAAK,6BAAa,IAAI,IAAI;EAI5B,KAAK,6BAA6B,OAAO,KAAK,mBAAmB,CAAC,CAAC;EACnE,MAAM,eAAuC,CAAC;EAE9C,oBAAoB,SAAS,OAAO,SAAS;GAC3C,IAAI,OAAO,UAAU,UAAU;IAC7B,KAAK,0BAA0B,IAAI,OAAO,IAAI;IAC9C;GACF;GACA,MAAM,YAAY,IAAI,WAAW,KAAK;GACtC,aAAa,KAAK,CAAC,WAAW,IAAI,CAAC;GACnC,KAAK,0BAA0B,IAAI,MAAM,SAAS;EACpD,CAAC;EACD,KAAK,8BAA8B,aAAa,MAAM,GAAG,MACvD,mBAAmB,EAAE,IAAI,EAAE,EAAE,CAC/B;EACA,KAAK,uBACH,wCAAwB,IAAI,IAAoB;EAClD,KAAK,uBAAuB,uBACxB,IAAI,IAAI,CAAC,GAAG,oBAAoB,CAAC,CAAC,KAAK,CAAC,KAAK,WAAW,CAAC,OAAO,GAAG,CAAC,CAAC,oBACrE,IAAI,IAAoB;EAC5B,KAAK,kBAAkB;EAKvB,MAAM,wBAHuB,CAAC,GAAG,KAAK,qBAAqB,KAAK,CAAC,CAAC,CAAC,IACjE,YAE+C,CAAC,CAAC,KAAK,GAAG;EAC3D,IAAI;GACF,KAAK,2BAA2B,IAAI,OAAO,uBAAuB,GAAG;EACvE,QAAQ;GACN,MAAM,IAAI,MAAM,qCAAqC;EACvD;CACF;CAEA,kBAAkB,SAAuB;EACvC,IAAI,KAAK,mBAAmB,KAAK,UAAU,GACzC,KAAK,6BAAa,IAAI,IAAI;EAE5B,KAAK,iBAAiB;EACtB,IAAI,YAAY,GACd,KAAK,aAAa,KAAA;CAEtB;CAEA,kBAAwB;EACtB,KAAK,YAAY,MAAM;CACzB;CAEA,CAAC,sBACC,MACA,gBACwC;EACxC,IAAI,aAAa;EACjB,IAAI,kBAAkB;EAEtB,OAAO,MAAM;GACX,MAAM,mBAAmB,KAAK,qBAC5B,MACA,gBACA,UACF;GACA,MAAM,wBAAwB,mBAAmB;GAEjD,MAAM,WAAW,yBAAyB,KAAK;GAE/C,MAAM,oBACJ,eAAe,KAAK,aAAa,KAAK,SAClC,OACA,KAAK,MAAM,YAAY,QAAQ;GAErC,KAAK,MAAM,CAAC,UAAU,kBAAkB,SAAS,KAAK,eAAe,GAAG;IACtE,MAAM,QAAQ,KAAK,qBAAqB,KAAK;IAC7C,IAAI,UAAU,KAAA,GAAW;KACvB,kBAAkB;KAClB,MAAM,CAAC,KAAK;KAEZ;IACF;IAEA,MAAM,SAAS,KAAK,eAAe,KAAK;IACxC,kBAAkB,OAAO;IACzB,MAAM;GACR;GAEA,IAAI,0BAA0B,KAAA,GAAW;IACvC,MAAM,eAAe,iBAAkB;IACvC,MAAM,oBAAoB,KAAK,qBAAqB,IAAI,YAAY;IACpE,IAAI,sBAAsB,KAAA,GACxB,MAAM,IAAI,MACR,kBAAkB,aAAa,uCACjC;IAEF,MAAM,CAAC,iBAAiB;IACxB,aAAa,wBAAwB,aAAa;IAClD,kBAAkB;GACpB,OACE;EAEJ;EAEA,OAAO;CACT;CAEA,aAAa,MAAc,gBAAwC;EACjE,IAAI,aAAa;EACjB,MAAM,cAAwB,CAAC;EAE/B,OAAO,MAAM;GACX,MAAM,mBAAmB,KAAK,qBAC5B,MACA,gBACA,UACF;GACA,MAAM,wBAAwB,mBAAmB;GAEjD,MAAM,WAAW,yBAAyB,KAAK;GAE/C,MAAM,oBACJ,eAAe,KAAK,aAAa,KAAK,SAClC,OACA,KAAK,MAAM,YAAY,QAAQ;GAErC,KAAK,MAAM,CAAC,UAAU,kBAAkB,SAAS,KAAK,eAAe,GAAG;IACtE,MAAM,QAAQ,KAAK,qBAAqB,KAAK;IAC7C,IAAI,UAAU,KAAA,GAAW;KACvB,YAAY,KAAK,KAAK;KAEtB;IACF;IAEA,MAAM,SAAS,KAAK,eAAe,KAAK;IACxC,YAAY,KAAK,GAAG,MAAM;GAC5B;GAEA,IAAI,0BAA0B,KAAA,GAAW;IACvC,MAAM,eAAe,iBAAkB;IACvC,MAAM,oBAAoB,KAAK,qBAAqB,IAAI,YAAY;IACpE,IAAI,sBAAsB,KAAA,GACxB,MAAM,IAAI,MACR,kBAAkB,aAAa,uCACjC;IAEF,YAAY,KAAK,iBAAiB;IAClC,aAAa,wBAAwB,aAAa;GACpD,OACE;EAEJ;EAEA,OAAO;CACT;CAEA,YAAY,MAAc,gBAAsC;EAC9D,IAAI,aAAa;EACjB,IAAI,cAAc;EAElB,OAAO,MAAM;GACX,MAAM,mBAAmB,KAAK,qBAC5B,MACA,gBACA,UACF;GACA,MAAM,wBAAwB,mBAAmB;GAEjD,MAAM,WAAW,yBAAyB,KAAK;GAE/C,MAAM,oBACJ,eAAe,KAAK,aAAa,KAAK,SAClC,OACA,KAAK,MAAM,YAAY,QAAQ;GAErC,KAAK,MAAM,CAAC,UAAU,kBAAkB,SAAS,KAAK,eAAe,GAAG;IAEtE,IADc,KAAK,qBAAqB,KAChC,MAAM,KAAA,GAAW;KACvB;KAEA;IACF;IAEA,MAAM,SAAS,KAAK,eAAe,KAAK;IACxC,eAAe,OAAO;GACxB;GAEA,IAAI,0BAA0B,KAAA,GAAW;IACvC,MAAM,eAAe,iBAAkB;IAEvC,IAD0B,KAAK,qBAAqB,IAAI,YACpC,MAAM,KAAA,GACxB,MAAM,IAAI,MACR,kBAAkB,aAAa,uCACjC;IAEF;IACA,aAAa,wBAAwB,aAAa;GACpD,OACE;EAEJ;EAEA,OAAO;CACT;CAEA,CAAC,sBACC,QAC4C;EAC5C,KAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,aAAa,KAAK,eAAe,KAAK;GAC5C,IAAI,YACF,MAAM;EAEV;CACF;CAEA,aAAa,QAAkC;EAC7C,IAAI,UAAU;EACd,IAAI,YAAY;EAEhB,KAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,aAAa,KAAK,eAAe,KAAK;GAC5C,IAAI,eAAe,KAAA,GACjB,MAAM,IAAI,MAAM,SAAS,MAAM,kCAAkC;GAEnE,IAAI,OAAO,eAAe,UAAU;IAClC,IAAI,cAAc,aAAa;KAC7B,WAAW,QAAQ,OAAO,WAAW,EAAE,QAAQ,KAAK,CAAC;KACrD,YAAY;IACd;IACA,WAAW;GACb,OAAO;IACL,MAAM,YAAY,IAAI,WAAW,UAAU,SAAS,WAAW,MAAM;IACrE,UAAU,IAAI,SAAS;IACvB,UAAU,IAAI,YAAY,UAAU,MAAM;IAC1C,YAAY;GACd;EACF;EAEA,IAAI,cAAc,aAChB,WAAW,QAAQ,OAAO,WAAW,EAAE,QAAQ,KAAK,CAAC;EAEvD,OAAO;CACT;CAEA,OAAO,0BACL,QACqC;EACrC,WAAW,MAAM,SAAS,QAAQ;GAChC,MAAM,qBAAqB,KAAK,eAAe,KAAK;GACpD,IAAI,oBACF,MAAM;EAEV;CACF;CAEA,qBAA6B,KAAiC;EAC5D,OAAO,KAAK,0BAA0B,IAAI,GAAG;CAC/C;CAEA,4BAAoC,KAAqB;EACvD,MAAM,QAAQ,KAAK,qBAAqB,GAAG;EAC3C,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,MACR,wDAAwD,KAC1D;EAEF,OAAO;CACT;CAEA,oBAA4B,KAAqC;EAC/D,MAAM,cAAc,mBAAmB,GAAG;EAE1C,IAAI,gBAAgB,KAAA,GAClB,OAAO,KAAK,qBAAqB,WAAW;EAI9C,MAAM,QAAQ,KAAK,aAAa,GAAG;EACnC,IAAI,UAAU,IACZ,OAAO,KAAK,4BAA4B,MAAM,CAAE;CAIpD;CAEA,2BAAmC,KAAyB;EAC1D,MAAM,QAAQ,KAAK,oBAAoB,GAAG;EAC1C,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,MACR,wDAAwD,IAAI,SAAS,GACvE;EAEF,OAAO;CACT;CAGA,aAAqB,KAAyB;EAC5C,IAAI,MAAM;EACV,IAAI,OAAO,KAAK,4BAA4B,SAAS;EAErD,OAAO,OAAO,MAAM;GAClB,MAAM,MAAO,MAAM,SAAU;GAC7B,MAAM,SAAS,KAAK,4BAA4B,IAAI,CAAE;GACtD,IAAI,MAAM;GACV,MAAM,YAAY,KAAK,IAAI,OAAO,QAAQ,IAAI,MAAM;GACpD,KAAK,IAAI,IAAI,GAAG,IAAI,WAAW,KAAK;IAClC,MAAM,OAAO,KAAM,IAAI;IACvB,IAAI,QAAQ,GAAG;GACjB;GACA,IAAI,QAAQ,GACV,MAAM,OAAO,SAAS,IAAI;GAE5B,IAAI,QAAQ,GACV,OAAO;GAET,IAAI,MAAM,GACR,MAAM,MAAM;QAEZ,OAAO,MAAM;EAEjB;EACA,OAAO;CACT;CAEA,qBACE,MACA,gBACA,YACiD;EACjD,IAAI,cAAc;EAElB,OAAO,MAAM;GACX,KAAK,yBAAyB,YAAY;GAC1C,MAAM,mBAAmB,KAAK,yBAAyB,KAAK,IAAI;GAEhE,IAAI,CAAC,kBACH;GAGF,MAAM,eAAe,iBAAiB;GAEtC,IAAI,gBAAgB,IAAI,YAAY,GAElC,OAAO,CADwB,iBAAiB,QAAQ,aACxB,YAAY;GAG9C,cAAc,iBAAiB,QAAQ,cAAc;EACvD;CACF;CAEA,eAAuB,WAAoD;EACzE,MAAM,QAAQ,KAAK,oBAAoB;EACvC,IAAI,OAAO,UAAU,UACnB,OAAO;EAET,IAAI,OAAO,UAAU,UAAU;GAC7B,MAAM,aAAa,KAAK,0BAA0B,IAAI,SAAS;GAC/D,IAAI,YACF,OAAO;EAEX;EACA,OAAO,KAAK,qBAAqB,IAAI,SAAS;CAChD;CAEA,gBAAwB,KAAa,OAAuB;EAC1D,IAAI,CAAC,KAAK,YAAY;EAEtB,IAAI,KAAK,WAAW,QAAQ,KAAK,gBAAgB;GAE/C,MAAM,WAAW,KAAK,WAAW,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC;GAC/C,KAAK,WAAW,OAAO,QAAQ;EACjC;EACA,KAAK,WAAW,IAAI,KAAK,KAAK;CAChC;CAEA,eAAuB,OAAyB;EAC9C,IAAI,MAAM,WAAW,KAAK,QAAQ,MAAM,YAAY,CAAC,CAAE,GACrD,OAAO,CAAC,KAAK,4BAA4B,KAAK,CAAC;EAGjD,IAAI,KAAK,YAAY,IAAI,KAAK,GAAG;GAC/B,MAAM,SAAS,KAAK,WAAW,IAAI,KAAK;GAExC,KAAK,WAAW,OAAO,KAAK;GAC5B,KAAK,WAAW,IAAI,OAAO,MAAM;GACjC,OAAO;EACT;EAEA,MAAM,aAAa,KAAK,YAAY,OAAO,KAAK;EAChD,MAAM,SAAS,KAAK,cAAc,UAAU;EAC5C,KAAK,gBAAgB,OAAO,MAAM;EAClC,OAAO;CACT;CAEA,cAEE,OACU;EACV,MAAM,EAAE,WAAW;EAMnB,MAAM,OAAO,IAAI,WAAW,SAAS,CAAC;EACtC,MAAM,WAAW,IAAI,WAAW,SAAS,CAAC;EAC1C,MAAM,YAAY,IAAI,aAAa,SAAS,CAAC;EAC7C,MAAM,OAAO,IAAI,WAAW,SAAS,CAAC;EAEtC,MAAM,eAAe,eAA+B;GAClD,MAAM,SAAS,KAAK;GACpB,IAAI,UAAU,QAAQ,OAAO,OAAO;GAIpC,OAHa,KAAK,oBAChB,MAAM,SAAS,YAAY,KAAK,OAAQ,CAEhC,KAAK,OAAO;EACxB;EAGA,MAAM,OAAiB,CAAC;EACxB,MAAM,WAAW,GAAW,MAC1B,KAAK,KAAM,KAAK,MACf,KAAK,OAAO,KAAK,MAAM,KAAK,IAAI,KAAM,KAAK,IAAI;EAClD,MAAM,QAAQ,GAAW,MAAc;GACrC,MAAM,OAAO,KAAK;GAClB,MAAM,aAAa,KAAK,IAAI;GAC5B,KAAK,KAAK,KAAK;GACf,KAAK,IAAI,KAAK,KAAK,IAAI;GACvB,KAAK,KAAK;GACV,KAAK,IAAI,KAAK;EAChB;EACA,MAAM,QAAQ,MAAc,eAAuB;GACjD,KAAK,KAAK,MAAM,UAAU;GAC1B,IAAI,QAAQ,KAAK,SAAS;GAC1B,OAAO,QAAQ,GAAG;IAChB,MAAM,UAAY,SAAS,KAAK,KAAM,KAAM;IAC5C,IAAI,CAAC,QAAQ,OAAO,MAAM,GAAG;IAC7B,KAAK,OAAO,MAAM;IAClB,QAAQ;GACV;EACF;EACA,MAAM,YAAY;GAChB,MAAM,OAAO,KAAK,SAAS;GAC3B,KAAK,GAAG,IAAI;GACZ,MAAM,OAAO,KAAK;GAClB,MAAM,aAAa,KAAK,OAAO;GAC/B,KAAK,SAAS;GACd,IAAI,QAAQ;GACZ,OAAO,MAAM;IACX,MAAM,OAAO,QAAQ,IAAI;IACzB,MAAM,QAAQ,OAAO;IACrB,IAAI,SAAS;IACb,IAAI,OAAO,KAAK,UAAU,QAAQ,MAAM,MAAM,GAAG,SAAS;IAC1D,IAAI,QAAQ,KAAK,UAAU,QAAQ,OAAO,MAAM,GAAG,SAAS;IAC5D,IAAI,WAAW,OAAO;IACtB,KAAK,OAAO,MAAM;IAClB,QAAQ;GACV;GACA,OAAO,CAAC,MAAM,UAAU;EAC1B;EAEA,KAAK,IAAI,IAAI,GAAG,KAAK,QAAQ,KAAK;GAChC,KAAK,KAAK,IAAI;GACd,SAAS,KAAK,IAAI;GAClB,KAAK,KAAK;EACZ;EACA,KAAK,IAAI,IAAI,GAAG,IAAI,SAAS,GAAG,KAAK;GACnC,UAAU,KAAK,YAAY,CAAC;GAC5B,IAAI,UAAU,OAAO,OAAO,mBAAmB,KAAK,UAAU,IAAK,CAAC;EACtE;EAGA,OAAO,KAAK,SAAS,GAAG;GACtB,MAAM,CAAC,MAAM,cAAc,IAAI;GAE/B,IAAI,CAAC,KAAK,eAAe,UAAU,gBAAgB,MAAM;GAGzD,MAAM,SAAS,KAAK;GACpB,KAAK,UAAU;GACf,KAAK,cAAc,KAAK;GACxB,SAAS,KAAK,WAAY;GAE1B,UAAU,cAAc,YAAY,UAAU;GAC9C,IAAI,UAAU,gBAAgB,OAAO,mBACnC,KAAK,UAAU,aAAc,UAAU;GAGzC,MAAM,SAAS,SAAS;GACxB,IAAI,UAAU,GAAG;IACf,UAAU,UAAU,YAAY,MAAM;IACtC,IAAI,UAAU,YAAY,OAAO,mBAC/B,KAAK,UAAU,SAAU,MAAM;GAEnC;EACF;EAGA,MAAM,SAAmB,CAAC;EAC1B,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,IAAI,KAAK,IACnC,OAAO,KAAK,KAAK,2BAA2B,MAAM,SAAS,GAAG,KAAK,EAAG,CAAC,CAAC;EAE1E,OAAO;CACT;AACF"}
+{"version":3,"file":"BytePairEncodingCore.js","names":[],"sources":["../src/BytePairEncodingCore.ts"],"sourcesContent":["import { DEFAULT_MERGE_CACHE_SIZE } from './constants.js'\nimport {\n  compareUint8Arrays,\n  isAscii,\n  startsWithByteOrderMark,\n  tryConvertToString,\n} from './utfUtil.js'\nimport { escapeRegExp } from './util.js'\n\nexport type RawBytePairRanks = readonly (string | readonly number[])[]\n\nexport interface BytePairEncodingConfig {\n  bytePairRankDecoder: RawBytePairRanks\n  specialTokensEncoder?: Map<string, number>\n  tokenSplitRegex: RegExp\n  /**\n   * LRU cache for merged tokens pairs.\n   * Increasing this value should make encoding similar strings faster,\n   * but will consume more memory.\n   * @default 100000\n   */\n  mergeCacheSize?: number\n}\n\nconst emptyBuffer = new Uint8Array(0)\n\n// `ignoreBOM: true` keeps a leading U+FEFF in the decoded output.\nexport const decoder = new TextDecoder('utf8', { ignoreBOM: true })\n\nexport class BytePairEncodingCore {\n  readonly mergeableBytePairRankCount: number\n  /**\n   * an array where the index is the BPE rank,\n   * and the value is the string or the array of bytes that it decodes to\n   * it may contain holes if token is unused\n   */\n  private bytePairRankDecoder: RawBytePairRanks\n  private bytePairNonUtfRankDecoder = new Map<number, Uint8Array>()\n  private bytePairNonUtfSortedEncoder: readonly [Uint8Array, number][]\n  /**\n   * a reverse map of the bytePairRankDecoder,\n   * where the key is the string and the value is the rank\n   * values that cannot be represented as a string are present in `bytePairNonUtfSortedEncoder`\n   */\n  private bytePairStringRankEncoder: Map<string, number>\n  private tokenSplitRegex: RegExp\n  private specialTokensEncoder: Map<string, number>\n  private specialTokensDecoder: Map<number, string>\n  private specialTokenPatternRegex: RegExp\n  private textEncoder = new TextEncoder()\n  private mergeCache?: Map<string, number[]>\n  private mergeCacheSize: number\n\n  constructor({\n    bytePairRankDecoder,\n    specialTokensEncoder,\n    tokenSplitRegex,\n    mergeCacheSize = DEFAULT_MERGE_CACHE_SIZE,\n  }: BytePairEncodingConfig) {\n    this.bytePairRankDecoder = bytePairRankDecoder\n    this.bytePairStringRankEncoder = new Map<string, number>()\n    this.mergeCacheSize = mergeCacheSize\n    if (mergeCacheSize > 0) {\n      this.mergeCache = new Map()\n    }\n\n    // size without array holes (which may be present in the encoder)\n    this.mergeableBytePairRankCount = Object.keys(bytePairRankDecoder).length\n    const binaryLookup: [Uint8Array, number][] = []\n    // forEach skips array holes:\n    bytePairRankDecoder.forEach((value, rank) => {\n      if (typeof value === 'string') {\n        this.bytePairStringRankEncoder.set(value, rank)\n        return\n      }\n      const byteArray = new Uint8Array(value)\n      binaryLookup.push([byteArray, rank])\n      this.bytePairNonUtfRankDecoder.set(rank, byteArray)\n      if (startsWithByteOrderMark(byteArray)) {\n        // The rank tables keep tokens that start with a UTF-8 byte order mark\n        // (EF BB BF) as raw bytes, because a plain decode would strip the mark.\n        // They are still valid UTF-8, so they must also be reachable through\n        // the string lookup that getBpeRankFromBytes uses for valid UTF-8.\n        const asString = tryConvertToString(byteArray)\n        if (asString !== undefined) {\n          this.bytePairStringRankEncoder.set(asString, rank)\n        }\n      }\n    })\n    this.bytePairNonUtfSortedEncoder = binaryLookup.sort((a, b) =>\n      compareUint8Arrays(a[0], b[0]),\n    )\n    this.specialTokensEncoder =\n      specialTokensEncoder ?? new Map<string, number>()\n    this.specialTokensDecoder = specialTokensEncoder\n      ? new Map([...specialTokensEncoder].map(([key, value]) => [value, key]))\n      : new Map<number, string>()\n    this.tokenSplitRegex = tokenSplitRegex\n\n    const escapedSpecialTokens = [...this.specialTokensEncoder.keys()].map(\n      escapeRegExp,\n    )\n    const allSpecialTokensRegex = escapedSpecialTokens.join('|')\n    try {\n      this.specialTokenPatternRegex = new RegExp(allSpecialTokensRegex, 'y')\n    } catch {\n      throw new Error('Invalid regular expression pattern.')\n    }\n  }\n\n  setMergeCacheSize(newSize: number): void {\n    if (this.mergeCacheSize === 0 && newSize > 0) {\n      this.mergeCache = new Map()\n    }\n    this.mergeCacheSize = newSize\n    if (newSize === 0) {\n      this.mergeCache = undefined\n    }\n  }\n\n  clearMergeCache(): void {\n    this.mergeCache?.clear()\n  }\n\n  *encodeNativeGenerator(\n    text: string,\n    allowedSpecial?: Set<string>,\n  ): Generator<number[], number, undefined> {\n    let startIndex = 0\n    let lastTokenLength = 0\n\n    while (true) {\n      const nextSpecialMatch = this.findNextSpecialToken(\n        text,\n        allowedSpecial,\n        startIndex,\n      )\n      const nextSpecialStartIndex = nextSpecialMatch?.[0]\n\n      const endIndex = nextSpecialStartIndex ?? text.length\n\n      const textBeforeSpecial =\n        startIndex === 0 && endIndex === text.length\n          ? text\n          : text.slice(startIndex, endIndex)\n\n      for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {\n        const token = this.getBpeRankFromString(match)\n        if (token !== undefined) {\n          lastTokenLength = 1\n          yield [token]\n\n          continue\n        }\n\n        const tokens = this.bytePairEncode(match)\n        lastTokenLength = tokens.length\n        yield tokens\n      }\n\n      if (nextSpecialStartIndex !== undefined) {\n        const specialToken = nextSpecialMatch![1]\n        const specialTokenValue = this.specialTokensEncoder.get(specialToken)\n        if (specialTokenValue === undefined) {\n          throw new Error(\n            `Special token \"${specialToken}\" is not in the special token encoder.`,\n          )\n        }\n        yield [specialTokenValue]\n        startIndex = nextSpecialStartIndex + specialToken.length\n        lastTokenLength = 1\n      } else {\n        break\n      }\n    }\n\n    return lastTokenLength\n  }\n\n  encodeNative(text: string, allowedSpecial?: Set<string>): number[] {\n    let startIndex = 0\n    const tokensArray: number[] = [] // Flat list to collect the tokens\n\n    while (true) {\n      const nextSpecialMatch = this.findNextSpecialToken(\n        text,\n        allowedSpecial,\n        startIndex,\n      )\n      const nextSpecialStartIndex = nextSpecialMatch?.[0]\n\n      const endIndex = nextSpecialStartIndex ?? text.length\n\n      const textBeforeSpecial =\n        startIndex === 0 && endIndex === text.length\n          ? text\n          : text.slice(startIndex, endIndex)\n\n      for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {\n        const token = this.getBpeRankFromString(match)\n        if (token !== undefined) {\n          tokensArray.push(token)\n\n          continue\n        }\n\n        const tokens = this.bytePairEncode(match)\n        tokensArray.push(...tokens)\n      }\n\n      if (nextSpecialStartIndex !== undefined) {\n        const specialToken = nextSpecialMatch![1]\n        const specialTokenValue = this.specialTokensEncoder.get(specialToken)\n        if (specialTokenValue === undefined) {\n          throw new Error(\n            `Special token \"${specialToken}\" is not in the special token encoder.`,\n          )\n        }\n        tokensArray.push(specialTokenValue)\n        startIndex = nextSpecialStartIndex + specialToken.length\n      } else {\n        break\n      }\n    }\n\n    return tokensArray\n  }\n\n  countNative(text: string, allowedSpecial?: Set<string>): number {\n    let startIndex = 0\n    let tokensCount = 0\n\n    while (true) {\n      const nextSpecialMatch = this.findNextSpecialToken(\n        text,\n        allowedSpecial,\n        startIndex,\n      )\n      const nextSpecialStartIndex = nextSpecialMatch?.[0]\n\n      const endIndex = nextSpecialStartIndex ?? text.length\n\n      const textBeforeSpecial =\n        startIndex === 0 && endIndex === text.length\n          ? text\n          : text.slice(startIndex, endIndex)\n\n      for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {\n        const token = this.getBpeRankFromString(match)\n        if (token !== undefined) {\n          tokensCount++\n\n          continue\n        }\n\n        const tokens = this.bytePairEncode(match)\n        tokensCount += tokens.length\n      }\n\n      if (nextSpecialStartIndex !== undefined) {\n        const specialToken = nextSpecialMatch![1]\n        const specialTokenValue = this.specialTokensEncoder.get(specialToken)\n        if (specialTokenValue === undefined) {\n          throw new Error(\n            `Special token \"${specialToken}\" is not in the special token encoder.`,\n          )\n        }\n        tokensCount++\n        startIndex = nextSpecialStartIndex + specialToken.length\n      } else {\n        break\n      }\n    }\n\n    return tokensCount\n  }\n\n  *decodeNativeGenerator(\n    tokens: Iterable<number>,\n  ): Generator<Uint8Array | string, void, void> {\n    for (const token of tokens) {\n      const tokenBytes = this.tryDecodeToken(token)\n      if (tokenBytes) {\n        yield tokenBytes\n      }\n    }\n  }\n\n  decodeNative(tokens: Iterable<number>): string {\n    let decoded = ''\n    let intBuffer = emptyBuffer\n\n    for (const token of tokens) {\n      const tokenBytes = this.tryDecodeToken(token)\n      if (tokenBytes === undefined) {\n        throw new Error(`Token ${token} is not in the byte pair encoder.`)\n      }\n      if (typeof tokenBytes === 'string') {\n        if (intBuffer !== emptyBuffer) {\n          decoded += decoder.decode(intBuffer, { stream: true })\n          intBuffer = emptyBuffer\n        }\n        decoded += tokenBytes\n      } else {\n        const newBuffer = new Uint8Array(intBuffer.length + tokenBytes.length)\n        newBuffer.set(intBuffer)\n        newBuffer.set(tokenBytes, intBuffer.length)\n        intBuffer = newBuffer\n      }\n    }\n\n    if (intBuffer !== emptyBuffer) {\n      decoded += decoder.decode(intBuffer, { stream: true })\n    }\n    return decoded\n  }\n\n  async *decodeNativeAsyncIterable(\n    tokens: AsyncIterable<number>,\n  ): AsyncGenerator<Uint8Array | string> {\n    for await (const token of tokens) {\n      const tokenBytesOrString = this.tryDecodeToken(token)\n      if (tokenBytesOrString) {\n        yield tokenBytesOrString\n      }\n    }\n  }\n\n  private getBpeRankFromString(key: string): number | undefined {\n    return this.bytePairStringRankEncoder.get(key)\n  }\n\n  private getBpeRankFromStringOrThrow(key: string): number {\n    const value = this.getBpeRankFromString(key)\n    if (value === undefined) {\n      throw new Error(\n        `The byte-pair encoding does not contain a value for: ${key}`,\n      )\n    }\n    return value\n  }\n\n  private getBpeRankFromBytes(key: Uint8Array): number | undefined {\n    const keyAsString = tryConvertToString(key)\n\n    if (keyAsString !== undefined) {\n      return this.getBpeRankFromString(keyAsString)\n    }\n\n    // Perform binary search on the binary keys\n    const index = this.binarySearch(key)\n    if (index !== -1) {\n      return this.bytePairNonUtfSortedEncoder[index]![1]\n    }\n\n    return undefined\n  }\n\n  private getBpeRankFromBytesOrThrow(key: Uint8Array): number {\n    const value = this.getBpeRankFromBytes(key)\n    if (value === undefined) {\n      throw new Error(\n        `The byte-pair encoding does not contain a value for: ${key.toString()}`,\n      )\n    }\n    return value\n  }\n\n  // Binary search on the binary keys\n  private binarySearch(key: Uint8Array): number {\n    let low = 0\n    let high = this.bytePairNonUtfSortedEncoder.length - 1\n\n    while (low <= high) {\n      const mid = (low + high) >>> 1\n      const midKey = this.bytePairNonUtfSortedEncoder[mid]![0]\n      let cmp = 0\n      const maxLength = Math.min(midKey.length, key.length)\n      for (let i = 0; i < maxLength; i++) {\n        cmp = midKey[i]! - key[i]!\n        if (cmp !== 0) break\n      }\n      if (cmp === 0) {\n        cmp = midKey.length - key.length\n      }\n      if (cmp === 0) {\n        return mid\n      }\n      if (cmp < 0) {\n        low = mid + 1\n      } else {\n        high = mid - 1\n      }\n    }\n    return -1\n  }\n\n  private findNextSpecialToken(\n    text: string,\n    allowedSpecial: Set<string> | undefined,\n    startIndex: number,\n  ): [startIndex: number, token: string] | undefined {\n    let searchIndex = startIndex\n\n    while (true) {\n      this.specialTokenPatternRegex.lastIndex = searchIndex\n      const nextSpecialMatch = this.specialTokenPatternRegex.exec(text)\n\n      if (!nextSpecialMatch) {\n        return undefined\n      }\n\n      const specialToken = nextSpecialMatch[0]\n\n      if (allowedSpecial?.has(specialToken)) {\n        const specialTokenStartIndex = nextSpecialMatch.index + searchIndex\n        return [specialTokenStartIndex, specialToken]\n      }\n\n      searchIndex = nextSpecialMatch.index + searchIndex + 1\n    }\n  }\n\n  private tryDecodeToken(tokenRank: number): Uint8Array | string | undefined {\n    const value = this.bytePairRankDecoder[tokenRank]\n    if (typeof value === 'string') {\n      return value\n    }\n    if (typeof value === 'object') {\n      const fromBinary = this.bytePairNonUtfRankDecoder.get(tokenRank)\n      if (fromBinary) {\n        return fromBinary\n      }\n    }\n    return this.specialTokensDecoder.get(tokenRank)\n  }\n\n  private addToMergeCache(key: string, value: number[]): void {\n    if (!this.mergeCache) return\n\n    if (this.mergeCache.size >= this.mergeCacheSize) {\n      // Remove least recently used item (first item)\n      const firstKey = this.mergeCache.keys().next().value!\n      this.mergeCache.delete(firstKey)\n    }\n    this.mergeCache.set(key, value)\n  }\n\n  private bytePairEncode(input: string): number[] {\n    if (input.length === 1 && isAscii(input.codePointAt(0)!)) {\n      return [this.getBpeRankFromStringOrThrow(input)]\n    }\n\n    if (this.mergeCache?.has(input)) {\n      const result = this.mergeCache.get(input)!\n      // Move to end to mark as recently used\n      this.mergeCache.delete(input)\n      this.mergeCache.set(input, result)\n      return result\n    }\n\n    const inputBytes = this.textEncoder.encode(input)\n    const result = this.bytePairMerge(inputBytes)\n    this.addToMergeCache(input, result)\n    return result\n  }\n\n  private bytePairMerge(\n    // Input array of bytes to process\n    piece: Uint8Array,\n  ): number[] {\n    const { length } = piece\n    // Partitions form a linked list over byte offsets: a live partition starting at\n    // offset 'i' spans [i, next[i]). 'pairRanks[i]' is the rank of merging it with\n    // the partition after it. Merges come off a min-heap ordered by (rank, offset),\n    // which picks the same pair as scanning for the lowest rank, leftmost first,\n    // without rescanning the piece after every merge.\n    const next = new Int32Array(length + 1)\n    const previous = new Int32Array(length + 1)\n    const pairRanks = new Float64Array(length + 1)\n    const live = new Uint8Array(length + 1)\n\n    const getPairRank = (startIndex: number): number => {\n      const middle = next[startIndex]!\n      if (middle >= length) return Number.POSITIVE_INFINITY\n      const rank = this.getBpeRankFromBytes(\n        piece.subarray(startIndex, next[middle]!),\n      )\n      return rank ?? Number.POSITIVE_INFINITY\n    }\n\n    // Heap entries are packed as [rank, startIndex] pairs in a flat array\n    const heap: number[] = []\n    const isLower = (a: number, b: number) =>\n      heap[a]! < heap[b]! ||\n      (heap[a] === heap[b] && heap[a + 1]! < heap[b + 1]!)\n    const swap = (a: number, b: number) => {\n      const rank = heap[a]!\n      const startIndex = heap[a + 1]!\n      heap[a] = heap[b]!\n      heap[a + 1] = heap[b + 1]!\n      heap[b] = rank\n      heap[b + 1] = startIndex\n    }\n    const push = (rank: number, startIndex: number) => {\n      heap.push(rank, startIndex)\n      let index = heap.length - 2\n      while (index > 0) {\n        const parent = (((index >> 1) - 1) >> 1) << 1\n        if (!isLower(index, parent)) break\n        swap(index, parent)\n        index = parent\n      }\n    }\n    const pop = () => {\n      const last = heap.length - 2\n      swap(0, last)\n      const rank = heap[last]!\n      const startIndex = heap[last + 1]!\n      heap.length = last\n      let index = 0\n      while (true) {\n        const left = index * 2 + 2\n        const right = left + 2\n        let lowest = index\n        if (left < heap.length && isLower(left, lowest)) lowest = left\n        if (right < heap.length && isLower(right, lowest)) lowest = right\n        if (lowest === index) break\n        swap(index, lowest)\n        index = lowest\n      }\n      return [rank, startIndex] as const\n    }\n\n    for (let i = 0; i <= length; i++) {\n      next[i] = i + 1\n      previous[i] = i - 1\n      live[i] = 1\n    }\n    for (let i = 0; i < length - 1; i++) {\n      pairRanks[i] = getPairRank(i)\n      if (pairRanks[i] !== Number.POSITIVE_INFINITY) push(pairRanks[i]!, i)\n    }\n\n    // Iteratively merge byte pairs until no more useful merges can be done\n    while (heap.length > 0) {\n      const [rank, startIndex] = pop()\n      // Skip entries made stale by an earlier merge\n      if (!live[startIndex] || pairRanks[startIndex] !== rank) continue\n\n      // Merge the partition at 'startIndex' with the one after it\n      const merged = next[startIndex]!\n      live[merged] = 0\n      next[startIndex] = next[merged]!\n      previous[next[merged]!] = startIndex\n\n      pairRanks[startIndex] = getPairRank(startIndex)\n      if (pairRanks[startIndex] !== Number.POSITIVE_INFINITY) {\n        push(pairRanks[startIndex]!, startIndex)\n      }\n\n      const before = previous[startIndex]!\n      if (before >= 0) {\n        pairRanks[before] = getPairRank(before)\n        if (pairRanks[before] !== Number.POSITIVE_INFINITY) {\n          push(pairRanks[before]!, before)\n        }\n      }\n    }\n\n    // Create the final output by applying the transform function to each partitioned range\n    const output: number[] = []\n    for (let i = 0; i < length; i = next[i]!) {\n      output.push(this.getBpeRankFromBytesOrThrow(piece.subarray(i, next[i]!)))\n    }\n    return output\n  }\n}\n"],"mappings":";;;;AAwBA,MAAM,8BAAc,IAAI,WAAW,CAAC;AAGpC,MAAa,UAAU,IAAI,YAAY,QAAQ,EAAE,WAAW,KAAK,CAAC;AAElE,IAAa,uBAAb,MAAkC;CAChC;;;;;;CAMA;CACA,4CAAoC,IAAI,IAAwB;CAChE;;;;;;CAMA;CACA;CACA;CACA;CACA;CACA,cAAsB,IAAI,YAAY;CACtC;CACA;CAEA,YAAY,EACV,qBACA,sBACA,iBACA,iBAAiB,4BACQ;EACzB,KAAK,sBAAsB;EAC3B,KAAK,4CAA4B,IAAI,IAAoB;EACzD,KAAK,iBAAiB;EACtB,IAAI,iBAAiB,GACnB,KAAK,6BAAa,IAAI,IAAI;EAI5B,KAAK,6BAA6B,OAAO,KAAK,mBAAmB,CAAC,CAAC;EACnE,MAAM,eAAuC,CAAC;EAE9C,oBAAoB,SAAS,OAAO,SAAS;GAC3C,IAAI,OAAO,UAAU,UAAU;IAC7B,KAAK,0BAA0B,IAAI,OAAO,IAAI;IAC9C;GACF;GACA,MAAM,YAAY,IAAI,WAAW,KAAK;GACtC,aAAa,KAAK,CAAC,WAAW,IAAI,CAAC;GACnC,KAAK,0BAA0B,IAAI,MAAM,SAAS;GAClD,IAAI,wBAAwB,SAAS,GAAG;IAKtC,MAAM,WAAW,mBAAmB,SAAS;IAC7C,IAAI,aAAa,KAAA,GACf,KAAK,0BAA0B,IAAI,UAAU,IAAI;GAErD;EACF,CAAC;EACD,KAAK,8BAA8B,aAAa,MAAM,GAAG,MACvD,mBAAmB,EAAE,IAAI,EAAE,EAAE,CAC/B;EACA,KAAK,uBACH,wCAAwB,IAAI,IAAoB;EAClD,KAAK,uBAAuB,uBACxB,IAAI,IAAI,CAAC,GAAG,oBAAoB,CAAC,CAAC,KAAK,CAAC,KAAK,WAAW,CAAC,OAAO,GAAG,CAAC,CAAC,oBACrE,IAAI,IAAoB;EAC5B,KAAK,kBAAkB;EAKvB,MAAM,wBAHuB,CAAC,GAAG,KAAK,qBAAqB,KAAK,CAAC,CAAC,CAAC,IACjE,YAE+C,CAAC,CAAC,KAAK,GAAG;EAC3D,IAAI;GACF,KAAK,2BAA2B,IAAI,OAAO,uBAAuB,GAAG;EACvE,QAAQ;GACN,MAAM,IAAI,MAAM,qCAAqC;EACvD;CACF;CAEA,kBAAkB,SAAuB;EACvC,IAAI,KAAK,mBAAmB,KAAK,UAAU,GACzC,KAAK,6BAAa,IAAI,IAAI;EAE5B,KAAK,iBAAiB;EACtB,IAAI,YAAY,GACd,KAAK,aAAa,KAAA;CAEtB;CAEA,kBAAwB;EACtB,KAAK,YAAY,MAAM;CACzB;CAEA,CAAC,sBACC,MACA,gBACwC;EACxC,IAAI,aAAa;EACjB,IAAI,kBAAkB;EAEtB,OAAO,MAAM;GACX,MAAM,mBAAmB,KAAK,qBAC5B,MACA,gBACA,UACF;GACA,MAAM,wBAAwB,mBAAmB;GAEjD,MAAM,WAAW,yBAAyB,KAAK;GAE/C,MAAM,oBACJ,eAAe,KAAK,aAAa,KAAK,SAClC,OACA,KAAK,MAAM,YAAY,QAAQ;GAErC,KAAK,MAAM,CAAC,UAAU,kBAAkB,SAAS,KAAK,eAAe,GAAG;IACtE,MAAM,QAAQ,KAAK,qBAAqB,KAAK;IAC7C,IAAI,UAAU,KAAA,GAAW;KACvB,kBAAkB;KAClB,MAAM,CAAC,KAAK;KAEZ;IACF;IAEA,MAAM,SAAS,KAAK,eAAe,KAAK;IACxC,kBAAkB,OAAO;IACzB,MAAM;GACR;GAEA,IAAI,0BAA0B,KAAA,GAAW;IACvC,MAAM,eAAe,iBAAkB;IACvC,MAAM,oBAAoB,KAAK,qBAAqB,IAAI,YAAY;IACpE,IAAI,sBAAsB,KAAA,GACxB,MAAM,IAAI,MACR,kBAAkB,aAAa,uCACjC;IAEF,MAAM,CAAC,iBAAiB;IACxB,aAAa,wBAAwB,aAAa;IAClD,kBAAkB;GACpB,OACE;EAEJ;EAEA,OAAO;CACT;CAEA,aAAa,MAAc,gBAAwC;EACjE,IAAI,aAAa;EACjB,MAAM,cAAwB,CAAC;EAE/B,OAAO,MAAM;GACX,MAAM,mBAAmB,KAAK,qBAC5B,MACA,gBACA,UACF;GACA,MAAM,wBAAwB,mBAAmB;GAEjD,MAAM,WAAW,yBAAyB,KAAK;GAE/C,MAAM,oBACJ,eAAe,KAAK,aAAa,KAAK,SAClC,OACA,KAAK,MAAM,YAAY,QAAQ;GAErC,KAAK,MAAM,CAAC,UAAU,kBAAkB,SAAS,KAAK,eAAe,GAAG;IACtE,MAAM,QAAQ,KAAK,qBAAqB,KAAK;IAC7C,IAAI,UAAU,KAAA,GAAW;KACvB,YAAY,KAAK,KAAK;KAEtB;IACF;IAEA,MAAM,SAAS,KAAK,eAAe,KAAK;IACxC,YAAY,KAAK,GAAG,MAAM;GAC5B;GAEA,IAAI,0BAA0B,KAAA,GAAW;IACvC,MAAM,eAAe,iBAAkB;IACvC,MAAM,oBAAoB,KAAK,qBAAqB,IAAI,YAAY;IACpE,IAAI,sBAAsB,KAAA,GACxB,MAAM,IAAI,MACR,kBAAkB,aAAa,uCACjC;IAEF,YAAY,KAAK,iBAAiB;IAClC,aAAa,wBAAwB,aAAa;GACpD,OACE;EAEJ;EAEA,OAAO;CACT;CAEA,YAAY,MAAc,gBAAsC;EAC9D,IAAI,aAAa;EACjB,IAAI,cAAc;EAElB,OAAO,MAAM;GACX,MAAM,mBAAmB,KAAK,qBAC5B,MACA,gBACA,UACF;GACA,MAAM,wBAAwB,mBAAmB;GAEjD,MAAM,WAAW,yBAAyB,KAAK;GAE/C,MAAM,oBACJ,eAAe,KAAK,aAAa,KAAK,SAClC,OACA,KAAK,MAAM,YAAY,QAAQ;GAErC,KAAK,MAAM,CAAC,UAAU,kBAAkB,SAAS,KAAK,eAAe,GAAG;IAEtE,IADc,KAAK,qBAAqB,KAChC,MAAM,KAAA,GAAW;KACvB;KAEA;IACF;IAEA,MAAM,SAAS,KAAK,eAAe,KAAK;IACxC,eAAe,OAAO;GACxB;GAEA,IAAI,0BAA0B,KAAA,GAAW;IACvC,MAAM,eAAe,iBAAkB;IAEvC,IAD0B,KAAK,qBAAqB,IAAI,YACpC,MAAM,KAAA,GACxB,MAAM,IAAI,MACR,kBAAkB,aAAa,uCACjC;IAEF;IACA,aAAa,wBAAwB,aAAa;GACpD,OACE;EAEJ;EAEA,OAAO;CACT;CAEA,CAAC,sBACC,QAC4C;EAC5C,KAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,aAAa,KAAK,eAAe,KAAK;GAC5C,IAAI,YACF,MAAM;EAEV;CACF;CAEA,aAAa,QAAkC;EAC7C,IAAI,UAAU;EACd,IAAI,YAAY;EAEhB,KAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,aAAa,KAAK,eAAe,KAAK;GAC5C,IAAI,eAAe,KAAA,GACjB,MAAM,IAAI,MAAM,SAAS,MAAM,kCAAkC;GAEnE,IAAI,OAAO,eAAe,UAAU;IAClC,IAAI,cAAc,aAAa;KAC7B,WAAW,QAAQ,OAAO,WAAW,EAAE,QAAQ,KAAK,CAAC;KACrD,YAAY;IACd;IACA,WAAW;GACb,OAAO;IACL,MAAM,YAAY,IAAI,WAAW,UAAU,SAAS,WAAW,MAAM;IACrE,UAAU,IAAI,SAAS;IACvB,UAAU,IAAI,YAAY,UAAU,MAAM;IAC1C,YAAY;GACd;EACF;EAEA,IAAI,cAAc,aAChB,WAAW,QAAQ,OAAO,WAAW,EAAE,QAAQ,KAAK,CAAC;EAEvD,OAAO;CACT;CAEA,OAAO,0BACL,QACqC;EACrC,WAAW,MAAM,SAAS,QAAQ;GAChC,MAAM,qBAAqB,KAAK,eAAe,KAAK;GACpD,IAAI,oBACF,MAAM;EAEV;CACF;CAEA,qBAA6B,KAAiC;EAC5D,OAAO,KAAK,0BAA0B,IAAI,GAAG;CAC/C;CAEA,4BAAoC,KAAqB;EACvD,MAAM,QAAQ,KAAK,qBAAqB,GAAG;EAC3C,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,MACR,wDAAwD,KAC1D;EAEF,OAAO;CACT;CAEA,oBAA4B,KAAqC;EAC/D,MAAM,cAAc,mBAAmB,GAAG;EAE1C,IAAI,gBAAgB,KAAA,GAClB,OAAO,KAAK,qBAAqB,WAAW;EAI9C,MAAM,QAAQ,KAAK,aAAa,GAAG;EACnC,IAAI,UAAU,IACZ,OAAO,KAAK,4BAA4B,MAAM,CAAE;CAIpD;CAEA,2BAAmC,KAAyB;EAC1D,MAAM,QAAQ,KAAK,oBAAoB,GAAG;EAC1C,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,MACR,wDAAwD,IAAI,SAAS,GACvE;EAEF,OAAO;CACT;CAGA,aAAqB,KAAyB;EAC5C,IAAI,MAAM;EACV,IAAI,OAAO,KAAK,4BAA4B,SAAS;EAErD,OAAO,OAAO,MAAM;GAClB,MAAM,MAAO,MAAM,SAAU;GAC7B,MAAM,SAAS,KAAK,4BAA4B,IAAI,CAAE;GACtD,IAAI,MAAM;GACV,MAAM,YAAY,KAAK,IAAI,OAAO,QAAQ,IAAI,MAAM;GACpD,KAAK,IAAI,IAAI,GAAG,IAAI,WAAW,KAAK;IAClC,MAAM,OAAO,KAAM,IAAI;IACvB,IAAI,QAAQ,GAAG;GACjB;GACA,IAAI,QAAQ,GACV,MAAM,OAAO,SAAS,IAAI;GAE5B,IAAI,QAAQ,GACV,OAAO;GAET,IAAI,MAAM,GACR,MAAM,MAAM;QAEZ,OAAO,MAAM;EAEjB;EACA,OAAO;CACT;CAEA,qBACE,MACA,gBACA,YACiD;EACjD,IAAI,cAAc;EAElB,OAAO,MAAM;GACX,KAAK,yBAAyB,YAAY;GAC1C,MAAM,mBAAmB,KAAK,yBAAyB,KAAK,IAAI;GAEhE,IAAI,CAAC,kBACH;GAGF,MAAM,eAAe,iBAAiB;GAEtC,IAAI,gBAAgB,IAAI,YAAY,GAElC,OAAO,CADwB,iBAAiB,QAAQ,aACxB,YAAY;GAG9C,cAAc,iBAAiB,QAAQ,cAAc;EACvD;CACF;CAEA,eAAuB,WAAoD;EACzE,MAAM,QAAQ,KAAK,oBAAoB;EACvC,IAAI,OAAO,UAAU,UACnB,OAAO;EAET,IAAI,OAAO,UAAU,UAAU;GAC7B,MAAM,aAAa,KAAK,0BAA0B,IAAI,SAAS;GAC/D,IAAI,YACF,OAAO;EAEX;EACA,OAAO,KAAK,qBAAqB,IAAI,SAAS;CAChD;CAEA,gBAAwB,KAAa,OAAuB;EAC1D,IAAI,CAAC,KAAK,YAAY;EAEtB,IAAI,KAAK,WAAW,QAAQ,KAAK,gBAAgB;GAE/C,MAAM,WAAW,KAAK,WAAW,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC;GAC/C,KAAK,WAAW,OAAO,QAAQ;EACjC;EACA,KAAK,WAAW,IAAI,KAAK,KAAK;CAChC;CAEA,eAAuB,OAAyB;EAC9C,IAAI,MAAM,WAAW,KAAK,QAAQ,MAAM,YAAY,CAAC,CAAE,GACrD,OAAO,CAAC,KAAK,4BAA4B,KAAK,CAAC;EAGjD,IAAI,KAAK,YAAY,IAAI,KAAK,GAAG;GAC/B,MAAM,SAAS,KAAK,WAAW,IAAI,KAAK;GAExC,KAAK,WAAW,OAAO,KAAK;GAC5B,KAAK,WAAW,IAAI,OAAO,MAAM;GACjC,OAAO;EACT;EAEA,MAAM,aAAa,KAAK,YAAY,OAAO,KAAK;EAChD,MAAM,SAAS,KAAK,cAAc,UAAU;EAC5C,KAAK,gBAAgB,OAAO,MAAM;EAClC,OAAO;CACT;CAEA,cAEE,OACU;EACV,MAAM,EAAE,WAAW;EAMnB,MAAM,OAAO,IAAI,WAAW,SAAS,CAAC;EACtC,MAAM,WAAW,IAAI,WAAW,SAAS,CAAC;EAC1C,MAAM,YAAY,IAAI,aAAa,SAAS,CAAC;EAC7C,MAAM,OAAO,IAAI,WAAW,SAAS,CAAC;EAEtC,MAAM,eAAe,eAA+B;GAClD,MAAM,SAAS,KAAK;GACpB,IAAI,UAAU,QAAQ,OAAO,OAAO;GAIpC,OAHa,KAAK,oBAChB,MAAM,SAAS,YAAY,KAAK,OAAQ,CAEhC,KAAK,OAAO;EACxB;EAGA,MAAM,OAAiB,CAAC;EACxB,MAAM,WAAW,GAAW,MAC1B,KAAK,KAAM,KAAK,MACf,KAAK,OAAO,KAAK,MAAM,KAAK,IAAI,KAAM,KAAK,IAAI;EAClD,MAAM,QAAQ,GAAW,MAAc;GACrC,MAAM,OAAO,KAAK;GAClB,MAAM,aAAa,KAAK,IAAI;GAC5B,KAAK,KAAK,KAAK;GACf,KAAK,IAAI,KAAK,KAAK,IAAI;GACvB,KAAK,KAAK;GACV,KAAK,IAAI,KAAK;EAChB;EACA,MAAM,QAAQ,MAAc,eAAuB;GACjD,KAAK,KAAK,MAAM,UAAU;GAC1B,IAAI,QAAQ,KAAK,SAAS;GAC1B,OAAO,QAAQ,GAAG;IAChB,MAAM,UAAY,SAAS,KAAK,KAAM,KAAM;IAC5C,IAAI,CAAC,QAAQ,OAAO,MAAM,GAAG;IAC7B,KAAK,OAAO,MAAM;IAClB,QAAQ;GACV;EACF;EACA,MAAM,YAAY;GAChB,MAAM,OAAO,KAAK,SAAS;GAC3B,KAAK,GAAG,IAAI;GACZ,MAAM,OAAO,KAAK;GAClB,MAAM,aAAa,KAAK,OAAO;GAC/B,KAAK,SAAS;GACd,IAAI,QAAQ;GACZ,OAAO,MAAM;IACX,MAAM,OAAO,QAAQ,IAAI;IACzB,MAAM,QAAQ,OAAO;IACrB,IAAI,SAAS;IACb,IAAI,OAAO,KAAK,UAAU,QAAQ,MAAM,MAAM,GAAG,SAAS;IAC1D,IAAI,QAAQ,KAAK,UAAU,QAAQ,OAAO,MAAM,GAAG,SAAS;IAC5D,IAAI,WAAW,OAAO;IACtB,KAAK,OAAO,MAAM;IAClB,QAAQ;GACV;GACA,OAAO,CAAC,MAAM,UAAU;EAC1B;EAEA,KAAK,IAAI,IAAI,GAAG,KAAK,QAAQ,KAAK;GAChC,KAAK,KAAK,IAAI;GACd,SAAS,KAAK,IAAI;GAClB,KAAK,KAAK;EACZ;EACA,KAAK,IAAI,IAAI,GAAG,IAAI,SAAS,GAAG,KAAK;GACnC,UAAU,KAAK,YAAY,CAAC;GAC5B,IAAI,UAAU,OAAO,OAAO,mBAAmB,KAAK,UAAU,IAAK,CAAC;EACtE;EAGA,OAAO,KAAK,SAAS,GAAG;GACtB,MAAM,CAAC,MAAM,cAAc,IAAI;GAE/B,IAAI,CAAC,KAAK,eAAe,UAAU,gBAAgB,MAAM;GAGzD,MAAM,SAAS,KAAK;GACpB,KAAK,UAAU;GACf,KAAK,cAAc,KAAK;GACxB,SAAS,KAAK,WAAY;GAE1B,UAAU,cAAc,YAAY,UAAU;GAC9C,IAAI,UAAU,gBAAgB,OAAO,mBACnC,KAAK,UAAU,aAAc,UAAU;GAGzC,MAAM,SAAS,SAAS;GACxB,IAAI,UAAU,GAAG;IACf,UAAU,UAAU,YAAY,MAAM;IACtC,IAAI,UAAU,YAAY,OAAO,mBAC/B,KAAK,UAAU,SAAU,MAAM;GAEnC;EACF;EAGA,MAAM,SAAmB,CAAC;EAC1B,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,IAAI,KAAK,IACnC,OAAO,KAAK,KAAK,2BAA2B,MAAM,SAAS,GAAG,KAAK,EAAG,CAAC,CAAC;EAE1E,OAAO;CACT;AACF"}

package/esm/GptEncoding.d.ts

@@ -61,7 +61,7 @@ declare class GptEncoding {
    * Also mentioned in section 6 of this document: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
    * @param encodeOptions Options controlling how special tokens are handled.
    */
-  encodeChatGenerator(chat: Iterable<ChatMessage>, model?: "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "gpt2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions): Generator<number[], void, undefined>;
+  encodeChatGenerator(chat: Iterable<ChatMessage>, model?: "gpt2" | "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions): Generator<number[], void, undefined>;
   /**
    * Encodes a chat into a single array of tokens.
    * Warning: gpt-3.5-turbo and gpt-4 chat format may change over time.
@@ -70,7 +70,7 @@ declare class GptEncoding {
    * Also mentioned in section 6 of this document: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
    * @param encodeOptions Options controlling how special tokens are handled.
    */
-  encodeChat(chat: readonly ChatMessage[], model?: "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "gpt2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions): number[];
+  encodeChat(chat: readonly ChatMessage[], model?: "gpt2" | "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions): number[];
   /**
    * Checks whether the provided input stays within the provided token limit.
    * @param input The string or chat messages to evaluate.

package/esm/encoding/cl100k_base.d.ts

@@ -5,7 +5,7 @@ import { EncodeOptions, GptEncoding } from "../GptEncoding.js";
 import { EndOfPrompt, EndOfText, FimMiddle, FimPrefix, FimSuffix, HarmonyCall, HarmonyChannel, HarmonyConstrain, HarmonyEnd, HarmonyMessage, HarmonyReturn, HarmonyStart, HarmonyStartOfText, ImEnd, ImSep, ImStart } from "../specialTokens.js";
 //#region src/encoding/cl100k_base.d.ts
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<ChatMessage>, tokenLimit: number, encodeOptions?: EncodeOptions) => false | number, countTokens: (input: string | Iterable<ChatMessage>, encodeOptions?: EncodeOptions) => number, encodeChat: (chat: readonly ChatMessage[], model?: "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "gpt2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions) => number[], encodeChatGenerator: (chat: Iterable<ChatMessage>, model?: "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "gpt2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions) => Generator<number[], void, undefined>, vocabularySize: number, setMergeCacheSize: (size: number) => void, clearMergeCache: () => void, estimateCost: (tokenCount: number, modelSpec?: ModelSpec | undefined) => PriceData;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<ChatMessage>, tokenLimit: number, encodeOptions?: EncodeOptions) => false | number, countTokens: (input: string | Iterable<ChatMessage>, encodeOptions?: EncodeOptions) => number, encodeChat: (chat: readonly ChatMessage[], model?: "gpt2" | "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions) => number[], encodeChatGenerator: (chat: Iterable<ChatMessage>, model?: "gpt2" | "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions) => Generator<number[], void, undefined>, vocabularySize: number, setMergeCacheSize: (size: number) => void, clearMergeCache: () => void, estimateCost: (tokenCount: number, modelSpec?: ModelSpec | undefined) => PriceData;
 //#endregion
 export { ALL_SPECIAL_TOKENS, DEFAULT_MERGE_CACHE_SIZE, EndOfPrompt, EndOfText, FimMiddle, FimPrefix, FimSuffix, HarmonyCall, HarmonyChannel, HarmonyConstrain, HarmonyEnd, HarmonyMessage, HarmonyReturn, HarmonyStart, HarmonyStartOfText, ImEnd, ImSep, ImStart, clearMergeCache, countTokens, decode, decodeAsyncGenerator, decodeGenerator, api as default, encode, encodeChat, encodeChatGenerator, encodeGenerator, estimateCost, isWithinTokenLimit, setMergeCacheSize, vocabularySize };
 //# sourceMappingURL=cl100k_base.d.ts.map