gpt-tokenizer 4.0.0 → 4.0.2

package/cjs/BytePairEncodingCore.js

@@ -4,7 +4,7 @@ const require_utfUtil = require("./utfUtil.js");
 const require_util = require("./util.js");
 //#region src/BytePairEncodingCore.ts
 const emptyBuffer = /* @__PURE__ */ new Uint8Array(0);
-const decoder = new TextDecoder("utf8");
+const decoder = new TextDecoder("utf8", { ignoreBOM: true });
 var BytePairEncodingCore = class {
 	mergeableBytePairRankCount;
 	/**
@@ -43,6 +43,10 @@ var BytePairEncodingCore = class {
 			const byteArray = new Uint8Array(value);
 			binaryLookup.push([byteArray, rank]);
 			this.bytePairNonUtfRankDecoder.set(rank, byteArray);
+			if (require_utfUtil.startsWithByteOrderMark(byteArray)) {
+				const asString = require_utfUtil.tryConvertToString(byteArray);
+				if (asString !== void 0) this.bytePairStringRankEncoder.set(asString, rank);
+			}
 		});
 		this.bytePairNonUtfSortedEncoder = binaryLookup.sort((a, b) => require_utfUtil.compareUint8Arrays(a[0], b[0]));
 		this.specialTokensEncoder = specialTokensEncoder ?? /* @__PURE__ */ new Map();
@@ -259,41 +263,81 @@ var BytePairEncodingCore = class {
 		return result;
 	}
 	bytePairMerge(piece) {
-		const starts = [];
-		const ranks = [];
-		const getRank = (startIndex, pairStart = starts[startIndex], pairEnd = starts[startIndex + 2]) => {
-			if (pairEnd === void 0) return Number.POSITIVE_INFINITY;
-			const key = piece.subarray(pairStart, pairEnd);
-			return this.getBpeRankFromBytes(key) ?? Number.POSITIVE_INFINITY;
+		const { length } = piece;
+		const next = new Int32Array(length + 1);
+		const previous = new Int32Array(length + 1);
+		const pairRanks = new Float64Array(length + 1);
+		const live = new Uint8Array(length + 1);
+		const getPairRank = (startIndex) => {
+			const middle = next[startIndex];
+			if (middle >= length) return Number.POSITIVE_INFINITY;
+			return this.getBpeRankFromBytes(piece.subarray(startIndex, next[middle])) ?? Number.POSITIVE_INFINITY;
+		};
+		const heap = [];
+		const isLower = (a, b) => heap[a] < heap[b] || heap[a] === heap[b] && heap[a + 1] < heap[b + 1];
+		const swap = (a, b) => {
+			const rank = heap[a];
+			const startIndex = heap[a + 1];
+			heap[a] = heap[b];
+			heap[a + 1] = heap[b + 1];
+			heap[b] = rank;
+			heap[b + 1] = startIndex;
 		};
-		for (let i = 0; i <= piece.length; i++) {
-			starts.push(i);
-			if (i < piece.length - 1) ranks.push(getRank(i, i, i + 2));
-			else ranks.push(Number.POSITIVE_INFINITY);
+		const push = (rank, startIndex) => {
+			heap.push(rank, startIndex);
+			let index = heap.length - 2;
+			while (index > 0) {
+				const parent = (index >> 1) - 1 >> 1 << 1;
+				if (!isLower(index, parent)) break;
+				swap(index, parent);
+				index = parent;
+			}
+		};
+		const pop = () => {
+			const last = heap.length - 2;
+			swap(0, last);
+			const rank = heap[last];
+			const startIndex = heap[last + 1];
+			heap.length = last;
+			let index = 0;
+			while (true) {
+				const left = index * 2 + 2;
+				const right = left + 2;
+				let lowest = index;
+				if (left < heap.length && isLower(left, lowest)) lowest = left;
+				if (right < heap.length && isLower(right, lowest)) lowest = right;
+				if (lowest === index) break;
+				swap(index, lowest);
+				index = lowest;
+			}
+			return [rank, startIndex];
+		};
+		for (let i = 0; i <= length; i++) {
+			next[i] = i + 1;
+			previous[i] = i - 1;
+			live[i] = 1;
 		}
-		while (starts.length > 1) {
-			let lowestRank = Number.POSITIVE_INFINITY;
-			let lowestPartitionIndex = -1;
-			for (let i = 0; i < ranks.length - 1; i++) {
-				const rank = ranks[i];
-				if (rank < lowestRank) {
-					lowestRank = rank;
-					lowestPartitionIndex = i;
-				}
+		for (let i = 0; i < length - 1; i++) {
+			pairRanks[i] = getPairRank(i);
+			if (pairRanks[i] !== Number.POSITIVE_INFINITY) push(pairRanks[i], i);
+		}
+		while (heap.length > 0) {
+			const [rank, startIndex] = pop();
+			if (!live[startIndex] || pairRanks[startIndex] !== rank) continue;
+			const merged = next[startIndex];
+			live[merged] = 0;
+			next[startIndex] = next[merged];
+			previous[next[merged]] = startIndex;
+			pairRanks[startIndex] = getPairRank(startIndex);
+			if (pairRanks[startIndex] !== Number.POSITIVE_INFINITY) push(pairRanks[startIndex], startIndex);
+			const before = previous[startIndex];
+			if (before >= 0) {
+				pairRanks[before] = getPairRank(before);
+				if (pairRanks[before] !== Number.POSITIVE_INFINITY) push(pairRanks[before], before);
 			}
-			if (lowestRank === Number.POSITIVE_INFINITY || lowestPartitionIndex === -1) break;
-			starts.splice(lowestPartitionIndex + 1, 1);
-			ranks.splice(lowestPartitionIndex, 1);
-			ranks[lowestPartitionIndex] = getRank(lowestPartitionIndex);
-			if (lowestPartitionIndex > 0) ranks[lowestPartitionIndex - 1] = getRank(lowestPartitionIndex - 1);
 		}
 		const output = [];
-		for (let i = 0; i < starts.length - 1; i++) {
-			const pairStart = starts[i];
-			const pairEnd = starts[i + 1];
-			const bpeValue = this.getBpeRankFromBytesOrThrow(piece.subarray(pairStart, pairEnd));
-			output.push(bpeValue);
-		}
+		for (let i = 0; i < length; i = next[i]) output.push(this.getBpeRankFromBytesOrThrow(piece.subarray(i, next[i])));
 		return output;
 	}
 };

package/cjs/BytePairEncodingCore.js.map

@@ -1 +1 @@
-{"version":3,"file":"BytePairEncodingCore.js","names":["DEFAULT_MERGE_CACHE_SIZE","compareUint8Arrays","escapeRegExp","tryConvertToString","isAscii"],"sources":["../src/BytePairEncodingCore.ts"],"sourcesContent":["import { DEFAULT_MERGE_CACHE_SIZE } from './constants.js'\nimport { compareUint8Arrays, isAscii, tryConvertToString } from './utfUtil.js'\nimport { escapeRegExp } from './util.js'\n\nexport type RawBytePairRanks = readonly (string | readonly number[])[]\n\nexport interface BytePairEncodingConfig {\n  bytePairRankDecoder: RawBytePairRanks\n  specialTokensEncoder?: Map<string, number>\n  tokenSplitRegex: RegExp\n  /**\n   * LRU cache for merged tokens pairs.\n   * Increasing this value should make encoding similar strings faster,\n   * but will consume more memory.\n   * @default 100000\n   */\n  mergeCacheSize?: number\n}\n\nconst emptyBuffer = new Uint8Array(0)\n\nexport const decoder = new TextDecoder('utf8')\n\nexport class BytePairEncodingCore {\n  readonly mergeableBytePairRankCount: number\n  /**\n   * an array where the index is the BPE rank,\n   * and the value is the string or the array of bytes that it decodes to\n   * it may contain holes if token is unused\n   */\n  private bytePairRankDecoder: RawBytePairRanks\n  private bytePairNonUtfRankDecoder = new Map<number, Uint8Array>()\n  private bytePairNonUtfSortedEncoder: readonly [Uint8Array, number][]\n  /**\n   * a reverse map of the bytePairRankDecoder,\n   * where the key is the string and the value is the rank\n   * values that cannot be represented as a string are present in `bytePairNonUtfSortedEncoder`\n   */\n  private bytePairStringRankEncoder: Map<string, number>\n  private tokenSplitRegex: RegExp\n  private specialTokensEncoder: Map<string, number>\n  private specialTokensDecoder: Map<number, string>\n  private specialTokenPatternRegex: RegExp\n  private textEncoder = new TextEncoder()\n  private mergeCache?: Map<string, number[]>\n  private mergeCacheSize: number\n\n  constructor({\n    bytePairRankDecoder,\n    specialTokensEncoder,\n    tokenSplitRegex,\n    mergeCacheSize = DEFAULT_MERGE_CACHE_SIZE,\n  }: BytePairEncodingConfig) {\n    this.bytePairRankDecoder = bytePairRankDecoder\n    this.bytePairStringRankEncoder = new Map<string, number>()\n    this.mergeCacheSize = mergeCacheSize\n    if (mergeCacheSize > 0) {\n      this.mergeCache = new Map()\n    }\n\n    // size without array holes (which may be present in the encoder)\n    this.mergeableBytePairRankCount = Object.keys(bytePairRankDecoder).length\n    const binaryLookup: [Uint8Array, number][] = []\n    // forEach skips array holes:\n    bytePairRankDecoder.forEach((value, rank) => {\n      if (typeof value === 'string') {\n        this.bytePairStringRankEncoder.set(value, rank)\n        return\n      }\n      const byteArray = new Uint8Array(value)\n      binaryLookup.push([byteArray, rank])\n      this.bytePairNonUtfRankDecoder.set(rank, byteArray)\n    })\n    this.bytePairNonUtfSortedEncoder = binaryLookup.sort((a, b) =>\n      compareUint8Arrays(a[0], b[0]),\n    )\n    this.specialTokensEncoder =\n      specialTokensEncoder ?? new Map<string, number>()\n    this.specialTokensDecoder = specialTokensEncoder\n      ? new Map([...specialTokensEncoder].map(([key, value]) => [value, key]))\n      : new Map<number, string>()\n    this.tokenSplitRegex = tokenSplitRegex\n\n    const escapedSpecialTokens = [...this.specialTokensEncoder.keys()].map(\n      escapeRegExp,\n    )\n    const allSpecialTokensRegex = escapedSpecialTokens.join('|')\n    try {\n      this.specialTokenPatternRegex = new RegExp(allSpecialTokensRegex, 'y')\n    } catch {\n      throw new Error('Invalid regular expression pattern.')\n    }\n  }\n\n  setMergeCacheSize(newSize: number): void {\n    if (this.mergeCacheSize === 0 && newSize > 0) {\n      this.mergeCache = new Map()\n    }\n    this.mergeCacheSize = newSize\n    if (newSize === 0) {\n      this.mergeCache = undefined\n    }\n  }\n\n  clearMergeCache(): void {\n    this.mergeCache?.clear()\n  }\n\n  *encodeNativeGenerator(\n    text: string,\n    allowedSpecial?: Set<string>,\n  ): Generator<number[], number, undefined> {\n    let startIndex = 0\n    let lastTokenLength = 0\n\n    while (true) {\n      const nextSpecialMatch = this.findNextSpecialToken(\n        text,\n        allowedSpecial,\n        startIndex,\n      )\n      const nextSpecialStartIndex = nextSpecialMatch?.[0]\n\n      const endIndex = nextSpecialStartIndex ?? text.length\n\n      const textBeforeSpecial =\n        startIndex === 0 && endIndex === text.length\n          ? text\n          : text.slice(startIndex, endIndex)\n\n      for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {\n        const token = this.getBpeRankFromString(match)\n        if (token !== undefined) {\n          lastTokenLength = 1\n          yield [token]\n\n          continue\n        }\n\n        const tokens = this.bytePairEncode(match)\n        lastTokenLength = tokens.length\n        yield tokens\n      }\n\n      if (nextSpecialStartIndex !== undefined) {\n        const specialToken = nextSpecialMatch![1]\n        const specialTokenValue = this.specialTokensEncoder.get(specialToken)\n        if (specialTokenValue === undefined) {\n          throw new Error(\n            `Special token \"${specialToken}\" is not in the special token encoder.`,\n          )\n        }\n        yield [specialTokenValue]\n        startIndex = nextSpecialStartIndex + specialToken.length\n        lastTokenLength = 1\n      } else {\n        break\n      }\n    }\n\n    return lastTokenLength\n  }\n\n  encodeNative(text: string, allowedSpecial?: Set<string>): number[] {\n    let startIndex = 0\n    const tokensArray: number[] = [] // Flat list to collect the tokens\n\n    while (true) {\n      const nextSpecialMatch = this.findNextSpecialToken(\n        text,\n        allowedSpecial,\n        startIndex,\n      )\n      const nextSpecialStartIndex = nextSpecialMatch?.[0]\n\n      const endIndex = nextSpecialStartIndex ?? text.length\n\n      const textBeforeSpecial =\n        startIndex === 0 && endIndex === text.length\n          ? text\n          : text.slice(startIndex, endIndex)\n\n      for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {\n        const token = this.getBpeRankFromString(match)\n        if (token !== undefined) {\n          tokensArray.push(token)\n\n          continue\n        }\n\n        const tokens = this.bytePairEncode(match)\n        tokensArray.push(...tokens)\n      }\n\n      if (nextSpecialStartIndex !== undefined) {\n        const specialToken = nextSpecialMatch![1]\n        const specialTokenValue = this.specialTokensEncoder.get(specialToken)\n        if (specialTokenValue === undefined) {\n          throw new Error(\n            `Special token \"${specialToken}\" is not in the special token encoder.`,\n          )\n        }\n        tokensArray.push(specialTokenValue)\n        startIndex = nextSpecialStartIndex + specialToken.length\n      } else {\n        break\n      }\n    }\n\n    return tokensArray\n  }\n\n  countNative(text: string, allowedSpecial?: Set<string>): number {\n    let startIndex = 0\n    let tokensCount = 0\n\n    while (true) {\n      const nextSpecialMatch = this.findNextSpecialToken(\n        text,\n        allowedSpecial,\n        startIndex,\n      )\n      const nextSpecialStartIndex = nextSpecialMatch?.[0]\n\n      const endIndex = nextSpecialStartIndex ?? text.length\n\n      const textBeforeSpecial =\n        startIndex === 0 && endIndex === text.length\n          ? text\n          : text.slice(startIndex, endIndex)\n\n      for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {\n        const token = this.getBpeRankFromString(match)\n        if (token !== undefined) {\n          tokensCount++\n\n          continue\n        }\n\n        const tokens = this.bytePairEncode(match)\n        tokensCount += tokens.length\n      }\n\n      if (nextSpecialStartIndex !== undefined) {\n        const specialToken = nextSpecialMatch![1]\n        const specialTokenValue = this.specialTokensEncoder.get(specialToken)\n        if (specialTokenValue === undefined) {\n          throw new Error(\n            `Special token \"${specialToken}\" is not in the special token encoder.`,\n          )\n        }\n        tokensCount++\n        startIndex = nextSpecialStartIndex + specialToken.length\n      } else {\n        break\n      }\n    }\n\n    return tokensCount\n  }\n\n  *decodeNativeGenerator(\n    tokens: Iterable<number>,\n  ): Generator<Uint8Array | string, void, void> {\n    for (const token of tokens) {\n      const tokenBytes = this.tryDecodeToken(token)\n      if (tokenBytes) {\n        yield tokenBytes\n      }\n    }\n  }\n\n  decodeNative(tokens: Iterable<number>): string {\n    let decoded = ''\n    let intBuffer = emptyBuffer\n\n    for (const token of tokens) {\n      const tokenBytes = this.tryDecodeToken(token)\n      if (tokenBytes === undefined) {\n        throw new Error(`Token ${token} is not in the byte pair encoder.`)\n      }\n      if (typeof tokenBytes === 'string') {\n        if (intBuffer !== emptyBuffer) {\n          decoded += decoder.decode(intBuffer, { stream: true })\n          intBuffer = emptyBuffer\n        }\n        decoded += tokenBytes\n      } else {\n        const newBuffer = new Uint8Array(intBuffer.length + tokenBytes.length)\n        newBuffer.set(intBuffer)\n        newBuffer.set(tokenBytes, intBuffer.length)\n        intBuffer = newBuffer\n      }\n    }\n\n    if (intBuffer !== emptyBuffer) {\n      decoded += decoder.decode(intBuffer, { stream: true })\n    }\n    return decoded\n  }\n\n  async *decodeNativeAsyncIterable(\n    tokens: AsyncIterable<number>,\n  ): AsyncGenerator<Uint8Array | string> {\n    for await (const token of tokens) {\n      const tokenBytesOrString = this.tryDecodeToken(token)\n      if (tokenBytesOrString) {\n        yield tokenBytesOrString\n      }\n    }\n  }\n\n  private getBpeRankFromString(key: string): number | undefined {\n    return this.bytePairStringRankEncoder.get(key)\n  }\n\n  private getBpeRankFromStringOrThrow(key: string): number {\n    const value = this.getBpeRankFromString(key)\n    if (value === undefined) {\n      throw new Error(\n        `The byte-pair encoding does not contain a value for: ${key}`,\n      )\n    }\n    return value\n  }\n\n  private getBpeRankFromBytes(key: Uint8Array): number | undefined {\n    const keyAsString = tryConvertToString(key)\n\n    if (keyAsString !== undefined) {\n      return this.getBpeRankFromString(keyAsString)\n    }\n\n    // Perform binary search on the binary keys\n    const index = this.binarySearch(key)\n    if (index !== -1) {\n      return this.bytePairNonUtfSortedEncoder[index]![1]\n    }\n\n    return undefined\n  }\n\n  private getBpeRankFromBytesOrThrow(key: Uint8Array): number {\n    const value = this.getBpeRankFromBytes(key)\n    if (value === undefined) {\n      throw new Error(\n        `The byte-pair encoding does not contain a value for: ${key.toString()}`,\n      )\n    }\n    return value\n  }\n\n  // Binary search on the binary keys\n  private binarySearch(key: Uint8Array): number {\n    let low = 0\n    let high = this.bytePairNonUtfSortedEncoder.length - 1\n\n    while (low <= high) {\n      const mid = (low + high) >>> 1\n      const midKey = this.bytePairNonUtfSortedEncoder[mid]![0]\n      let cmp = 0\n      const maxLength = Math.min(midKey.length, key.length)\n      for (let i = 0; i < maxLength; i++) {\n        cmp = midKey[i]! - key[i]!\n        if (cmp !== 0) break\n      }\n      if (cmp === 0) {\n        cmp = midKey.length - key.length\n      }\n      if (cmp === 0) {\n        return mid\n      }\n      if (cmp < 0) {\n        low = mid + 1\n      } else {\n        high = mid - 1\n      }\n    }\n    return -1\n  }\n\n  private findNextSpecialToken(\n    text: string,\n    allowedSpecial: Set<string> | undefined,\n    startIndex: number,\n  ): [startIndex: number, token: string] | undefined {\n    let searchIndex = startIndex\n\n    while (true) {\n      this.specialTokenPatternRegex.lastIndex = searchIndex\n      const nextSpecialMatch = this.specialTokenPatternRegex.exec(text)\n\n      if (!nextSpecialMatch) {\n        return undefined\n      }\n\n      const specialToken = nextSpecialMatch[0]\n\n      if (allowedSpecial?.has(specialToken)) {\n        const specialTokenStartIndex = nextSpecialMatch.index + searchIndex\n        return [specialTokenStartIndex, specialToken]\n      }\n\n      searchIndex = nextSpecialMatch.index + searchIndex + 1\n    }\n  }\n\n  private tryDecodeToken(tokenRank: number): Uint8Array | string | undefined {\n    const value = this.bytePairRankDecoder[tokenRank]\n    if (typeof value === 'string') {\n      return value\n    }\n    if (typeof value === 'object') {\n      const fromBinary = this.bytePairNonUtfRankDecoder.get(tokenRank)\n      if (fromBinary) {\n        return fromBinary\n      }\n    }\n    return this.specialTokensDecoder.get(tokenRank)\n  }\n\n  private addToMergeCache(key: string, value: number[]): void {\n    if (!this.mergeCache) return\n\n    if (this.mergeCache.size >= this.mergeCacheSize) {\n      // Remove least recently used item (first item)\n      const firstKey = this.mergeCache.keys().next().value!\n      this.mergeCache.delete(firstKey)\n    }\n    this.mergeCache.set(key, value)\n  }\n\n  private bytePairEncode(input: string): number[] {\n    if (input.length === 1 && isAscii(input.codePointAt(0)!)) {\n      return [this.getBpeRankFromStringOrThrow(input)]\n    }\n\n    if (this.mergeCache?.has(input)) {\n      const result = this.mergeCache.get(input)!\n      // Move to end to mark as recently used\n      this.mergeCache.delete(input)\n      this.mergeCache.set(input, result)\n      return result\n    }\n\n    const inputBytes = this.textEncoder.encode(input)\n    const result = this.bytePairMerge(inputBytes)\n    this.addToMergeCache(input, result)\n    return result\n  }\n\n  private bytePairMerge(\n    // Input array of bytes to process\n    piece: Uint8Array,\n  ): number[] {\n    // 'starts' holds the start indices of each partition\n    const starts: number[] = []\n    // 'ranks' holds the BPE ranks of each partition pair\n    const ranks: number[] = []\n\n    // Helper function to get the rank of a byte pair starting at 'startIndex'\n    const getRank = (\n      startIndex: number,\n      pairStart = starts[startIndex],\n      pairEnd = starts[startIndex + 2],\n    ): number => {\n      if (pairEnd === undefined) {\n        // No valid pair exists\n        return Number.POSITIVE_INFINITY\n      }\n\n      // Extract the byte pair\n      const key = piece.subarray(pairStart, pairEnd)\n\n      // Retrieve the BPE rank of this byte pair (if it exists)\n      const rank = this.getBpeRankFromBytes(key)\n      return rank ?? Number.POSITIVE_INFINITY\n    }\n\n    // Initialize the 'starts' array with all possible start indices\n    for (let i = 0; i <= piece.length; i++) {\n      starts.push(i)\n      if (i < piece.length - 1) {\n        // Initialize the BPE values for all adjacent pairs\n        ranks.push(getRank(i, i, i + 2))\n      } else {\n        // Initialize BPE values to infinity for the last pair\n        ranks.push(Number.POSITIVE_INFINITY)\n      }\n    }\n\n    // Iteratively merge byte pairs until no more useful merges can be done\n    while (starts.length > 1) {\n      let lowestRank = Number.POSITIVE_INFINITY\n      let lowestPartitionIndex = -1\n\n      // Find the partition with the minimum rank\n      for (let i = 0; i < ranks.length - 1; i++) {\n        const rank = ranks[i]!\n        if (rank < lowestRank) {\n          lowestRank = rank\n          lowestPartitionIndex = i\n        }\n      }\n\n      // If no valid pair is left to merge, exit the loop\n      if (\n        lowestRank === Number.POSITIVE_INFINITY ||\n        lowestPartitionIndex === -1\n      ) {\n        break\n      }\n\n      // Merge the pair at 'lowestPartitionIndex' by removing the next start index\n      starts.splice(lowestPartitionIndex + 1, 1)\n      // Remove the BPE value of the merged pair\n      ranks.splice(lowestPartitionIndex, 1)\n\n      // Update the current merged pair's rank\n      ranks[lowestPartitionIndex] = getRank(lowestPartitionIndex)\n\n      // Update the rank of the previous pair, if it exists\n      if (lowestPartitionIndex > 0) {\n        ranks[lowestPartitionIndex - 1] = getRank(lowestPartitionIndex - 1)\n      }\n    }\n\n    // Create the final output by applying the transform function to each partitioned range\n    const output: number[] = []\n    for (let i = 0; i < starts.length - 1; i++) {\n      const pairStart = starts[i]\n      const pairEnd = starts[i + 1]\n      const bpeValue = this.getBpeRankFromBytesOrThrow(\n        piece.subarray(pairStart, pairEnd),\n      )\n      output.push(bpeValue)\n    }\n    return output\n  }\n}\n"],"mappings":";;;;;AAmBA,MAAM,8BAAc,IAAI,WAAW,CAAC;AAEpC,MAAa,UAAU,IAAI,YAAY,MAAM;AAE7C,IAAa,uBAAb,MAAkC;CAChC;;;;;;CAMA;CACA,4CAAoC,IAAI,IAAwB;CAChE;;;;;;CAMA;CACA;CACA;CACA;CACA;CACA,cAAsB,IAAI,YAAY;CACtC;CACA;CAEA,YAAY,EACV,qBACA,sBACA,iBACA,iBAAiBA,kBAAAA,4BACQ;EACzB,KAAK,sBAAsB;EAC3B,KAAK,4CAA4B,IAAI,IAAoB;EACzD,KAAK,iBAAiB;EACtB,IAAI,iBAAiB,GACnB,KAAK,6BAAa,IAAI,IAAI;EAI5B,KAAK,6BAA6B,OAAO,KAAK,mBAAmB,CAAC,CAAC;EACnE,MAAM,eAAuC,CAAC;EAE9C,oBAAoB,SAAS,OAAO,SAAS;GAC3C,IAAI,OAAO,UAAU,UAAU;IAC7B,KAAK,0BAA0B,IAAI,OAAO,IAAI;IAC9C;GACF;GACA,MAAM,YAAY,IAAI,WAAW,KAAK;GACtC,aAAa,KAAK,CAAC,WAAW,IAAI,CAAC;GACnC,KAAK,0BAA0B,IAAI,MAAM,SAAS;EACpD,CAAC;EACD,KAAK,8BAA8B,aAAa,MAAM,GAAG,MACvDC,gBAAAA,mBAAmB,EAAE,IAAI,EAAE,EAAE,CAC/B;EACA,KAAK,uBACH,wCAAwB,IAAI,IAAoB;EAClD,KAAK,uBAAuB,uBACxB,IAAI,IAAI,CAAC,GAAG,oBAAoB,CAAC,CAAC,KAAK,CAAC,KAAK,WAAW,CAAC,OAAO,GAAG,CAAC,CAAC,oBACrE,IAAI,IAAoB;EAC5B,KAAK,kBAAkB;EAKvB,MAAM,wBAHuB,CAAC,GAAG,KAAK,qBAAqB,KAAK,CAAC,CAAC,CAAC,IACjEC,aAAAA,YAE+C,CAAC,CAAC,KAAK,GAAG;EAC3D,IAAI;GACF,KAAK,2BAA2B,IAAI,OAAO,uBAAuB,GAAG;EACvE,QAAQ;GACN,MAAM,IAAI,MAAM,qCAAqC;EACvD;CACF;CAEA,kBAAkB,SAAuB;EACvC,IAAI,KAAK,mBAAmB,KAAK,UAAU,GACzC,KAAK,6BAAa,IAAI,IAAI;EAE5B,KAAK,iBAAiB;EACtB,IAAI,YAAY,GACd,KAAK,aAAa,KAAA;CAEtB;CAEA,kBAAwB;EACtB,KAAK,YAAY,MAAM;CACzB;CAEA,CAAC,sBACC,MACA,gBACwC;EACxC,IAAI,aAAa;EACjB,IAAI,kBAAkB;EAEtB,OAAO,MAAM;GACX,MAAM,mBAAmB,KAAK,qBAC5B,MACA,gBACA,UACF;GACA,MAAM,wBAAwB,mBAAmB;GAEjD,MAAM,WAAW,yBAAyB,KAAK;GAE/C,MAAM,oBACJ,eAAe,KAAK,aAAa,KAAK,SAClC,OACA,KAAK,MAAM,YAAY,QAAQ;GAErC,KAAK,MAAM,CAAC,UAAU,kBAAkB,SAAS,KAAK,eAAe,GAAG;IACtE,MAAM,QAAQ,KAAK,qBAAqB,KAAK;IAC7C,IAAI,UAAU,KAAA,GAAW;KACvB,kBAAkB;KAClB,MAAM,CAAC,KAAK;KAEZ;IACF;IAEA,MAAM,SAAS,KAAK,eAAe,KAAK;IACxC,kBAAkB,OAAO;IACzB,MAAM;GACR;GAEA,IAAI,0BAA0B,KAAA,GAAW;IACvC,MAAM,eAAe,iBAAkB;IACvC,MAAM,oBAAoB,KAAK,qBAAqB,IAAI,YAAY;IACpE,IAAI,sBAAsB,KAAA,GACxB,MAAM,IAAI,MACR,kBAAkB,aAAa,uCACjC;IAEF,MAAM,CAAC,iBAAiB;IACxB,aAAa,wBAAwB,aAAa;IAClD,kBAAkB;GACpB,OACE;EAEJ;EAEA,OAAO;CACT;CAEA,aAAa,MAAc,gBAAwC;EACjE,IAAI,aAAa;EACjB,MAAM,cAAwB,CAAC;EAE/B,OAAO,MAAM;GACX,MAAM,mBAAmB,KAAK,qBAC5B,MACA,gBACA,UACF;GACA,MAAM,wBAAwB,mBAAmB;GAEjD,MAAM,WAAW,yBAAyB,KAAK;GAE/C,MAAM,oBACJ,eAAe,KAAK,aAAa,KAAK,SAClC,OACA,KAAK,MAAM,YAAY,QAAQ;GAErC,KAAK,MAAM,CAAC,UAAU,kBAAkB,SAAS,KAAK,eAAe,GAAG;IACtE,MAAM,QAAQ,KAAK,qBAAqB,KAAK;IAC7C,IAAI,UAAU,KAAA,GAAW;KACvB,YAAY,KAAK,KAAK;KAEtB;IACF;IAEA,MAAM,SAAS,KAAK,eAAe,KAAK;IACxC,YAAY,KAAK,GAAG,MAAM;GAC5B;GAEA,IAAI,0BAA0B,KAAA,GAAW;IACvC,MAAM,eAAe,iBAAkB;IACvC,MAAM,oBAAoB,KAAK,qBAAqB,IAAI,YAAY;IACpE,IAAI,sBAAsB,KAAA,GACxB,MAAM,IAAI,MACR,kBAAkB,aAAa,uCACjC;IAEF,YAAY,KAAK,iBAAiB;IAClC,aAAa,wBAAwB,aAAa;GACpD,OACE;EAEJ;EAEA,OAAO;CACT;CAEA,YAAY,MAAc,gBAAsC;EAC9D,IAAI,aAAa;EACjB,IAAI,cAAc;EAElB,OAAO,MAAM;GACX,MAAM,mBAAmB,KAAK,qBAC5B,MACA,gBACA,UACF;GACA,MAAM,wBAAwB,mBAAmB;GAEjD,MAAM,WAAW,yBAAyB,KAAK;GAE/C,MAAM,oBACJ,eAAe,KAAK,aAAa,KAAK,SAClC,OACA,KAAK,MAAM,YAAY,QAAQ;GAErC,KAAK,MAAM,CAAC,UAAU,kBAAkB,SAAS,KAAK,eAAe,GAAG;IAEtE,IADc,KAAK,qBAAqB,KAChC,MAAM,KAAA,GAAW;KACvB;KAEA;IACF;IAEA,MAAM,SAAS,KAAK,eAAe,KAAK;IACxC,eAAe,OAAO;GACxB;GAEA,IAAI,0BAA0B,KAAA,GAAW;IACvC,MAAM,eAAe,iBAAkB;IAEvC,IAD0B,KAAK,qBAAqB,IAAI,YACpC,MAAM,KAAA,GACxB,MAAM,IAAI,MACR,kBAAkB,aAAa,uCACjC;IAEF;IACA,aAAa,wBAAwB,aAAa;GACpD,OACE;EAEJ;EAEA,OAAO;CACT;CAEA,CAAC,sBACC,QAC4C;EAC5C,KAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,aAAa,KAAK,eAAe,KAAK;GAC5C,IAAI,YACF,MAAM;EAEV;CACF;CAEA,aAAa,QAAkC;EAC7C,IAAI,UAAU;EACd,IAAI,YAAY;EAEhB,KAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,aAAa,KAAK,eAAe,KAAK;GAC5C,IAAI,eAAe,KAAA,GACjB,MAAM,IAAI,MAAM,SAAS,MAAM,kCAAkC;GAEnE,IAAI,OAAO,eAAe,UAAU;IAClC,IAAI,cAAc,aAAa;KAC7B,WAAW,QAAQ,OAAO,WAAW,EAAE,QAAQ,KAAK,CAAC;KACrD,YAAY;IACd;IACA,WAAW;GACb,OAAO;IACL,MAAM,YAAY,IAAI,WAAW,UAAU,SAAS,WAAW,MAAM;IACrE,UAAU,IAAI,SAAS;IACvB,UAAU,IAAI,YAAY,UAAU,MAAM;IAC1C,YAAY;GACd;EACF;EAEA,IAAI,cAAc,aAChB,WAAW,QAAQ,OAAO,WAAW,EAAE,QAAQ,KAAK,CAAC;EAEvD,OAAO;CACT;CAEA,OAAO,0BACL,QACqC;EACrC,WAAW,MAAM,SAAS,QAAQ;GAChC,MAAM,qBAAqB,KAAK,eAAe,KAAK;GACpD,IAAI,oBACF,MAAM;EAEV;CACF;CAEA,qBAA6B,KAAiC;EAC5D,OAAO,KAAK,0BAA0B,IAAI,GAAG;CAC/C;CAEA,4BAAoC,KAAqB;EACvD,MAAM,QAAQ,KAAK,qBAAqB,GAAG;EAC3C,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,MACR,wDAAwD,KAC1D;EAEF,OAAO;CACT;CAEA,oBAA4B,KAAqC;EAC/D,MAAM,cAAcC,gBAAAA,mBAAmB,GAAG;EAE1C,IAAI,gBAAgB,KAAA,GAClB,OAAO,KAAK,qBAAqB,WAAW;EAI9C,MAAM,QAAQ,KAAK,aAAa,GAAG;EACnC,IAAI,UAAU,IACZ,OAAO,KAAK,4BAA4B,MAAM,CAAE;CAIpD;CAEA,2BAAmC,KAAyB;EAC1D,MAAM,QAAQ,KAAK,oBAAoB,GAAG;EAC1C,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,MACR,wDAAwD,IAAI,SAAS,GACvE;EAEF,OAAO;CACT;CAGA,aAAqB,KAAyB;EAC5C,IAAI,MAAM;EACV,IAAI,OAAO,KAAK,4BAA4B,SAAS;EAErD,OAAO,OAAO,MAAM;GAClB,MAAM,MAAO,MAAM,SAAU;GAC7B,MAAM,SAAS,KAAK,4BAA4B,IAAI,CAAE;GACtD,IAAI,MAAM;GACV,MAAM,YAAY,KAAK,IAAI,OAAO,QAAQ,IAAI,MAAM;GACpD,KAAK,IAAI,IAAI,GAAG,IAAI,WAAW,KAAK;IAClC,MAAM,OAAO,KAAM,IAAI;IACvB,IAAI,QAAQ,GAAG;GACjB;GACA,IAAI,QAAQ,GACV,MAAM,OAAO,SAAS,IAAI;GAE5B,IAAI,QAAQ,GACV,OAAO;GAET,IAAI,MAAM,GACR,MAAM,MAAM;QAEZ,OAAO,MAAM;EAEjB;EACA,OAAO;CACT;CAEA,qBACE,MACA,gBACA,YACiD;EACjD,IAAI,cAAc;EAElB,OAAO,MAAM;GACX,KAAK,yBAAyB,YAAY;GAC1C,MAAM,mBAAmB,KAAK,yBAAyB,KAAK,IAAI;GAEhE,IAAI,CAAC,kBACH;GAGF,MAAM,eAAe,iBAAiB;GAEtC,IAAI,gBAAgB,IAAI,YAAY,GAElC,OAAO,CADwB,iBAAiB,QAAQ,aACxB,YAAY;GAG9C,cAAc,iBAAiB,QAAQ,cAAc;EACvD;CACF;CAEA,eAAuB,WAAoD;EACzE,MAAM,QAAQ,KAAK,oBAAoB;EACvC,IAAI,OAAO,UAAU,UACnB,OAAO;EAET,IAAI,OAAO,UAAU,UAAU;GAC7B,MAAM,aAAa,KAAK,0BAA0B,IAAI,SAAS;GAC/D,IAAI,YACF,OAAO;EAEX;EACA,OAAO,KAAK,qBAAqB,IAAI,SAAS;CAChD;CAEA,gBAAwB,KAAa,OAAuB;EAC1D,IAAI,CAAC,KAAK,YAAY;EAEtB,IAAI,KAAK,WAAW,QAAQ,KAAK,gBAAgB;GAE/C,MAAM,WAAW,KAAK,WAAW,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC;GAC/C,KAAK,WAAW,OAAO,QAAQ;EACjC;EACA,KAAK,WAAW,IAAI,KAAK,KAAK;CAChC;CAEA,eAAuB,OAAyB;EAC9C,IAAI,MAAM,WAAW,KAAKC,gBAAAA,QAAQ,MAAM,YAAY,CAAC,CAAE,GACrD,OAAO,CAAC,KAAK,4BAA4B,KAAK,CAAC;EAGjD,IAAI,KAAK,YAAY,IAAI,KAAK,GAAG;GAC/B,MAAM,SAAS,KAAK,WAAW,IAAI,KAAK;GAExC,KAAK,WAAW,OAAO,KAAK;GAC5B,KAAK,WAAW,IAAI,OAAO,MAAM;GACjC,OAAO;EACT;EAEA,MAAM,aAAa,KAAK,YAAY,OAAO,KAAK;EAChD,MAAM,SAAS,KAAK,cAAc,UAAU;EAC5C,KAAK,gBAAgB,OAAO,MAAM;EAClC,OAAO;CACT;CAEA,cAEE,OACU;EAEV,MAAM,SAAmB,CAAC;EAE1B,MAAM,QAAkB,CAAC;EAGzB,MAAM,WACJ,YACA,YAAY,OAAO,aACnB,UAAU,OAAO,aAAa,OACnB;GACX,IAAI,YAAY,KAAA,GAEd,OAAO,OAAO;GAIhB,MAAM,MAAM,MAAM,SAAS,WAAW,OAAO;GAI7C,OADa,KAAK,oBAAoB,GAC5B,KAAK,OAAO;EACxB;EAGA,KAAK,IAAI,IAAI,GAAG,KAAK,MAAM,QAAQ,KAAK;GACtC,OAAO,KAAK,CAAC;GACb,IAAI,IAAI,MAAM,SAAS,GAErB,MAAM,KAAK,QAAQ,GAAG,GAAG,IAAI,CAAC,CAAC;QAG/B,MAAM,KAAK,OAAO,iBAAiB;EAEvC;EAGA,OAAO,OAAO,SAAS,GAAG;GACxB,IAAI,aAAa,OAAO;GACxB,IAAI,uBAAuB;GAG3B,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,SAAS,GAAG,KAAK;IACzC,MAAM,OAAO,MAAM;IACnB,IAAI,OAAO,YAAY;KACrB,aAAa;KACb,uBAAuB;IACzB;GACF;GAGA,IACE,eAAe,OAAO,qBACtB,yBAAyB,IAEzB;GAIF,OAAO,OAAO,uBAAuB,GAAG,CAAC;GAEzC,MAAM,OAAO,sBAAsB,CAAC;GAGpC,MAAM,wBAAwB,QAAQ,oBAAoB;GAG1D,IAAI,uBAAuB,GACzB,MAAM,uBAAuB,KAAK,QAAQ,uBAAuB,CAAC;EAEtE;EAGA,MAAM,SAAmB,CAAC;EAC1B,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,SAAS,GAAG,KAAK;GAC1C,MAAM,YAAY,OAAO;GACzB,MAAM,UAAU,OAAO,IAAI;GAC3B,MAAM,WAAW,KAAK,2BACpB,MAAM,SAAS,WAAW,OAAO,CACnC;GACA,OAAO,KAAK,QAAQ;EACtB;EACA,OAAO;CACT;AACF"}
+{"version":3,"file":"BytePairEncodingCore.js","names":["DEFAULT_MERGE_CACHE_SIZE","startsWithByteOrderMark","tryConvertToString","compareUint8Arrays","escapeRegExp","isAscii"],"sources":["../src/BytePairEncodingCore.ts"],"sourcesContent":["import { DEFAULT_MERGE_CACHE_SIZE } from './constants.js'\nimport {\n  compareUint8Arrays,\n  isAscii,\n  startsWithByteOrderMark,\n  tryConvertToString,\n} from './utfUtil.js'\nimport { escapeRegExp } from './util.js'\n\nexport type RawBytePairRanks = readonly (string | readonly number[])[]\n\nexport interface BytePairEncodingConfig {\n  bytePairRankDecoder: RawBytePairRanks\n  specialTokensEncoder?: Map<string, number>\n  tokenSplitRegex: RegExp\n  /**\n   * LRU cache for merged tokens pairs.\n   * Increasing this value should make encoding similar strings faster,\n   * but will consume more memory.\n   * @default 100000\n   */\n  mergeCacheSize?: number\n}\n\nconst emptyBuffer = new Uint8Array(0)\n\n// `ignoreBOM: true` keeps a leading U+FEFF in the decoded output.\nexport const decoder = new TextDecoder('utf8', { ignoreBOM: true })\n\nexport class BytePairEncodingCore {\n  readonly mergeableBytePairRankCount: number\n  /**\n   * an array where the index is the BPE rank,\n   * and the value is the string or the array of bytes that it decodes to\n   * it may contain holes if token is unused\n   */\n  private bytePairRankDecoder: RawBytePairRanks\n  private bytePairNonUtfRankDecoder = new Map<number, Uint8Array>()\n  private bytePairNonUtfSortedEncoder: readonly [Uint8Array, number][]\n  /**\n   * a reverse map of the bytePairRankDecoder,\n   * where the key is the string and the value is the rank\n   * values that cannot be represented as a string are present in `bytePairNonUtfSortedEncoder`\n   */\n  private bytePairStringRankEncoder: Map<string, number>\n  private tokenSplitRegex: RegExp\n  private specialTokensEncoder: Map<string, number>\n  private specialTokensDecoder: Map<number, string>\n  private specialTokenPatternRegex: RegExp\n  private textEncoder = new TextEncoder()\n  private mergeCache?: Map<string, number[]>\n  private mergeCacheSize: number\n\n  constructor({\n    bytePairRankDecoder,\n    specialTokensEncoder,\n    tokenSplitRegex,\n    mergeCacheSize = DEFAULT_MERGE_CACHE_SIZE,\n  }: BytePairEncodingConfig) {\n    this.bytePairRankDecoder = bytePairRankDecoder\n    this.bytePairStringRankEncoder = new Map<string, number>()\n    this.mergeCacheSize = mergeCacheSize\n    if (mergeCacheSize > 0) {\n      this.mergeCache = new Map()\n    }\n\n    // size without array holes (which may be present in the encoder)\n    this.mergeableBytePairRankCount = Object.keys(bytePairRankDecoder).length\n    const binaryLookup: [Uint8Array, number][] = []\n    // forEach skips array holes:\n    bytePairRankDecoder.forEach((value, rank) => {\n      if (typeof value === 'string') {\n        this.bytePairStringRankEncoder.set(value, rank)\n        return\n      }\n      const byteArray = new Uint8Array(value)\n      binaryLookup.push([byteArray, rank])\n      this.bytePairNonUtfRankDecoder.set(rank, byteArray)\n      if (startsWithByteOrderMark(byteArray)) {\n        // The rank tables keep tokens that start with a UTF-8 byte order mark\n        // (EF BB BF) as raw bytes, because a plain decode would strip the mark.\n        // They are still valid UTF-8, so they must also be reachable through\n        // the string lookup that getBpeRankFromBytes uses for valid UTF-8.\n        const asString = tryConvertToString(byteArray)\n        if (asString !== undefined) {\n          this.bytePairStringRankEncoder.set(asString, rank)\n        }\n      }\n    })\n    this.bytePairNonUtfSortedEncoder = binaryLookup.sort((a, b) =>\n      compareUint8Arrays(a[0], b[0]),\n    )\n    this.specialTokensEncoder =\n      specialTokensEncoder ?? new Map<string, number>()\n    this.specialTokensDecoder = specialTokensEncoder\n      ? new Map([...specialTokensEncoder].map(([key, value]) => [value, key]))\n      : new Map<number, string>()\n    this.tokenSplitRegex = tokenSplitRegex\n\n    const escapedSpecialTokens = [...this.specialTokensEncoder.keys()].map(\n      escapeRegExp,\n    )\n    const allSpecialTokensRegex = escapedSpecialTokens.join('|')\n    try {\n      this.specialTokenPatternRegex = new RegExp(allSpecialTokensRegex, 'y')\n    } catch {\n      throw new Error('Invalid regular expression pattern.')\n    }\n  }\n\n  setMergeCacheSize(newSize: number): void {\n    if (this.mergeCacheSize === 0 && newSize > 0) {\n      this.mergeCache = new Map()\n    }\n    this.mergeCacheSize = newSize\n    if (newSize === 0) {\n      this.mergeCache = undefined\n    }\n  }\n\n  clearMergeCache(): void {\n    this.mergeCache?.clear()\n  }\n\n  *encodeNativeGenerator(\n    text: string,\n    allowedSpecial?: Set<string>,\n  ): Generator<number[], number, undefined> {\n    let startIndex = 0\n    let lastTokenLength = 0\n\n    while (true) {\n      const nextSpecialMatch = this.findNextSpecialToken(\n        text,\n        allowedSpecial,\n        startIndex,\n      )\n      const nextSpecialStartIndex = nextSpecialMatch?.[0]\n\n      const endIndex = nextSpecialStartIndex ?? text.length\n\n      const textBeforeSpecial =\n        startIndex === 0 && endIndex === text.length\n          ? text\n          : text.slice(startIndex, endIndex)\n\n      for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {\n        const token = this.getBpeRankFromString(match)\n        if (token !== undefined) {\n          lastTokenLength = 1\n          yield [token]\n\n          continue\n        }\n\n        const tokens = this.bytePairEncode(match)\n        lastTokenLength = tokens.length\n        yield tokens\n      }\n\n      if (nextSpecialStartIndex !== undefined) {\n        const specialToken = nextSpecialMatch![1]\n        const specialTokenValue = this.specialTokensEncoder.get(specialToken)\n        if (specialTokenValue === undefined) {\n          throw new Error(\n            `Special token \"${specialToken}\" is not in the special token encoder.`,\n          )\n        }\n        yield [specialTokenValue]\n        startIndex = nextSpecialStartIndex + specialToken.length\n        lastTokenLength = 1\n      } else {\n        break\n      }\n    }\n\n    return lastTokenLength\n  }\n\n  encodeNative(text: string, allowedSpecial?: Set<string>): number[] {\n    let startIndex = 0\n    const tokensArray: number[] = [] // Flat list to collect the tokens\n\n    while (true) {\n      const nextSpecialMatch = this.findNextSpecialToken(\n        text,\n        allowedSpecial,\n        startIndex,\n      )\n      const nextSpecialStartIndex = nextSpecialMatch?.[0]\n\n      const endIndex = nextSpecialStartIndex ?? text.length\n\n      const textBeforeSpecial =\n        startIndex === 0 && endIndex === text.length\n          ? text\n          : text.slice(startIndex, endIndex)\n\n      for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {\n        const token = this.getBpeRankFromString(match)\n        if (token !== undefined) {\n          tokensArray.push(token)\n\n          continue\n        }\n\n        const tokens = this.bytePairEncode(match)\n        tokensArray.push(...tokens)\n      }\n\n      if (nextSpecialStartIndex !== undefined) {\n        const specialToken = nextSpecialMatch![1]\n        const specialTokenValue = this.specialTokensEncoder.get(specialToken)\n        if (specialTokenValue === undefined) {\n          throw new Error(\n            `Special token \"${specialToken}\" is not in the special token encoder.`,\n          )\n        }\n        tokensArray.push(specialTokenValue)\n        startIndex = nextSpecialStartIndex + specialToken.length\n      } else {\n        break\n      }\n    }\n\n    return tokensArray\n  }\n\n  countNative(text: string, allowedSpecial?: Set<string>): number {\n    let startIndex = 0\n    let tokensCount = 0\n\n    while (true) {\n      const nextSpecialMatch = this.findNextSpecialToken(\n        text,\n        allowedSpecial,\n        startIndex,\n      )\n      const nextSpecialStartIndex = nextSpecialMatch?.[0]\n\n      const endIndex = nextSpecialStartIndex ?? text.length\n\n      const textBeforeSpecial =\n        startIndex === 0 && endIndex === text.length\n          ? text\n          : text.slice(startIndex, endIndex)\n\n      for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {\n        const token = this.getBpeRankFromString(match)\n        if (token !== undefined) {\n          tokensCount++\n\n          continue\n        }\n\n        const tokens = this.bytePairEncode(match)\n        tokensCount += tokens.length\n      }\n\n      if (nextSpecialStartIndex !== undefined) {\n        const specialToken = nextSpecialMatch![1]\n        const specialTokenValue = this.specialTokensEncoder.get(specialToken)\n        if (specialTokenValue === undefined) {\n          throw new Error(\n            `Special token \"${specialToken}\" is not in the special token encoder.`,\n          )\n        }\n        tokensCount++\n        startIndex = nextSpecialStartIndex + specialToken.length\n      } else {\n        break\n      }\n    }\n\n    return tokensCount\n  }\n\n  *decodeNativeGenerator(\n    tokens: Iterable<number>,\n  ): Generator<Uint8Array | string, void, void> {\n    for (const token of tokens) {\n      const tokenBytes = this.tryDecodeToken(token)\n      if (tokenBytes) {\n        yield tokenBytes\n      }\n    }\n  }\n\n  decodeNative(tokens: Iterable<number>): string {\n    let decoded = ''\n    let intBuffer = emptyBuffer\n\n    for (const token of tokens) {\n      const tokenBytes = this.tryDecodeToken(token)\n      if (tokenBytes === undefined) {\n        throw new Error(`Token ${token} is not in the byte pair encoder.`)\n      }\n      if (typeof tokenBytes === 'string') {\n        if (intBuffer !== emptyBuffer) {\n          decoded += decoder.decode(intBuffer, { stream: true })\n          intBuffer = emptyBuffer\n        }\n        decoded += tokenBytes\n      } else {\n        const newBuffer = new Uint8Array(intBuffer.length + tokenBytes.length)\n        newBuffer.set(intBuffer)\n        newBuffer.set(tokenBytes, intBuffer.length)\n        intBuffer = newBuffer\n      }\n    }\n\n    if (intBuffer !== emptyBuffer) {\n      decoded += decoder.decode(intBuffer, { stream: true })\n    }\n    return decoded\n  }\n\n  async *decodeNativeAsyncIterable(\n    tokens: AsyncIterable<number>,\n  ): AsyncGenerator<Uint8Array | string> {\n    for await (const token of tokens) {\n      const tokenBytesOrString = this.tryDecodeToken(token)\n      if (tokenBytesOrString) {\n        yield tokenBytesOrString\n      }\n    }\n  }\n\n  private getBpeRankFromString(key: string): number | undefined {\n    return this.bytePairStringRankEncoder.get(key)\n  }\n\n  private getBpeRankFromStringOrThrow(key: string): number {\n    const value = this.getBpeRankFromString(key)\n    if (value === undefined) {\n      throw new Error(\n        `The byte-pair encoding does not contain a value for: ${key}`,\n      )\n    }\n    return value\n  }\n\n  private getBpeRankFromBytes(key: Uint8Array): number | undefined {\n    const keyAsString = tryConvertToString(key)\n\n    if (keyAsString !== undefined) {\n      return this.getBpeRankFromString(keyAsString)\n    }\n\n    // Perform binary search on the binary keys\n    const index = this.binarySearch(key)\n    if (index !== -1) {\n      return this.bytePairNonUtfSortedEncoder[index]![1]\n    }\n\n    return undefined\n  }\n\n  private getBpeRankFromBytesOrThrow(key: Uint8Array): number {\n    const value = this.getBpeRankFromBytes(key)\n    if (value === undefined) {\n      throw new Error(\n        `The byte-pair encoding does not contain a value for: ${key.toString()}`,\n      )\n    }\n    return value\n  }\n\n  // Binary search on the binary keys\n  private binarySearch(key: Uint8Array): number {\n    let low = 0\n    let high = this.bytePairNonUtfSortedEncoder.length - 1\n\n    while (low <= high) {\n      const mid = (low + high) >>> 1\n      const midKey = this.bytePairNonUtfSortedEncoder[mid]![0]\n      let cmp = 0\n      const maxLength = Math.min(midKey.length, key.length)\n      for (let i = 0; i < maxLength; i++) {\n        cmp = midKey[i]! - key[i]!\n        if (cmp !== 0) break\n      }\n      if (cmp === 0) {\n        cmp = midKey.length - key.length\n      }\n      if (cmp === 0) {\n        return mid\n      }\n      if (cmp < 0) {\n        low = mid + 1\n      } else {\n        high = mid - 1\n      }\n    }\n    return -1\n  }\n\n  private findNextSpecialToken(\n    text: string,\n    allowedSpecial: Set<string> | undefined,\n    startIndex: number,\n  ): [startIndex: number, token: string] | undefined {\n    let searchIndex = startIndex\n\n    while (true) {\n      this.specialTokenPatternRegex.lastIndex = searchIndex\n      const nextSpecialMatch = this.specialTokenPatternRegex.exec(text)\n\n      if (!nextSpecialMatch) {\n        return undefined\n      }\n\n      const specialToken = nextSpecialMatch[0]\n\n      if (allowedSpecial?.has(specialToken)) {\n        const specialTokenStartIndex = nextSpecialMatch.index + searchIndex\n        return [specialTokenStartIndex, specialToken]\n      }\n\n      searchIndex = nextSpecialMatch.index + searchIndex + 1\n    }\n  }\n\n  private tryDecodeToken(tokenRank: number): Uint8Array | string | undefined {\n    const value = this.bytePairRankDecoder[tokenRank]\n    if (typeof value === 'string') {\n      return value\n    }\n    if (typeof value === 'object') {\n      const fromBinary = this.bytePairNonUtfRankDecoder.get(tokenRank)\n      if (fromBinary) {\n        return fromBinary\n      }\n    }\n    return this.specialTokensDecoder.get(tokenRank)\n  }\n\n  private addToMergeCache(key: string, value: number[]): void {\n    if (!this.mergeCache) return\n\n    if (this.mergeCache.size >= this.mergeCacheSize) {\n      // Remove least recently used item (first item)\n      const firstKey = this.mergeCache.keys().next().value!\n      this.mergeCache.delete(firstKey)\n    }\n    this.mergeCache.set(key, value)\n  }\n\n  private bytePairEncode(input: string): number[] {\n    if (input.length === 1 && isAscii(input.codePointAt(0)!)) {\n      return [this.getBpeRankFromStringOrThrow(input)]\n    }\n\n    if (this.mergeCache?.has(input)) {\n      const result = this.mergeCache.get(input)!\n      // Move to end to mark as recently used\n      this.mergeCache.delete(input)\n      this.mergeCache.set(input, result)\n      return result\n    }\n\n    const inputBytes = this.textEncoder.encode(input)\n    const result = this.bytePairMerge(inputBytes)\n    this.addToMergeCache(input, result)\n    return result\n  }\n\n  private bytePairMerge(\n    // Input array of bytes to process\n    piece: Uint8Array,\n  ): number[] {\n    const { length } = piece\n    // Partitions form a linked list over byte offsets: a live partition starting at\n    // offset 'i' spans [i, next[i]). 'pairRanks[i]' is the rank of merging it with\n    // the partition after it. Merges come off a min-heap ordered by (rank, offset),\n    // which picks the same pair as scanning for the lowest rank, leftmost first,\n    // without rescanning the piece after every merge.\n    const next = new Int32Array(length + 1)\n    const previous = new Int32Array(length + 1)\n    const pairRanks = new Float64Array(length + 1)\n    const live = new Uint8Array(length + 1)\n\n    const getPairRank = (startIndex: number): number => {\n      const middle = next[startIndex]!\n      if (middle >= length) return Number.POSITIVE_INFINITY\n      const rank = this.getBpeRankFromBytes(\n        piece.subarray(startIndex, next[middle]!),\n      )\n      return rank ?? Number.POSITIVE_INFINITY\n    }\n\n    // Heap entries are packed as [rank, startIndex] pairs in a flat array\n    const heap: number[] = []\n    const isLower = (a: number, b: number) =>\n      heap[a]! < heap[b]! ||\n      (heap[a] === heap[b] && heap[a + 1]! < heap[b + 1]!)\n    const swap = (a: number, b: number) => {\n      const rank = heap[a]!\n      const startIndex = heap[a + 1]!\n      heap[a] = heap[b]!\n      heap[a + 1] = heap[b + 1]!\n      heap[b] = rank\n      heap[b + 1] = startIndex\n    }\n    const push = (rank: number, startIndex: number) => {\n      heap.push(rank, startIndex)\n      let index = heap.length - 2\n      while (index > 0) {\n        const parent = (((index >> 1) - 1) >> 1) << 1\n        if (!isLower(index, parent)) break\n        swap(index, parent)\n        index = parent\n      }\n    }\n    const pop = () => {\n      const last = heap.length - 2\n      swap(0, last)\n      const rank = heap[last]!\n      const startIndex = heap[last + 1]!\n      heap.length = last\n      let index = 0\n      while (true) {\n        const left = index * 2 + 2\n        const right = left + 2\n        let lowest = index\n        if (left < heap.length && isLower(left, lowest)) lowest = left\n        if (right < heap.length && isLower(right, lowest)) lowest = right\n        if (lowest === index) break\n        swap(index, lowest)\n        index = lowest\n      }\n      return [rank, startIndex] as const\n    }\n\n    for (let i = 0; i <= length; i++) {\n      next[i] = i + 1\n      previous[i] = i - 1\n      live[i] = 1\n    }\n    for (let i = 0; i < length - 1; i++) {\n      pairRanks[i] = getPairRank(i)\n      if (pairRanks[i] !== Number.POSITIVE_INFINITY) push(pairRanks[i]!, i)\n    }\n\n    // Iteratively merge byte pairs until no more useful merges can be done\n    while (heap.length > 0) {\n      const [rank, startIndex] = pop()\n      // Skip entries made stale by an earlier merge\n      if (!live[startIndex] || pairRanks[startIndex] !== rank) continue\n\n      // Merge the partition at 'startIndex' with the one after it\n      const merged = next[startIndex]!\n      live[merged] = 0\n      next[startIndex] = next[merged]!\n      previous[next[merged]!] = startIndex\n\n      pairRanks[startIndex] = getPairRank(startIndex)\n      if (pairRanks[startIndex] !== Number.POSITIVE_INFINITY) {\n        push(pairRanks[startIndex]!, startIndex)\n      }\n\n      const before = previous[startIndex]!\n      if (before >= 0) {\n        pairRanks[before] = getPairRank(before)\n        if (pairRanks[before] !== Number.POSITIVE_INFINITY) {\n          push(pairRanks[before]!, before)\n        }\n      }\n    }\n\n    // Create the final output by applying the transform function to each partitioned range\n    const output: number[] = []\n    for (let i = 0; i < length; i = next[i]!) {\n      output.push(this.getBpeRankFromBytesOrThrow(piece.subarray(i, next[i]!)))\n    }\n    return output\n  }\n}\n"],"mappings":";;;;;AAwBA,MAAM,8BAAc,IAAI,WAAW,CAAC;AAGpC,MAAa,UAAU,IAAI,YAAY,QAAQ,EAAE,WAAW,KAAK,CAAC;AAElE,IAAa,uBAAb,MAAkC;CAChC;;;;;;CAMA;CACA,4CAAoC,IAAI,IAAwB;CAChE;;;;;;CAMA;CACA;CACA;CACA;CACA;CACA,cAAsB,IAAI,YAAY;CACtC;CACA;CAEA,YAAY,EACV,qBACA,sBACA,iBACA,iBAAiBA,kBAAAA,4BACQ;EACzB,KAAK,sBAAsB;EAC3B,KAAK,4CAA4B,IAAI,IAAoB;EACzD,KAAK,iBAAiB;EACtB,IAAI,iBAAiB,GACnB,KAAK,6BAAa,IAAI,IAAI;EAI5B,KAAK,6BAA6B,OAAO,KAAK,mBAAmB,CAAC,CAAC;EACnE,MAAM,eAAuC,CAAC;EAE9C,oBAAoB,SAAS,OAAO,SAAS;GAC3C,IAAI,OAAO,UAAU,UAAU;IAC7B,KAAK,0BAA0B,IAAI,OAAO,IAAI;IAC9C;GACF;GACA,MAAM,YAAY,IAAI,WAAW,KAAK;GACtC,aAAa,KAAK,CAAC,WAAW,IAAI,CAAC;GACnC,KAAK,0BAA0B,IAAI,MAAM,SAAS;GAClD,IAAIC,gBAAAA,wBAAwB,SAAS,GAAG;IAKtC,MAAM,WAAWC,gBAAAA,mBAAmB,SAAS;IAC7C,IAAI,aAAa,KAAA,GACf,KAAK,0BAA0B,IAAI,UAAU,IAAI;GAErD;EACF,CAAC;EACD,KAAK,8BAA8B,aAAa,MAAM,GAAG,MACvDC,gBAAAA,mBAAmB,EAAE,IAAI,EAAE,EAAE,CAC/B;EACA,KAAK,uBACH,wCAAwB,IAAI,IAAoB;EAClD,KAAK,uBAAuB,uBACxB,IAAI,IAAI,CAAC,GAAG,oBAAoB,CAAC,CAAC,KAAK,CAAC,KAAK,WAAW,CAAC,OAAO,GAAG,CAAC,CAAC,oBACrE,IAAI,IAAoB;EAC5B,KAAK,kBAAkB;EAKvB,MAAM,wBAHuB,CAAC,GAAG,KAAK,qBAAqB,KAAK,CAAC,CAAC,CAAC,IACjEC,aAAAA,YAE+C,CAAC,CAAC,KAAK,GAAG;EAC3D,IAAI;GACF,KAAK,2BAA2B,IAAI,OAAO,uBAAuB,GAAG;EACvE,QAAQ;GACN,MAAM,IAAI,MAAM,qCAAqC;EACvD;CACF;CAEA,kBAAkB,SAAuB;EACvC,IAAI,KAAK,mBAAmB,KAAK,UAAU,GACzC,KAAK,6BAAa,IAAI,IAAI;EAE5B,KAAK,iBAAiB;EACtB,IAAI,YAAY,GACd,KAAK,aAAa,KAAA;CAEtB;CAEA,kBAAwB;EACtB,KAAK,YAAY,MAAM;CACzB;CAEA,CAAC,sBACC,MACA,gBACwC;EACxC,IAAI,aAAa;EACjB,IAAI,kBAAkB;EAEtB,OAAO,MAAM;GACX,MAAM,mBAAmB,KAAK,qBAC5B,MACA,gBACA,UACF;GACA,MAAM,wBAAwB,mBAAmB;GAEjD,MAAM,WAAW,yBAAyB,KAAK;GAE/C,MAAM,oBACJ,eAAe,KAAK,aAAa,KAAK,SAClC,OACA,KAAK,MAAM,YAAY,QAAQ;GAErC,KAAK,MAAM,CAAC,UAAU,kBAAkB,SAAS,KAAK,eAAe,GAAG;IACtE,MAAM,QAAQ,KAAK,qBAAqB,KAAK;IAC7C,IAAI,UAAU,KAAA,GAAW;KACvB,kBAAkB;KAClB,MAAM,CAAC,KAAK;KAEZ;IACF;IAEA,MAAM,SAAS,KAAK,eAAe,KAAK;IACxC,kBAAkB,OAAO;IACzB,MAAM;GACR;GAEA,IAAI,0BAA0B,KAAA,GAAW;IACvC,MAAM,eAAe,iBAAkB;IACvC,MAAM,oBAAoB,KAAK,qBAAqB,IAAI,YAAY;IACpE,IAAI,sBAAsB,KAAA,GACxB,MAAM,IAAI,MACR,kBAAkB,aAAa,uCACjC;IAEF,MAAM,CAAC,iBAAiB;IACxB,aAAa,wBAAwB,aAAa;IAClD,kBAAkB;GACpB,OACE;EAEJ;EAEA,OAAO;CACT;CAEA,aAAa,MAAc,gBAAwC;EACjE,IAAI,aAAa;EACjB,MAAM,cAAwB,CAAC;EAE/B,OAAO,MAAM;GACX,MAAM,mBAAmB,KAAK,qBAC5B,MACA,gBACA,UACF;GACA,MAAM,wBAAwB,mBAAmB;GAEjD,MAAM,WAAW,yBAAyB,KAAK;GAE/C,MAAM,oBACJ,eAAe,KAAK,aAAa,KAAK,SAClC,OACA,KAAK,MAAM,YAAY,QAAQ;GAErC,KAAK,MAAM,CAAC,UAAU,kBAAkB,SAAS,KAAK,eAAe,GAAG;IACtE,MAAM,QAAQ,KAAK,qBAAqB,KAAK;IAC7C,IAAI,UAAU,KAAA,GAAW;KACvB,YAAY,KAAK,KAAK;KAEtB;IACF;IAEA,MAAM,SAAS,KAAK,eAAe,KAAK;IACxC,YAAY,KAAK,GAAG,MAAM;GAC5B;GAEA,IAAI,0BAA0B,KAAA,GAAW;IACvC,MAAM,eAAe,iBAAkB;IACvC,MAAM,oBAAoB,KAAK,qBAAqB,IAAI,YAAY;IACpE,IAAI,sBAAsB,KAAA,GACxB,MAAM,IAAI,MACR,kBAAkB,aAAa,uCACjC;IAEF,YAAY,KAAK,iBAAiB;IAClC,aAAa,wBAAwB,aAAa;GACpD,OACE;EAEJ;EAEA,OAAO;CACT;CAEA,YAAY,MAAc,gBAAsC;EAC9D,IAAI,aAAa;EACjB,IAAI,cAAc;EAElB,OAAO,MAAM;GACX,MAAM,mBAAmB,KAAK,qBAC5B,MACA,gBACA,UACF;GACA,MAAM,wBAAwB,mBAAmB;GAEjD,MAAM,WAAW,yBAAyB,KAAK;GAE/C,MAAM,oBACJ,eAAe,KAAK,aAAa,KAAK,SAClC,OACA,KAAK,MAAM,YAAY,QAAQ;GAErC,KAAK,MAAM,CAAC,UAAU,kBAAkB,SAAS,KAAK,eAAe,GAAG;IAEtE,IADc,KAAK,qBAAqB,KAChC,MAAM,KAAA,GAAW;KACvB;KAEA;IACF;IAEA,MAAM,SAAS,KAAK,eAAe,KAAK;IACxC,eAAe,OAAO;GACxB;GAEA,IAAI,0BAA0B,KAAA,GAAW;IACvC,MAAM,eAAe,iBAAkB;IAEvC,IAD0B,KAAK,qBAAqB,IAAI,YACpC,MAAM,KAAA,GACxB,MAAM,IAAI,MACR,kBAAkB,aAAa,uCACjC;IAEF;IACA,aAAa,wBAAwB,aAAa;GACpD,OACE;EAEJ;EAEA,OAAO;CACT;CAEA,CAAC,sBACC,QAC4C;EAC5C,KAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,aAAa,KAAK,eAAe,KAAK;GAC5C,IAAI,YACF,MAAM;EAEV;CACF;CAEA,aAAa,QAAkC;EAC7C,IAAI,UAAU;EACd,IAAI,YAAY;EAEhB,KAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,aAAa,KAAK,eAAe,KAAK;GAC5C,IAAI,eAAe,KAAA,GACjB,MAAM,IAAI,MAAM,SAAS,MAAM,kCAAkC;GAEnE,IAAI,OAAO,eAAe,UAAU;IAClC,IAAI,cAAc,aAAa;KAC7B,WAAW,QAAQ,OAAO,WAAW,EAAE,QAAQ,KAAK,CAAC;KACrD,YAAY;IACd;IACA,WAAW;GACb,OAAO;IACL,MAAM,YAAY,IAAI,WAAW,UAAU,SAAS,WAAW,MAAM;IACrE,UAAU,IAAI,SAAS;IACvB,UAAU,IAAI,YAAY,UAAU,MAAM;IAC1C,YAAY;GACd;EACF;EAEA,IAAI,cAAc,aAChB,WAAW,QAAQ,OAAO,WAAW,EAAE,QAAQ,KAAK,CAAC;EAEvD,OAAO;CACT;CAEA,OAAO,0BACL,QACqC;EACrC,WAAW,MAAM,SAAS,QAAQ;GAChC,MAAM,qBAAqB,KAAK,eAAe,KAAK;GACpD,IAAI,oBACF,MAAM;EAEV;CACF;CAEA,qBAA6B,KAAiC;EAC5D,OAAO,KAAK,0BAA0B,IAAI,GAAG;CAC/C;CAEA,4BAAoC,KAAqB;EACvD,MAAM,QAAQ,KAAK,qBAAqB,GAAG;EAC3C,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,MACR,wDAAwD,KAC1D;EAEF,OAAO;CACT;CAEA,oBAA4B,KAAqC;EAC/D,MAAM,cAAcF,gBAAAA,mBAAmB,GAAG;EAE1C,IAAI,gBAAgB,KAAA,GAClB,OAAO,KAAK,qBAAqB,WAAW;EAI9C,MAAM,QAAQ,KAAK,aAAa,GAAG;EACnC,IAAI,UAAU,IACZ,OAAO,KAAK,4BAA4B,MAAM,CAAE;CAIpD;CAEA,2BAAmC,KAAyB;EAC1D,MAAM,QAAQ,KAAK,oBAAoB,GAAG;EAC1C,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,MACR,wDAAwD,IAAI,SAAS,GACvE;EAEF,OAAO;CACT;CAGA,aAAqB,KAAyB;EAC5C,IAAI,MAAM;EACV,IAAI,OAAO,KAAK,4BAA4B,SAAS;EAErD,OAAO,OAAO,MAAM;GAClB,MAAM,MAAO,MAAM,SAAU;GAC7B,MAAM,SAAS,KAAK,4BAA4B,IAAI,CAAE;GACtD,IAAI,MAAM;GACV,MAAM,YAAY,KAAK,IAAI,OAAO,QAAQ,IAAI,MAAM;GACpD,KAAK,IAAI,IAAI,GAAG,IAAI,WAAW,KAAK;IAClC,MAAM,OAAO,KAAM,IAAI;IACvB,IAAI,QAAQ,GAAG;GACjB;GACA,IAAI,QAAQ,GACV,MAAM,OAAO,SAAS,IAAI;GAE5B,IAAI,QAAQ,GACV,OAAO;GAET,IAAI,MAAM,GACR,MAAM,MAAM;QAEZ,OAAO,MAAM;EAEjB;EACA,OAAO;CACT;CAEA,qBACE,MACA,gBACA,YACiD;EACjD,IAAI,cAAc;EAElB,OAAO,MAAM;GACX,KAAK,yBAAyB,YAAY;GAC1C,MAAM,mBAAmB,KAAK,yBAAyB,KAAK,IAAI;GAEhE,IAAI,CAAC,kBACH;GAGF,MAAM,eAAe,iBAAiB;GAEtC,IAAI,gBAAgB,IAAI,YAAY,GAElC,OAAO,CADwB,iBAAiB,QAAQ,aACxB,YAAY;GAG9C,cAAc,iBAAiB,QAAQ,cAAc;EACvD;CACF;CAEA,eAAuB,WAAoD;EACzE,MAAM,QAAQ,KAAK,oBAAoB;EACvC,IAAI,OAAO,UAAU,UACnB,OAAO;EAET,IAAI,OAAO,UAAU,UAAU;GAC7B,MAAM,aAAa,KAAK,0BAA0B,IAAI,SAAS;GAC/D,IAAI,YACF,OAAO;EAEX;EACA,OAAO,KAAK,qBAAqB,IAAI,SAAS;CAChD;CAEA,gBAAwB,KAAa,OAAuB;EAC1D,IAAI,CAAC,KAAK,YAAY;EAEtB,IAAI,KAAK,WAAW,QAAQ,KAAK,gBAAgB;GAE/C,MAAM,WAAW,KAAK,WAAW,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC;GAC/C,KAAK,WAAW,OAAO,QAAQ;EACjC;EACA,KAAK,WAAW,IAAI,KAAK,KAAK;CAChC;CAEA,eAAuB,OAAyB;EAC9C,IAAI,MAAM,WAAW,KAAKG,gBAAAA,QAAQ,MAAM,YAAY,CAAC,CAAE,GACrD,OAAO,CAAC,KAAK,4BAA4B,KAAK,CAAC;EAGjD,IAAI,KAAK,YAAY,IAAI,KAAK,GAAG;GAC/B,MAAM,SAAS,KAAK,WAAW,IAAI,KAAK;GAExC,KAAK,WAAW,OAAO,KAAK;GAC5B,KAAK,WAAW,IAAI,OAAO,MAAM;GACjC,OAAO;EACT;EAEA,MAAM,aAAa,KAAK,YAAY,OAAO,KAAK;EAChD,MAAM,SAAS,KAAK,cAAc,UAAU;EAC5C,KAAK,gBAAgB,OAAO,MAAM;EAClC,OAAO;CACT;CAEA,cAEE,OACU;EACV,MAAM,EAAE,WAAW;EAMnB,MAAM,OAAO,IAAI,WAAW,SAAS,CAAC;EACtC,MAAM,WAAW,IAAI,WAAW,SAAS,CAAC;EAC1C,MAAM,YAAY,IAAI,aAAa,SAAS,CAAC;EAC7C,MAAM,OAAO,IAAI,WAAW,SAAS,CAAC;EAEtC,MAAM,eAAe,eAA+B;GAClD,MAAM,SAAS,KAAK;GACpB,IAAI,UAAU,QAAQ,OAAO,OAAO;GAIpC,OAHa,KAAK,oBAChB,MAAM,SAAS,YAAY,KAAK,OAAQ,CAEhC,KAAK,OAAO;EACxB;EAGA,MAAM,OAAiB,CAAC;EACxB,MAAM,WAAW,GAAW,MAC1B,KAAK,KAAM,KAAK,MACf,KAAK,OAAO,KAAK,MAAM,KAAK,IAAI,KAAM,KAAK,IAAI;EAClD,MAAM,QAAQ,GAAW,MAAc;GACrC,MAAM,OAAO,KAAK;GAClB,MAAM,aAAa,KAAK,IAAI;GAC5B,KAAK,KAAK,KAAK;GACf,KAAK,IAAI,KAAK,KAAK,IAAI;GACvB,KAAK,KAAK;GACV,KAAK,IAAI,KAAK;EAChB;EACA,MAAM,QAAQ,MAAc,eAAuB;GACjD,KAAK,KAAK,MAAM,UAAU;GAC1B,IAAI,QAAQ,KAAK,SAAS;GAC1B,OAAO,QAAQ,GAAG;IAChB,MAAM,UAAY,SAAS,KAAK,KAAM,KAAM;IAC5C,IAAI,CAAC,QAAQ,OAAO,MAAM,GAAG;IAC7B,KAAK,OAAO,MAAM;IAClB,QAAQ;GACV;EACF;EACA,MAAM,YAAY;GAChB,MAAM,OAAO,KAAK,SAAS;GAC3B,KAAK,GAAG,IAAI;GACZ,MAAM,OAAO,KAAK;GAClB,MAAM,aAAa,KAAK,OAAO;GAC/B,KAAK,SAAS;GACd,IAAI,QAAQ;GACZ,OAAO,MAAM;IACX,MAAM,OAAO,QAAQ,IAAI;IACzB,MAAM,QAAQ,OAAO;IACrB,IAAI,SAAS;IACb,IAAI,OAAO,KAAK,UAAU,QAAQ,MAAM,MAAM,GAAG,SAAS;IAC1D,IAAI,QAAQ,KAAK,UAAU,QAAQ,OAAO,MAAM,GAAG,SAAS;IAC5D,IAAI,WAAW,OAAO;IACtB,KAAK,OAAO,MAAM;IAClB,QAAQ;GACV;GACA,OAAO,CAAC,MAAM,UAAU;EAC1B;EAEA,KAAK,IAAI,IAAI,GAAG,KAAK,QAAQ,KAAK;GAChC,KAAK,KAAK,IAAI;GACd,SAAS,KAAK,IAAI;GAClB,KAAK,KAAK;EACZ;EACA,KAAK,IAAI,IAAI,GAAG,IAAI,SAAS,GAAG,KAAK;GACnC,UAAU,KAAK,YAAY,CAAC;GAC5B,IAAI,UAAU,OAAO,OAAO,mBAAmB,KAAK,UAAU,IAAK,CAAC;EACtE;EAGA,OAAO,KAAK,SAAS,GAAG;GACtB,MAAM,CAAC,MAAM,cAAc,IAAI;GAE/B,IAAI,CAAC,KAAK,eAAe,UAAU,gBAAgB,MAAM;GAGzD,MAAM,SAAS,KAAK;GACpB,KAAK,UAAU;GACf,KAAK,cAAc,KAAK;GACxB,SAAS,KAAK,WAAY;GAE1B,UAAU,cAAc,YAAY,UAAU;GAC9C,IAAI,UAAU,gBAAgB,OAAO,mBACnC,KAAK,UAAU,aAAc,UAAU;GAGzC,MAAM,SAAS,SAAS;GACxB,IAAI,UAAU,GAAG;IACf,UAAU,UAAU,YAAY,MAAM;IACtC,IAAI,UAAU,YAAY,OAAO,mBAC/B,KAAK,UAAU,SAAU,MAAM;GAEnC;EACF;EAGA,MAAM,SAAmB,CAAC;EAC1B,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,IAAI,KAAK,IACnC,OAAO,KAAK,KAAK,2BAA2B,MAAM,SAAS,GAAG,KAAK,EAAG,CAAC,CAAC;EAE1E,OAAO;CACT;AACF"}

package/cjs/GptEncoding.d.ts

@@ -61,7 +61,7 @@ declare class GptEncoding {
    * Also mentioned in section 6 of this document: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
    * @param encodeOptions Options controlling how special tokens are handled.
    */
-  encodeChatGenerator(chat: Iterable<ChatMessage>, model?: "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "gpt2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions): Generator<number[], void, undefined>;
+  encodeChatGenerator(chat: Iterable<ChatMessage>, model?: "gpt2" | "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions): Generator<number[], void, undefined>;
   /**
    * Encodes a chat into a single array of tokens.
    * Warning: gpt-3.5-turbo and gpt-4 chat format may change over time.
@@ -70,7 +70,7 @@ declare class GptEncoding {
    * Also mentioned in section 6 of this document: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
    * @param encodeOptions Options controlling how special tokens are handled.
    */
-  encodeChat(chat: readonly ChatMessage[], model?: "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "gpt2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions): number[];
+  encodeChat(chat: readonly ChatMessage[], model?: "gpt2" | "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions): number[];
   /**
    * Checks whether the provided input stays within the provided token limit.
    * @param input The string or chat messages to evaluate.

package/cjs/encoding/cl100k_base.d.ts

@@ -5,7 +5,7 @@ import { EncodeOptions, GptEncoding } from "../GptEncoding.js";
 import { EndOfPrompt, EndOfText, FimMiddle, FimPrefix, FimSuffix, HarmonyCall, HarmonyChannel, HarmonyConstrain, HarmonyEnd, HarmonyMessage, HarmonyReturn, HarmonyStart, HarmonyStartOfText, ImEnd, ImSep, ImStart } from "../specialTokens.js";
 //#region src/encoding/cl100k_base.d.ts
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<ChatMessage>, tokenLimit: number, encodeOptions?: EncodeOptions) => false | number, countTokens: (input: string | Iterable<ChatMessage>, encodeOptions?: EncodeOptions) => number, encodeChat: (chat: readonly ChatMessage[], model?: "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "gpt2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions) => number[], encodeChatGenerator: (chat: Iterable<ChatMessage>, model?: "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "gpt2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions) => Generator<number[], void, undefined>, vocabularySize: number, setMergeCacheSize: (size: number) => void, clearMergeCache: () => void, estimateCost: (tokenCount: number, modelSpec?: ModelSpec | undefined) => PriceData;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<ChatMessage>, tokenLimit: number, encodeOptions?: EncodeOptions) => false | number, countTokens: (input: string | Iterable<ChatMessage>, encodeOptions?: EncodeOptions) => number, encodeChat: (chat: readonly ChatMessage[], model?: "gpt2" | "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions) => number[], encodeChatGenerator: (chat: Iterable<ChatMessage>, model?: "gpt2" | "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions) => Generator<number[], void, undefined>, vocabularySize: number, setMergeCacheSize: (size: number) => void, clearMergeCache: () => void, estimateCost: (tokenCount: number, modelSpec?: ModelSpec | undefined) => PriceData;
 //#endregion
 export { ALL_SPECIAL_TOKENS, DEFAULT_MERGE_CACHE_SIZE, EndOfPrompt, EndOfText, FimMiddle, FimPrefix, FimSuffix, HarmonyCall, HarmonyChannel, HarmonyConstrain, HarmonyEnd, HarmonyMessage, HarmonyReturn, HarmonyStart, HarmonyStartOfText, ImEnd, ImSep, ImStart, clearMergeCache, countTokens, decode, decodeAsyncGenerator, decodeGenerator, api as default, encode, encodeChat, encodeChatGenerator, encodeGenerator, estimateCost, isWithinTokenLimit, setMergeCacheSize, vocabularySize };
 //# sourceMappingURL=cl100k_base.d.ts.map

package/cjs/encoding/o200k_base.d.ts

@@ -5,7 +5,7 @@ import { EncodeOptions, GptEncoding } from "../GptEncoding.js";
 import { EndOfPrompt, EndOfText, FimMiddle, FimPrefix, FimSuffix, HarmonyCall, HarmonyChannel, HarmonyConstrain, HarmonyEnd, HarmonyMessage, HarmonyReturn, HarmonyStart, HarmonyStartOfText, ImEnd, ImSep, ImStart } from "../specialTokens.js";
 //#region src/encoding/o200k_base.d.ts
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<ChatMessage>, tokenLimit: number, encodeOptions?: EncodeOptions) => false | number, countTokens: (input: string | Iterable<ChatMessage>, encodeOptions?: EncodeOptions) => number, encodeChat: (chat: readonly ChatMessage[], model?: "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "gpt2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions) => number[], encodeChatGenerator: (chat: Iterable<ChatMessage>, model?: "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "gpt2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions) => Generator<number[], void, undefined>, vocabularySize: number, setMergeCacheSize: (size: number) => void, clearMergeCache: () => void, estimateCost: (tokenCount: number, modelSpec?: ModelSpec | undefined) => PriceData;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<ChatMessage>, tokenLimit: number, encodeOptions?: EncodeOptions) => false | number, countTokens: (input: string | Iterable<ChatMessage>, encodeOptions?: EncodeOptions) => number, encodeChat: (chat: readonly ChatMessage[], model?: "gpt2" | "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions) => number[], encodeChatGenerator: (chat: Iterable<ChatMessage>, model?: "gpt2" | "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions) => Generator<number[], void, undefined>, vocabularySize: number, setMergeCacheSize: (size: number) => void, clearMergeCache: () => void, estimateCost: (tokenCount: number, modelSpec?: ModelSpec | undefined) => PriceData;
 //#endregion
 export { ALL_SPECIAL_TOKENS, DEFAULT_MERGE_CACHE_SIZE, EndOfPrompt, EndOfText, FimMiddle, FimPrefix, FimSuffix, HarmonyCall, HarmonyChannel, HarmonyConstrain, HarmonyEnd, HarmonyMessage, HarmonyReturn, HarmonyStart, HarmonyStartOfText, ImEnd, ImSep, ImStart, clearMergeCache, countTokens, decode, decodeAsyncGenerator, decodeGenerator, api as default, encode, encodeChat, encodeChatGenerator, encodeGenerator, estimateCost, isWithinTokenLimit, setMergeCacheSize, vocabularySize };
 //# sourceMappingURL=o200k_base.d.ts.map

package/cjs/encoding/o200k_harmony.d.ts

@@ -5,7 +5,7 @@ import { EncodeOptions, GptEncoding } from "../GptEncoding.js";
 import { EndOfPrompt, EndOfText, FimMiddle, FimPrefix, FimSuffix, HarmonyCall, HarmonyChannel, HarmonyConstrain, HarmonyEnd, HarmonyMessage, HarmonyReturn, HarmonyStart, HarmonyStartOfText, ImEnd, ImSep, ImStart } from "../specialTokens.js";
 //#region src/encoding/o200k_harmony.d.ts
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<ChatMessage>, tokenLimit: number, encodeOptions?: EncodeOptions) => false | number, countTokens: (input: string | Iterable<ChatMessage>, encodeOptions?: EncodeOptions) => number, encodeChat: (chat: readonly ChatMessage[], model?: "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "gpt2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions) => number[], encodeChatGenerator: (chat: Iterable<ChatMessage>, model?: "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "gpt2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions) => Generator<number[], void, undefined>, vocabularySize: number, setMergeCacheSize: (size: number) => void, clearMergeCache: () => void, estimateCost: (tokenCount: number, modelSpec?: ModelSpec | undefined) => PriceData;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<ChatMessage>, tokenLimit: number, encodeOptions?: EncodeOptions) => false | number, countTokens: (input: string | Iterable<ChatMessage>, encodeOptions?: EncodeOptions) => number, encodeChat: (chat: readonly ChatMessage[], model?: "gpt2" | "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions) => number[], encodeChatGenerator: (chat: Iterable<ChatMessage>, model?: "gpt2" | "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions) => Generator<number[], void, undefined>, vocabularySize: number, setMergeCacheSize: (size: number) => void, clearMergeCache: () => void, estimateCost: (tokenCount: number, modelSpec?: ModelSpec | undefined) => PriceData;
 //#endregion
 export { ALL_SPECIAL_TOKENS, DEFAULT_MERGE_CACHE_SIZE, EndOfPrompt, EndOfText, FimMiddle, FimPrefix, FimSuffix, HarmonyCall, HarmonyChannel, HarmonyConstrain, HarmonyEnd, HarmonyMessage, HarmonyReturn, HarmonyStart, HarmonyStartOfText, ImEnd, ImSep, ImStart, clearMergeCache, countTokens, decode, decodeAsyncGenerator, decodeGenerator, api as default, encode, encodeChat, encodeChatGenerator, encodeGenerator, estimateCost, isWithinTokenLimit, setMergeCacheSize, vocabularySize };
 //# sourceMappingURL=o200k_harmony.d.ts.map