gpt-tokenizer 4.0.1 → 4.0.2

package/esm/model/o4-mini-deep-research.d.ts

@@ -5,7 +5,7 @@ import { EncodeOptions, GptEncoding } from "../GptEncoding.js";
 import { EndOfPrompt, EndOfText, FimMiddle, FimPrefix, FimSuffix, HarmonyCall, HarmonyChannel, HarmonyConstrain, HarmonyEnd, HarmonyMessage, HarmonyReturn, HarmonyStart, HarmonyStartOfText, ImEnd, ImSep, ImStart } from "../specialTokens.js";
 //#region src/model/o4-mini-deep-research.d.ts
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<ChatMessage>, tokenLimit: number, encodeOptions?: EncodeOptions) => false | number, countTokens: (input: string | Iterable<ChatMessage>, encodeOptions?: EncodeOptions) => number, encodeChat: (chat: readonly ChatMessage[], model?: "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "gpt2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions) => number[], encodeChatGenerator: (chat: Iterable<ChatMessage>, model?: "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "gpt2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions) => Generator<number[], void, undefined>, vocabularySize: number, setMergeCacheSize: (size: number) => void, clearMergeCache: () => void, estimateCost: (tokenCount: number, modelSpec?: ModelSpec | undefined) => PriceData;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<ChatMessage>, tokenLimit: number, encodeOptions?: EncodeOptions) => false | number, countTokens: (input: string | Iterable<ChatMessage>, encodeOptions?: EncodeOptions) => number, encodeChat: (chat: readonly ChatMessage[], model?: "gpt2" | "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions) => number[], encodeChatGenerator: (chat: Iterable<ChatMessage>, model?: "gpt2" | "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions) => Generator<number[], void, undefined>, vocabularySize: number, setMergeCacheSize: (size: number) => void, clearMergeCache: () => void, estimateCost: (tokenCount: number, modelSpec?: ModelSpec | undefined) => PriceData;
 //#endregion
 export { ALL_SPECIAL_TOKENS, DEFAULT_MERGE_CACHE_SIZE, EndOfPrompt, EndOfText, FimMiddle, FimPrefix, FimSuffix, HarmonyCall, HarmonyChannel, HarmonyConstrain, HarmonyEnd, HarmonyMessage, HarmonyReturn, HarmonyStart, HarmonyStartOfText, ImEnd, ImSep, ImStart, clearMergeCache, countTokens, decode, decodeAsyncGenerator, decodeGenerator, api as default, encode, encodeChat, encodeChatGenerator, encodeGenerator, estimateCost, isWithinTokenLimit, setMergeCacheSize, vocabularySize };
 //# sourceMappingURL=o4-mini-deep-research.d.ts.map

package/esm/model/o4-mini.d.ts

@@ -5,7 +5,7 @@ import { EncodeOptions, GptEncoding } from "../GptEncoding.js";
 import { EndOfPrompt, EndOfText, FimMiddle, FimPrefix, FimSuffix, HarmonyCall, HarmonyChannel, HarmonyConstrain, HarmonyEnd, HarmonyMessage, HarmonyReturn, HarmonyStart, HarmonyStartOfText, ImEnd, ImSep, ImStart } from "../specialTokens.js";
 //#region src/model/o4-mini.d.ts
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<ChatMessage>, tokenLimit: number, encodeOptions?: EncodeOptions) => false | number, countTokens: (input: string | Iterable<ChatMessage>, encodeOptions?: EncodeOptions) => number, encodeChat: (chat: readonly ChatMessage[], model?: "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "gpt2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions) => number[], countChatCompletionTokens: ((request: ChatCompletionRequest) => number) | undefined, encodeChatGenerator: (chat: Iterable<ChatMessage>, model?: "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "gpt2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions) => Generator<number[], void, undefined>, vocabularySize: number, setMergeCacheSize: (size: number) => void, clearMergeCache: () => void, estimateCost: (tokenCount: number, modelSpec?: ModelSpec | undefined) => PriceData;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<ChatMessage>, tokenLimit: number, encodeOptions?: EncodeOptions) => false | number, countTokens: (input: string | Iterable<ChatMessage>, encodeOptions?: EncodeOptions) => number, encodeChat: (chat: readonly ChatMessage[], model?: "gpt2" | "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions) => number[], countChatCompletionTokens: ((request: ChatCompletionRequest) => number) | undefined, encodeChatGenerator: (chat: Iterable<ChatMessage>, model?: "gpt2" | "gpt-3.5-0301" | "gpt-3.5" | "gpt-3.5-turbo-0613" | "gpt-4-1106-preview" | "gpt-4-32k" | "gpt-2" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-001" | "code-davinci-002" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-001" | "code-cushman-002" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | "babbage-002" | "chat-latest" | "chatgpt-4o-latest" | "chatgpt-image-latest" | "codex-mini-latest" | "computer-use-preview-2025-03-11" | "computer-use-preview" | "dall-e-2" | "dall-e-3" | "davinci-002" | "gpt-5.6-sol" | "daybreak-blue-latest" | "gpt-5.6-cyber" | "daybreak-red-latest" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-instruct" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo" | "gpt-4-0125-preview" | "gpt-4-1106-vision-preview" | "gpt-4-turbo-preview" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo" | "gpt-4.1-mini-2025-04-14" | "gpt-4.1-mini" | "gpt-4.1-nano-2025-04-14" | "gpt-4.1-nano" | "gpt-4.1-2025-04-14" | "gpt-4.1" | "gpt-4.5-preview-2025-02-27" | "gpt-4.5-preview" | "gpt-4-0613" | "gpt-4-0314" | "gpt-4" | "gpt-4o-audio-preview-2025-06-03" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-search-preview-2025-03-11" | "gpt-4o-mini-search-preview" | "gpt-4o-mini-transcribe-2025-03-20" | "gpt-4o-mini-transcribe-2025-12-15" | "gpt-4o-mini-transcribe" | "gpt-4o-mini-tts-2025-03-20" | "gpt-4o-mini-tts-2025-12-15" | "gpt-4o-mini-tts" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" | "gpt-4o-realtime-preview-2025-06-03" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview" | "gpt-4o-search-preview-2025-03-11" | "gpt-4o-search-preview" | "gpt-4o-transcribe-diarize" | "gpt-4o-transcribe" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o" | "gpt-5-chat-latest" | "gpt-5-codex" | "gpt-5-mini-2025-08-07" | "gpt-5-mini" | "gpt-5-nano-2025-08-07" | "gpt-5-nano" | "gpt-5-pro-2025-10-06" | "gpt-5-pro" | "gpt-5.1-chat-latest" | "gpt-5.1-codex-max" | "gpt-5.1-codex-mini" | "gpt-5.1-codex" | "gpt-5.1-2025-11-13" | "gpt-5.1" | "gpt-5.2-chat-latest" | "gpt-5.2-codex" | "gpt-5.2-pro-2025-12-11" | "gpt-5.2-pro" | "gpt-5.2-2025-12-11" | "gpt-5.2" | "gpt-5.3-chat-latest" | "gpt-5.3-codex" | "gpt-5.4-mini-2026-03-17" | "gpt-5.4-mini" | "gpt-5.4-nano-2026-03-17" | "gpt-5.4-nano" | "gpt-5.4-pro-2026-03-05" | "gpt-5.4-pro" | "gpt-5.4-2026-03-05" | "gpt-5.4" | "gpt-5.5-pro-2026-04-23" | "gpt-5.5-pro" | "gpt-5.5-2026-04-23" | "gpt-5.5" | "gpt-5.6-luna" | "gpt-5.6-terra" | "gpt-5-2025-08-07" | "gpt-5" | "gpt-audio-1.5" | "gpt-audio-mini-2025-10-06" | "gpt-audio-mini-2025-12-15" | "gpt-audio-mini" | "gpt-audio-2025-08-28" | "gpt-audio" | "gpt-image-1-mini" | "gpt-image-1.5-2025-12-16" | "gpt-image-1.5" | "gpt-image-1" | "gpt-image-2-2026-04-21" | "gpt-image-2" | "gpt-live-transcribe" | "gpt-oss-120b" | "gpt-oss-20b" | "gpt-realtime-1.5" | "gpt-realtime-2.1-mini" | "gpt-realtime-2.1" | "gpt-realtime-2" | "gpt-realtime-mini-2025-10-06" | "gpt-realtime-mini-2025-12-15" | "gpt-realtime-mini" | "gpt-realtime-translate" | "gpt-realtime-whisper" | "gpt-realtime-2025-08-28" | "gpt-realtime" | "gpt-transcribe" | "o1-mini-2024-09-12" | "o1-mini" | "o1-preview-2024-09-12" | "o1-preview" | "o1-pro-2025-03-19" | "o1-pro" | "o1-2024-12-17" | "o1" | "o3-deep-research-2025-06-26" | "o3-deep-research" | "o3-mini-2025-01-31" | "o3-mini" | "o3-pro-2025-06-10" | "o3-pro" | "o3-2025-04-16" | "o3" | "o4-mini-deep-research-2025-06-26" | "o4-mini-deep-research" | "o4-mini-2025-04-16" | "o4-mini" | "omni-moderation-2024-09-26" | "omni-moderation-latest" | "sora-2-pro-2025-10-06" | "sora-2-pro" | "sora-2-2025-12-08" | "sora-2-2025-10-06" | "sora-2" | "text-embedding-3-large" | "text-embedding-3-small" | "text-embedding-ada-002" | "text-moderation-007" | "text-moderation-latest" | "text-moderation-stable" | "tts-1-hd" | "tts-1" | "whisper-1" | undefined, encodeOptions?: EncodeOptions & EncodeChatOptions) => Generator<number[], void, undefined>, vocabularySize: number, setMergeCacheSize: (size: number) => void, clearMergeCache: () => void, estimateCost: (tokenCount: number, modelSpec?: ModelSpec | undefined) => PriceData;
 //#endregion
 export { ALL_SPECIAL_TOKENS, DEFAULT_MERGE_CACHE_SIZE, EndOfPrompt, EndOfText, FimMiddle, FimPrefix, FimSuffix, HarmonyCall, HarmonyChannel, HarmonyConstrain, HarmonyEnd, HarmonyMessage, HarmonyReturn, HarmonyStart, HarmonyStartOfText, ImEnd, ImSep, ImStart, clearMergeCache, countChatCompletionTokens, countTokens, decode, decodeAsyncGenerator, decodeGenerator, api as default, encode, encodeChat, encodeChatGenerator, encodeGenerator, estimateCost, isWithinTokenLimit, setMergeCacheSize, vocabularySize };
 //# sourceMappingURL=o4-mini.d.ts.map

package/esm/utfUtil.d.ts

@@ -1,8 +1,9 @@
 //#region src/utfUtil.d.ts
 declare const isAscii: (codePoint: number) => boolean;
 declare function endsWithIncompleteUtfPairSurrogate(string: string): boolean;
+declare function startsWithByteOrderMark(bytes: Uint8Array): boolean;
 declare function tryConvertToString(arr: Uint8Array): string | undefined;
 declare function compareUint8Arrays(a: Uint8Array, b: Uint8Array): number;
 //#endregion
-export { compareUint8Arrays, endsWithIncompleteUtfPairSurrogate, isAscii, tryConvertToString };
+export { compareUint8Arrays, endsWithIncompleteUtfPairSurrogate, isAscii, startsWithByteOrderMark, tryConvertToString };
 //# sourceMappingURL=utfUtil.d.ts.map

package/esm/utfUtil.js

@@ -43,7 +43,13 @@ function isValidUTF8(bytes) {
 	}
 	return true;
 }
-const textDecoder = new TextDecoder("utf8", { fatal: false });
+function startsWithByteOrderMark(bytes) {
+	return bytes.length >= 3 && bytes[0] === 239 && bytes[1] === 187 && bytes[2] === 191;
+}
+const textDecoder = new TextDecoder("utf8", {
+	fatal: false,
+	ignoreBOM: true
+});
 function tryConvertToString(arr) {
 	if (!isValidUTF8(arr)) return;
 	return textDecoder.decode(arr);
@@ -54,6 +60,6 @@ function compareUint8Arrays(a, b) {
 	return a.length - b.length;
 }
 //#endregion
-export { compareUint8Arrays, endsWithIncompleteUtfPairSurrogate, isAscii, tryConvertToString };
+export { compareUint8Arrays, endsWithIncompleteUtfPairSurrogate, isAscii, startsWithByteOrderMark, tryConvertToString };
 
 //# sourceMappingURL=utfUtil.js.map

package/esm/utfUtil.js.map

@@ -1 +1 @@
-{"version":3,"file":"utfUtil.js","names":[],"sources":["../src/utfUtil.ts"],"sourcesContent":["export const isAscii = (codePoint: number) => codePoint <= 0x7f\n\nconst HIGH_SURROGATE_START = 55_296\nconst HIGH_SURROGATE_END = 56_319\n\nexport function endsWithIncompleteUtfPairSurrogate(string: string): boolean {\n  if (string.length === 0) return false\n  // Check if the last character is a high surrogate\n  const lastCharCode = string.charCodeAt(string.length - 1)\n  return (\n    lastCharCode >= HIGH_SURROGATE_START && lastCharCode <= HIGH_SURROGATE_END\n  )\n}\n\nfunction isValidUTF8(bytes: Uint8Array): boolean {\n  let i = 0\n  while (i < bytes.length) {\n    const byte1 = bytes[i]!\n\n    let numBytes = 0\n    let codePoint = 0\n\n    // Determine the number of bytes in the current UTF-8 character\n    if (byte1 <= 0x7f) {\n      // 1-byte character (ASCII)\n      numBytes = 1\n      codePoint = byte1\n    } else if ((byte1 & 0xe0) === 0xc0) {\n      // 2-byte character\n      numBytes = 2\n      codePoint = byte1 & 0x1f\n      if (byte1 <= 0xc1) return false // Overlong encoding not allowed\n    } else if ((byte1 & 0xf0) === 0xe0) {\n      // 3-byte character\n      numBytes = 3\n      codePoint = byte1 & 0x0f\n    } else if ((byte1 & 0xf8) === 0xf0) {\n      // 4-byte character\n      numBytes = 4\n      codePoint = byte1 & 0x07\n      if (byte1 > 0xf4) return false // Code points above U+10FFFF not allowed\n    } else {\n      // Invalid first byte of UTF-8 character\n      return false\n    }\n\n    // Ensure there are enough continuation bytes\n    if (i + numBytes > bytes.length) return false\n\n    // Process the continuation bytes\n    for (let j = 1; j < numBytes; j++) {\n      const byte = bytes[i + j]\n      if (byte === undefined || (byte & 0xc0) !== 0x80) return false // Continuation bytes must start with '10'\n      codePoint = (codePoint << 6) | (byte & 0x3f)\n    }\n\n    // Check for overlong encodings\n    if (numBytes === 2 && codePoint < 0x80) return false // Overlong 2-byte sequence\n    if (numBytes === 3 && codePoint < 2_048) return false // Overlong 3-byte sequence\n    if (numBytes === 4 && codePoint < 65_536) return false // Overlong 4-byte sequence\n\n    // Check for surrogate halves (U+D800 to U+DFFF)\n    if (codePoint >= 55_296 && codePoint <= 57_343) return false\n\n    // Check for code points above U+10FFFF\n    if (codePoint > 1_114_111) return false\n\n    // Move to the next character\n    i += numBytes\n  }\n  return true\n}\n\nconst textDecoder = new TextDecoder('utf8', { fatal: false })\n\nexport function tryConvertToString(arr: Uint8Array): string | undefined {\n  if (!isValidUTF8(arr)) {\n    return undefined\n  }\n  return textDecoder.decode(arr)\n}\n\n// Helper function to compare two Uint8Arrays lexicographically\nexport function compareUint8Arrays(a: Uint8Array, b: Uint8Array): number {\n  const len = Math.min(a.length, b.length)\n  for (let i = 0; i < len; i++) {\n    if (a[i] !== b[i]) {\n      return a[i]! - b[i]!\n    }\n  }\n  return a.length - b.length\n}\n"],"mappings":";AAAA,MAAa,WAAW,cAAsB,aAAa;AAE3D,MAAM,uBAAuB;AAC7B,MAAM,qBAAqB;AAE3B,SAAgB,mCAAmC,QAAyB;CAC1E,IAAI,OAAO,WAAW,GAAG,OAAO;CAEhC,MAAM,eAAe,OAAO,WAAW,OAAO,SAAS,CAAC;CACxD,OACE,gBAAgB,wBAAwB,gBAAgB;AAE5D;AAEA,SAAS,YAAY,OAA4B;CAC/C,IAAI,IAAI;CACR,OAAO,IAAI,MAAM,QAAQ;EACvB,MAAM,QAAQ,MAAM;EAEpB,IAAI,WAAW;EACf,IAAI,YAAY;EAGhB,IAAI,SAAS,KAAM;GAEjB,WAAW;GACX,YAAY;EACd,OAAO,KAAK,QAAQ,SAAU,KAAM;GAElC,WAAW;GACX,YAAY,QAAQ;GACpB,IAAI,SAAS,KAAM,OAAO;EAC5B,OAAO,KAAK,QAAQ,SAAU,KAAM;GAElC,WAAW;GACX,YAAY,QAAQ;EACtB,OAAO,KAAK,QAAQ,SAAU,KAAM;GAElC,WAAW;GACX,YAAY,QAAQ;GACpB,IAAI,QAAQ,KAAM,OAAO;EAC3B,OAEE,OAAO;EAIT,IAAI,IAAI,WAAW,MAAM,QAAQ,OAAO;EAGxC,KAAK,IAAI,IAAI,GAAG,IAAI,UAAU,KAAK;GACjC,MAAM,OAAO,MAAM,IAAI;GACvB,IAAI,SAAS,KAAA,MAAc,OAAO,SAAU,KAAM,OAAO;GACzD,YAAa,aAAa,IAAM,OAAO;EACzC;EAGA,IAAI,aAAa,KAAK,YAAY,KAAM,OAAO;EAC/C,IAAI,aAAa,KAAK,YAAY,MAAO,OAAO;EAChD,IAAI,aAAa,KAAK,YAAY,OAAQ,OAAO;EAGjD,IAAI,aAAa,SAAU,aAAa,OAAQ,OAAO;EAGvD,IAAI,YAAY,SAAW,OAAO;EAGlC,KAAK;CACP;CACA,OAAO;AACT;AAEA,MAAM,cAAc,IAAI,YAAY,QAAQ,EAAE,OAAO,MAAM,CAAC;AAE5D,SAAgB,mBAAmB,KAAqC;CACtE,IAAI,CAAC,YAAY,GAAG,GAClB;CAEF,OAAO,YAAY,OAAO,GAAG;AAC/B;AAGA,SAAgB,mBAAmB,GAAe,GAAuB;CACvE,MAAM,MAAM,KAAK,IAAI,EAAE,QAAQ,EAAE,MAAM;CACvC,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,KACvB,IAAI,EAAE,OAAO,EAAE,IACb,OAAO,EAAE,KAAM,EAAE;CAGrB,OAAO,EAAE,SAAS,EAAE;AACtB"}
+{"version":3,"file":"utfUtil.js","names":[],"sources":["../src/utfUtil.ts"],"sourcesContent":["export const isAscii = (codePoint: number) => codePoint <= 0x7f\n\nconst HIGH_SURROGATE_START = 55_296\nconst HIGH_SURROGATE_END = 56_319\n\nexport function endsWithIncompleteUtfPairSurrogate(string: string): boolean {\n  if (string.length === 0) return false\n  // Check if the last character is a high surrogate\n  const lastCharCode = string.charCodeAt(string.length - 1)\n  return (\n    lastCharCode >= HIGH_SURROGATE_START && lastCharCode <= HIGH_SURROGATE_END\n  )\n}\n\nfunction isValidUTF8(bytes: Uint8Array): boolean {\n  let i = 0\n  while (i < bytes.length) {\n    const byte1 = bytes[i]!\n\n    let numBytes = 0\n    let codePoint = 0\n\n    // Determine the number of bytes in the current UTF-8 character\n    if (byte1 <= 0x7f) {\n      // 1-byte character (ASCII)\n      numBytes = 1\n      codePoint = byte1\n    } else if ((byte1 & 0xe0) === 0xc0) {\n      // 2-byte character\n      numBytes = 2\n      codePoint = byte1 & 0x1f\n      if (byte1 <= 0xc1) return false // Overlong encoding not allowed\n    } else if ((byte1 & 0xf0) === 0xe0) {\n      // 3-byte character\n      numBytes = 3\n      codePoint = byte1 & 0x0f\n    } else if ((byte1 & 0xf8) === 0xf0) {\n      // 4-byte character\n      numBytes = 4\n      codePoint = byte1 & 0x07\n      if (byte1 > 0xf4) return false // Code points above U+10FFFF not allowed\n    } else {\n      // Invalid first byte of UTF-8 character\n      return false\n    }\n\n    // Ensure there are enough continuation bytes\n    if (i + numBytes > bytes.length) return false\n\n    // Process the continuation bytes\n    for (let j = 1; j < numBytes; j++) {\n      const byte = bytes[i + j]\n      if (byte === undefined || (byte & 0xc0) !== 0x80) return false // Continuation bytes must start with '10'\n      codePoint = (codePoint << 6) | (byte & 0x3f)\n    }\n\n    // Check for overlong encodings\n    if (numBytes === 2 && codePoint < 0x80) return false // Overlong 2-byte sequence\n    if (numBytes === 3 && codePoint < 2_048) return false // Overlong 3-byte sequence\n    if (numBytes === 4 && codePoint < 65_536) return false // Overlong 4-byte sequence\n\n    // Check for surrogate halves (U+D800 to U+DFFF)\n    if (codePoint >= 55_296 && codePoint <= 57_343) return false\n\n    // Check for code points above U+10FFFF\n    if (codePoint > 1_114_111) return false\n\n    // Move to the next character\n    i += numBytes\n  }\n  return true\n}\n\nexport function startsWithByteOrderMark(bytes: Uint8Array): boolean {\n  return (\n    bytes.length >= 3 &&\n    bytes[0] === 0xef &&\n    bytes[1] === 0xbb &&\n    bytes[2] === 0xbf\n  )\n}\n\n// `ignoreBOM: true` keeps a leading U+FEFF in the output instead of silently\n// stripping it, so the bytes EF BB BF map to U+FEFF rather than to an empty string.\nconst textDecoder = new TextDecoder('utf8', { fatal: false, ignoreBOM: true })\n\nexport function tryConvertToString(arr: Uint8Array): string | undefined {\n  if (!isValidUTF8(arr)) {\n    return undefined\n  }\n  return textDecoder.decode(arr)\n}\n\n// Helper function to compare two Uint8Arrays lexicographically\nexport function compareUint8Arrays(a: Uint8Array, b: Uint8Array): number {\n  const len = Math.min(a.length, b.length)\n  for (let i = 0; i < len; i++) {\n    if (a[i] !== b[i]) {\n      return a[i]! - b[i]!\n    }\n  }\n  return a.length - b.length\n}\n"],"mappings":";AAAA,MAAa,WAAW,cAAsB,aAAa;AAE3D,MAAM,uBAAuB;AAC7B,MAAM,qBAAqB;AAE3B,SAAgB,mCAAmC,QAAyB;CAC1E,IAAI,OAAO,WAAW,GAAG,OAAO;CAEhC,MAAM,eAAe,OAAO,WAAW,OAAO,SAAS,CAAC;CACxD,OACE,gBAAgB,wBAAwB,gBAAgB;AAE5D;AAEA,SAAS,YAAY,OAA4B;CAC/C,IAAI,IAAI;CACR,OAAO,IAAI,MAAM,QAAQ;EACvB,MAAM,QAAQ,MAAM;EAEpB,IAAI,WAAW;EACf,IAAI,YAAY;EAGhB,IAAI,SAAS,KAAM;GAEjB,WAAW;GACX,YAAY;EACd,OAAO,KAAK,QAAQ,SAAU,KAAM;GAElC,WAAW;GACX,YAAY,QAAQ;GACpB,IAAI,SAAS,KAAM,OAAO;EAC5B,OAAO,KAAK,QAAQ,SAAU,KAAM;GAElC,WAAW;GACX,YAAY,QAAQ;EACtB,OAAO,KAAK,QAAQ,SAAU,KAAM;GAElC,WAAW;GACX,YAAY,QAAQ;GACpB,IAAI,QAAQ,KAAM,OAAO;EAC3B,OAEE,OAAO;EAIT,IAAI,IAAI,WAAW,MAAM,QAAQ,OAAO;EAGxC,KAAK,IAAI,IAAI,GAAG,IAAI,UAAU,KAAK;GACjC,MAAM,OAAO,MAAM,IAAI;GACvB,IAAI,SAAS,KAAA,MAAc,OAAO,SAAU,KAAM,OAAO;GACzD,YAAa,aAAa,IAAM,OAAO;EACzC;EAGA,IAAI,aAAa,KAAK,YAAY,KAAM,OAAO;EAC/C,IAAI,aAAa,KAAK,YAAY,MAAO,OAAO;EAChD,IAAI,aAAa,KAAK,YAAY,OAAQ,OAAO;EAGjD,IAAI,aAAa,SAAU,aAAa,OAAQ,OAAO;EAGvD,IAAI,YAAY,SAAW,OAAO;EAGlC,KAAK;CACP;CACA,OAAO;AACT;AAEA,SAAgB,wBAAwB,OAA4B;CAClE,OACE,MAAM,UAAU,KAChB,MAAM,OAAO,OACb,MAAM,OAAO,OACb,MAAM,OAAO;AAEjB;AAIA,MAAM,cAAc,IAAI,YAAY,QAAQ;CAAE,OAAO;CAAO,WAAW;AAAK,CAAC;AAE7E,SAAgB,mBAAmB,KAAqC;CACtE,IAAI,CAAC,YAAY,GAAG,GAClB;CAEF,OAAO,YAAY,OAAO,GAAG;AAC/B;AAGA,SAAgB,mBAAmB,GAAe,GAAuB;CACvE,MAAM,MAAM,KAAK,IAAI,EAAE,QAAQ,EAAE,MAAM;CACvC,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,KACvB,IAAI,EAAE,OAAO,EAAE,IACb,OAAO,EAAE,KAAM,EAAE;CAGrB,OAAO,EAAE,SAAS,EAAE;AACtB"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gpt-tokenizer",
-  "version": "4.0.1",
+  "version": "4.0.2",
   "description": "A pure JavaScript BPE tokenizer with model metadata and pricing for GPT-5, GPT-4o, o-series, and other OpenAI models",
   "keywords": [
     "BPE",

package/src/BytePairEncodingCore.ts

@@ -1,5 +1,10 @@
 import { DEFAULT_MERGE_CACHE_SIZE } from './constants.js'
-import { compareUint8Arrays, isAscii, tryConvertToString } from './utfUtil.js'
+import {
+  compareUint8Arrays,
+  isAscii,
+  startsWithByteOrderMark,
+  tryConvertToString,
+} from './utfUtil.js'
 import { escapeRegExp } from './util.js'
 
 export type RawBytePairRanks = readonly (string | readonly number[])[]
@@ -19,7 +24,8 @@ export interface BytePairEncodingConfig {
 
 const emptyBuffer = new Uint8Array(0)
 
-export const decoder = new TextDecoder('utf8')
+// `ignoreBOM: true` keeps a leading U+FEFF in the decoded output.
+export const decoder = new TextDecoder('utf8', { ignoreBOM: true })
 
 export class BytePairEncodingCore {
   readonly mergeableBytePairRankCount: number
@@ -70,6 +76,16 @@ export class BytePairEncodingCore {
       const byteArray = new Uint8Array(value)
       binaryLookup.push([byteArray, rank])
       this.bytePairNonUtfRankDecoder.set(rank, byteArray)
+      if (startsWithByteOrderMark(byteArray)) {
+        // The rank tables keep tokens that start with a UTF-8 byte order mark
+        // (EF BB BF) as raw bytes, because a plain decode would strip the mark.
+        // They are still valid UTF-8, so they must also be reachable through
+        // the string lookup that getBpeRankFromBytes uses for valid UTF-8.
+        const asString = tryConvertToString(byteArray)
+        if (asString !== undefined) {
+          this.bytePairStringRankEncoder.set(asString, rank)
+        }
+      }
     })
     this.bytePairNonUtfSortedEncoder = binaryLookup.sort((a, b) =>
       compareUint8Arrays(a[0], b[0]),

package/src/encodingParams/constants.ts

@@ -1,18 +1,22 @@
+// The split patterns below use `\p{White_Space}` / `\P{White_Space}` where
+// tiktoken's patterns use `\s` / `\S`: the regex engine behind tiktoken treats
+// `\s` as the Unicode White_Space property, whereas JavaScript's `\s` differs
+// from it on exactly two code points (U+0085 and U+FEFF).
 export const R50K_TOKEN_SPLIT_REGEX =
-  /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu
+  /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\p{White_Space}\p{L}\p{N}]+|\p{White_Space}+(?!\P{White_Space})|\p{White_Space}+/gu
 
 const CONTRACTION_SUFFIX_PATTERN = String.raw`'(?:[sS]|[dD]|[mM]|[tT]|[lL][lL]|[vV][eE]|[rR][eE])`
 
 const OPTIONAL_CONTRACTION_SUFFIX = String.raw`(?:${CONTRACTION_SUFFIX_PATTERN})?`
 
-const CL100K_TOKEN_SPLIT_PATTERN = String.raw`${CONTRACTION_SUFFIX_PATTERN}|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s+$|\s*[\r\n]|\s+(?!\S)|\s`
+const CL100K_TOKEN_SPLIT_PATTERN = String.raw`${CONTRACTION_SUFFIX_PATTERN}|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\p{White_Space}\p{L}\p{N}]+[\r\n]*|\p{White_Space}+$|\p{White_Space}*[\r\n]|\p{White_Space}+(?!\P{White_Space})|\p{White_Space}`
 
 export const CL100K_TOKEN_SPLIT_REGEX = new RegExp(
   CL100K_TOKEN_SPLIT_PATTERN,
   'gu',
 )
 
-const O200K_TOKEN_SPLIT_PATTERN = String.raw`[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+${OPTIONAL_CONTRACTION_SUFFIX}|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*${OPTIONAL_CONTRACTION_SUFFIX}|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n/]*|\s*[\r\n]+|\s+(?!\S)|\s+`
+const O200K_TOKEN_SPLIT_PATTERN = String.raw`[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+${OPTIONAL_CONTRACTION_SUFFIX}|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*${OPTIONAL_CONTRACTION_SUFFIX}|\p{N}{1,3}| ?[^\p{White_Space}\p{L}\p{N}]+[\r\n/]*|\p{White_Space}*[\r\n]+|\p{White_Space}+(?!\P{White_Space})|\p{White_Space}+`
 
 export const O200K_TOKEN_SPLIT_REGEX = new RegExp(
   O200K_TOKEN_SPLIT_PATTERN,

package/src/unicodeWhiteSpace.test.ts

@@ -0,0 +1,247 @@
+import { describe, expect, test } from 'vite-plus/test'
+import { GptEncoding } from './GptEncoding.js'
+import { type EncodingName } from './mapping.js'
+import { resolveEncoding } from './resolveEncoding.js'
+
+const getEncoding = (encodingName: string) =>
+  GptEncoding.getEncodingApi(encodingName as EncodingName, resolveEncoding)
+
+/**
+ * Two things have to line up with tiktoken for these strings:
+ *
+ * - the split patterns must treat `\s` as the Unicode White_Space property.
+ *   JavaScript's own `\s` differs from it on exactly two code points: U+0085
+ *   (White_Space only) and U+FEFF (JavaScript `\s` only);
+ * - tokens that start with the UTF-8 byte order mark (EF BB BF, i.e. U+FEFF)
+ *   must stay reachable, even though a plain `TextDecoder` strips that mark.
+ *
+ * Reference vectors generated with the Python `tiktoken` package
+ * (`encoding.encode(text)`).
+ */
+const unicodeWhiteSpaceCases: [
+  encodingName: string,
+  text: string,
+  expected: number[],
+][] = [
+  ['cl100k_base', 'a\ufeffb', [64, 3305, 65]],
+  ['cl100k_base', 'Hello\ufeff\n', [9906, 62619]],
+  ['o200k_base', 'x\ufeff\ufeffy', [87, 135153, 88]],
+  ['o200k_base', 'a \u0085b', [64, 220, 126, 227, 65]],
+  ['cl100k_base', 'a \u0085b', [64, 220, 126, 227, 65]],
+  ['p50k_base', 'a \ufeffb', [64, 27332, 119, 123, 65]],
+  ['r50k_base', 'a \ufeffb', [64, 27332, 119, 123, 65]],
+  ['r50k_base', 'a \u0085b', [64, 220, 126, 227, 65]],
+  ['p50k_base', 'a \u0085b', [64, 220, 126, 227, 65]],
+  ['o200k_base', 'a\ufeffb', [64, 5574, 65]],
+  ['cl100k_base', 'x\ufeff\ufeffy', [87, 3305, 3305, 88]],
+  ['o200k_base', 'Hello\ufeff\n', [13225, 61992]],
+  ['o200k_base', '\ufeffHello world', [5574, 13225, 2375]],
+  ['cl100k_base', '\ufeffHello world', [3305, 9906, 1917]],
+  ['o200k_base', 'a\u0085b', [64, 126, 227, 65]],
+  ['cl100k_base', 'a\u0085\n\u0085b', [64, 126, 227, 198, 126, 227, 65]],
+  ['o200k_base', 'line one\u0085line two', [1137, 1001, 126, 227, 1137, 1920]],
+  ['cl100k_base', 'line one\u0085line two', [1074, 832, 126, 227, 1074, 1403]],
+  ['p50k_base', 'a\ufeffb', [64, 171, 119, 123, 65]],
+  ['r50k_base', 'a\ufeffb', [64, 171, 119, 123, 65]],
+]
+
+/**
+ * Ordinary whitespace must keep tokenizing exactly as before (and exactly like
+ * tiktoken): the change above only affects U+0085 and U+FEFF.
+ */
+const ordinaryWhiteSpaceCases: [
+  encodingName: string,
+  label: string,
+  text: string,
+  expected: number[],
+][] = [
+  ['cl100k_base', 'space', 'a  b', [64, 220, 293]],
+  ['cl100k_base', 'tabs', 'a\t\tb\tc', [64, 197, 2282, 1470]],
+  ['cl100k_base', 'newlines', 'a\n\nb\r\nc\n', [64, 271, 65, 319, 66, 198]],
+  ['cl100k_base', 'nbsp', 'a\u00a0b \u00a0c', [64, 4194, 65, 220, 4194, 66]],
+  [
+    'cl100k_base',
+    'ideographic space',
+    'a\u3000b\u3000\u3000c',
+    [64, 23249, 65, 23249, 23249, 66],
+  ],
+  [
+    'cl100k_base',
+    'line and paragraph separators',
+    'a\u2028b\u2029c \u2028',
+    [64, 378, 101, 65, 378, 102, 66, 639, 101],
+  ],
+  [
+    'cl100k_base',
+    'mixed unicode spaces',
+    'x\u2003y\u2009\u200az\u202f\u205f',
+    [87, 378, 225, 88, 378, 231, 378, 232, 89, 378, 107, 53233, 253],
+  ],
+  [
+    'cl100k_base',
+    'vertical tab and form feed',
+    'a\u000bb\fc',
+    [64, 199, 65, 200, 66],
+  ],
+  ['cl100k_base', 'trailing whitespace', 'hello   \n  ', [15339, 5996, 256]],
+  ['cl100k_base', 'leading whitespace', '   hello', [256, 24748]],
+  ['o200k_base', 'space', 'a  b', [64, 220, 287]],
+  ['o200k_base', 'tabs', 'a\t\tb\tc', [64, 197, 3722, 2736]],
+  ['o200k_base', 'newlines', 'a\n\nb\r\nc\n', [64, 279, 65, 370, 66, 198]],
+  ['o200k_base', 'nbsp', 'a\u00a0b \u00a0c', [64, 5310, 65, 220, 5310, 66]],
+  [
+    'o200k_base',
+    'ideographic space',
+    'a\u3000b\u3000\u3000c',
+    [64, 1397, 65, 1397, 1397, 66],
+  ],
+  [
+    'o200k_base',
+    'line and paragraph separators',
+    'a\u2028b\u2029c \u2028',
+    [64, 51008, 65, 318, 102, 66, 121514],
+  ],
+  [
+    'o200k_base',
+    'mixed unicode spaces',
+    'x\u2003y\u2009\u200az\u202f\u205f',
+    [87, 33203, 88, 29106, 86741, 89, 35971, 37127, 253],
+  ],
+  [
+    'o200k_base',
+    'vertical tab and form feed',
+    'a\u000bb\fc',
+    [64, 199, 65, 200, 66],
+  ],
+  ['o200k_base', 'trailing whitespace', 'hello   \n  ', [24912, 10190, 256]],
+  ['o200k_base', 'leading whitespace', '   hello', [256, 40617]],
+  ['p50k_base', 'space', 'a  b', [64, 220, 275]],
+  ['p50k_base', 'tabs', 'a\t\tb\tc', [64, 197, 197, 65, 197, 66]],
+  [
+    'p50k_base',
+    'newlines',
+    'a\n\nb\r\nc\n',
+    [64, 198, 198, 65, 201, 198, 66, 198],
+  ],
+  ['p50k_base', 'nbsp', 'a\u00a0b \u00a0c', [64, 1849, 65, 220, 1849, 66]],
+  [
+    'p50k_base',
+    'ideographic space',
+    'a\u3000b\u3000\u3000c',
+    [64, 5099, 222, 65, 5099, 222, 5099, 222, 66],
+  ],
+  [
+    'p50k_base',
+    'line and paragraph separators',
+    'a\u2028b\u2029c \u2028',
+    [64, 447, 101, 65, 447, 102, 66, 564, 101],
+  ],
+  [
+    'p50k_base',
+    'mixed unicode spaces',
+    'x\u2003y\u2009\u200az\u202f\u205f',
+    [87, 447, 225, 88, 447, 231, 447, 232, 89, 447, 107, 46256, 253],
+  ],
+  [
+    'p50k_base',
+    'vertical tab and form feed',
+    'a\u000bb\fc',
+    [64, 199, 65, 200, 66],
+  ],
+  [
+    'p50k_base',
+    'trailing whitespace',
+    'hello   \n  ',
+    [31373, 50258, 198, 50257],
+  ],
+  ['p50k_base', 'leading whitespace', '   hello', [50257, 23748]],
+  ['r50k_base', 'space', 'a  b', [64, 220, 275]],
+  ['r50k_base', 'tabs', 'a\t\tb\tc', [64, 197, 197, 65, 197, 66]],
+  [
+    'r50k_base',
+    'newlines',
+    'a\n\nb\r\nc\n',
+    [64, 198, 198, 65, 201, 198, 66, 198],
+  ],
+  ['r50k_base', 'nbsp', 'a\u00a0b \u00a0c', [64, 1849, 65, 220, 1849, 66]],
+  [
+    'r50k_base',
+    'ideographic space',
+    'a\u3000b\u3000\u3000c',
+    [64, 5099, 222, 65, 5099, 222, 5099, 222, 66],
+  ],
+  [
+    'r50k_base',
+    'line and paragraph separators',
+    'a\u2028b\u2029c \u2028',
+    [64, 447, 101, 65, 447, 102, 66, 564, 101],
+  ],
+  [
+    'r50k_base',
+    'mixed unicode spaces',
+    'x\u2003y\u2009\u200az\u202f\u205f',
+    [87, 447, 225, 88, 447, 231, 447, 232, 89, 447, 107, 46256, 253],
+  ],
+  [
+    'r50k_base',
+    'vertical tab and form feed',
+    'a\u000bb\fc',
+    [64, 199, 65, 200, 66],
+  ],
+  [
+    'r50k_base',
+    'trailing whitespace',
+    'hello   \n  ',
+    [31373, 220, 220, 220, 198, 220, 220],
+  ],
+  ['r50k_base', 'leading whitespace', '   hello', [220, 220, 23748]],
+]
+
+describe('Unicode White_Space handling in the split patterns', () => {
+  // Keep this test first: the shared TextDecoder only strips a BOM on its very
+  // first decode, so later decodes in this file would hide a regression.
+  test('decode keeps U+FEFF tokens', () => {
+    const encoding = getEncoding('cl100k_base')
+    const feff = String.fromCodePoint(0xfeff)
+
+    // 3305 is the single token for the bytes EF BB BF
+    expect(encoding.decode([3305])).toBe(feff)
+    expect(encoding.decode([64, 3305, 65])).toBe(`a${feff}b`)
+    // 62619 is the single token for the bytes EF BB BF 0A
+    expect(encoding.decode([9906, 62_619])).toBe(`Hello${feff}\n`)
+  })
+
+  describe.each(unicodeWhiteSpaceCases)(
+    '%s: %j',
+    (encodingName, text, expected) => {
+      const encoding = getEncoding(encodingName)
+
+      test('encode matches tiktoken', () => {
+        expect(encoding.encode(text)).toEqual(expected)
+      })
+
+      test('countTokens agrees with encode', () => {
+        expect(encoding.countTokens(text)).toBe(expected.length)
+      })
+
+      test('decode round-trips', () => {
+        expect(encoding.decode(encoding.encode(text))).toBe(text)
+      })
+    },
+  )
+
+  describe.each(ordinaryWhiteSpaceCases)(
+    '%s: %s',
+    (encodingName, _label, text, expected) => {
+      const encoding = getEncoding(encodingName)
+
+      test('encode matches tiktoken', () => {
+        expect(encoding.encode(text)).toEqual(expected)
+      })
+
+      test('decode round-trips', () => {
+        expect(encoding.decode(encoding.encode(text))).toBe(text)
+      })
+    },
+  )
+})

package/src/utfUtil.ts

@@ -71,7 +71,18 @@ function isValidUTF8(bytes: Uint8Array): boolean {
   return true
 }
 
-const textDecoder = new TextDecoder('utf8', { fatal: false })
+export function startsWithByteOrderMark(bytes: Uint8Array): boolean {
+  return (
+    bytes.length >= 3 &&
+    bytes[0] === 0xef &&
+    bytes[1] === 0xbb &&
+    bytes[2] === 0xbf
+  )
+}
+
+// `ignoreBOM: true` keeps a leading U+FEFF in the output instead of silently
+// stripping it, so the bytes EF BB BF map to U+FEFF rather than to an empty string.
+const textDecoder = new TextDecoder('utf8', { fatal: false, ignoreBOM: true })
 
 export function tryConvertToString(arr: Uint8Array): string | undefined {
   if (!isValidUTF8(arr)) {