gpt-tokenizer 3.0.0 → 3.0.1

package/src/__snapshots__/GptEncoding.test.ts.snap

@@ -7552,6 +7552,260 @@ exports[`o3-mini-2025-01-31 > chat functionality > encodes a chat correctly 1`]
 
 exports[`o3-mini-2025-01-31 > chat functionality > encodes a chat correctly 2`] = `"<|im_start|>system<|im_sep|>You are a helpful, pattern-following assistant that translates corporate jargon into plain English.<|im_end|><|im_start|>example_user<|im_sep|>New synergies will help drive top-line growth.<|im_end|><|im_start|>example_assistant<|im_sep|>Things working well together will increase revenue.<|im_end|><|im_start|>example_user<|im_sep|>Let's circle back when we have more bandwidth to touch base on opportunities for increased leverage.<|im_end|><|im_start|>example_assistant<|im_sep|>Let's talk later when we're less busy about how to do better.<|im_end|><|im_start|>user<|im_sep|>This late pivot means we don't have time to boil the ocean for the client deliverable.<|im_end|><|im_start|>assistant<|im_sep|>"`;
 
+exports[`o3-pro > chat functionality > encodes a chat correctly 1`] = `
+[
+  200003,
+  17360,
+  200005,
+  3575,
+  553,
+  261,
+  10297,
+  11,
+  8302,
+  156275,
+  289,
+  29186,
+  484,
+  84230,
+  16706,
+  160996,
+  1511,
+  21402,
+  7725,
+  13,
+  200004,
+  200003,
+  18582,
+  7146,
+  200005,
+  3443,
+  6157,
+  109242,
+  738,
+  1652,
+  7515,
+  2344,
+  11596,
+  8266,
+  13,
+  200004,
+  200003,
+  18582,
+  29140,
+  14055,
+  200005,
+  53046,
+  4113,
+  1775,
+  4717,
+  738,
+  7064,
+  17981,
+  13,
+  200004,
+  200003,
+  18582,
+  7146,
+  200005,
+  12845,
+  885,
+  22005,
+  1602,
+  1261,
+  581,
+  679,
+  945,
+  58356,
+  316,
+  7381,
+  3611,
+  402,
+  11206,
+  395,
+  11202,
+  48255,
+  13,
+  200004,
+  200003,
+  18582,
+  29140,
+  14055,
+  200005,
+  12845,
+  885,
+  4595,
+  4849,
+  1261,
+  581,
+  4118,
+  3760,
+  15827,
+  1078,
+  1495,
+  316,
+  621,
+  3432,
+  13,
+  200004,
+  200003,
+  1428,
+  200005,
+  2500,
+  7844,
+  44506,
+  4748,
+  581,
+  1700,
+  1507,
+  679,
+  1058,
+  316,
+  66049,
+  290,
+  25472,
+  395,
+  290,
+  3054,
+  7000,
+  562,
+  13,
+  200004,
+  200003,
+  173781,
+  200005,
+]
+`;
+
+exports[`o3-pro > chat functionality > encodes a chat correctly 2`] = `"<|im_start|>system<|im_sep|>You are a helpful, pattern-following assistant that translates corporate jargon into plain English.<|im_end|><|im_start|>example_user<|im_sep|>New synergies will help drive top-line growth.<|im_end|><|im_start|>example_assistant<|im_sep|>Things working well together will increase revenue.<|im_end|><|im_start|>example_user<|im_sep|>Let's circle back when we have more bandwidth to touch base on opportunities for increased leverage.<|im_end|><|im_start|>example_assistant<|im_sep|>Let's talk later when we're less busy about how to do better.<|im_end|><|im_start|>user<|im_sep|>This late pivot means we don't have time to boil the ocean for the client deliverable.<|im_end|><|im_start|>assistant<|im_sep|>"`;
+
+exports[`o3-pro-2025-06-10 > chat functionality > encodes a chat correctly 1`] = `
+[
+  200003,
+  17360,
+  200005,
+  3575,
+  553,
+  261,
+  10297,
+  11,
+  8302,
+  156275,
+  289,
+  29186,
+  484,
+  84230,
+  16706,
+  160996,
+  1511,
+  21402,
+  7725,
+  13,
+  200004,
+  200003,
+  18582,
+  7146,
+  200005,
+  3443,
+  6157,
+  109242,
+  738,
+  1652,
+  7515,
+  2344,
+  11596,
+  8266,
+  13,
+  200004,
+  200003,
+  18582,
+  29140,
+  14055,
+  200005,
+  53046,
+  4113,
+  1775,
+  4717,
+  738,
+  7064,
+  17981,
+  13,
+  200004,
+  200003,
+  18582,
+  7146,
+  200005,
+  12845,
+  885,
+  22005,
+  1602,
+  1261,
+  581,
+  679,
+  945,
+  58356,
+  316,
+  7381,
+  3611,
+  402,
+  11206,
+  395,
+  11202,
+  48255,
+  13,
+  200004,
+  200003,
+  18582,
+  29140,
+  14055,
+  200005,
+  12845,
+  885,
+  4595,
+  4849,
+  1261,
+  581,
+  4118,
+  3760,
+  15827,
+  1078,
+  1495,
+  316,
+  621,
+  3432,
+  13,
+  200004,
+  200003,
+  1428,
+  200005,
+  2500,
+  7844,
+  44506,
+  4748,
+  581,
+  1700,
+  1507,
+  679,
+  1058,
+  316,
+  66049,
+  290,
+  25472,
+  395,
+  290,
+  3054,
+  7000,
+  562,
+  13,
+  200004,
+  200003,
+  173781,
+  200005,
+]
+`;
+
+exports[`o3-pro-2025-06-10 > chat functionality > encodes a chat correctly 2`] = `"<|im_start|>system<|im_sep|>You are a helpful, pattern-following assistant that translates corporate jargon into plain English.<|im_end|><|im_start|>example_user<|im_sep|>New synergies will help drive top-line growth.<|im_end|><|im_start|>example_assistant<|im_sep|>Things working well together will increase revenue.<|im_end|><|im_start|>example_user<|im_sep|>Let's circle back when we have more bandwidth to touch base on opportunities for increased leverage.<|im_end|><|im_start|>example_assistant<|im_sep|>Let's talk later when we're less busy about how to do better.<|im_end|><|im_start|>user<|im_sep|>This late pivot means we don't have time to boil the ocean for the client deliverable.<|im_end|><|im_start|>assistant<|im_sep|>"`;
+
 exports[`o4-mini > chat functionality > encodes a chat correctly 1`] = `
 [
   200003,

package/src/codegen/modelScape.js

@@ -1,3 +1,5 @@
+// search for name: "Other models" in https://platform.openai.com/docs/models
+
 /**
  * Converts a string with hyphens and dots into a valid JavaScript variable name.
  * @param {string} name - The input string (e.g., 'gpt-4.1-mini').

package/src/model/o3-2025-04-16.ts

@@ -5,7 +5,7 @@ import { GptEncoding } from '../GptEncoding.js'
 export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
-const api = GptEncoding.getEncodingApiForModel('o3-2025-04-16', () => bpeRanks, {name:"o3-2025-04-16",slug:"o3-2025-04-16",performance:5,latency:1,modalities:{input:["text","image"],output:["text"]},context_window:200000,max_output_tokens:100000,knowledge_cutoff:new Date(1717200000000),supported_features:["streaming","structured_outputs","file_search","function_calling","file_uploads","image_input","prompt_caching","evals","stored_completions"],supported_endpoints:["chat_completions","responses","batch"],reasoning_tokens:true,price_data:{main:{input:15,cached_output:7.5,output:60},batch:{input:7.5,output:30}}})
+const api = GptEncoding.getEncodingApiForModel('o3-2025-04-16', () => bpeRanks, {name:"o3-2025-04-16",slug:"o3-2025-04-16",performance:5,latency:1,modalities:{input:["text","image"],output:["text"]},context_window:200000,max_output_tokens:100000,knowledge_cutoff:new Date(1717200000000),supported_features:["streaming","structured_outputs","file_search","function_calling","file_uploads","image_input","prompt_caching","evals","stored_completions"],supported_endpoints:["chat_completions","responses","batch"],reasoning_tokens:true,price_data:{main:{input:2,cached_output:.5,output:8},batch:{input:1,output:4}}})
 const {
   decode,
   decodeAsyncGenerator,

package/src/model/o3-pro-2025-06-10.ts

@@ -0,0 +1,40 @@
+/* eslint-disable import/extensions */
+import bpeRanks from '../bpeRanks/o200k_base.js'
+import { GptEncoding } from '../GptEncoding.js'
+
+export * from '../constants.js'
+export * from '../specialTokens.js'
+// prettier-ignore
+const api = GptEncoding.getEncodingApiForModel('o3-pro-2025-06-10', () => bpeRanks, {name:"o3-pro-2025-06-10",slug:"o3-pro-2025-06-10",performance:5,latency:1,modalities:{input:["text","image"],output:["text"]},context_window:200000,max_output_tokens:100000,knowledge_cutoff:new Date(1717200000000),supported_features:["structured_outputs","function_calling","image_input"],supported_endpoints:["responses","batch"],reasoning_tokens:true,price_data:{main:{input:20,output:80},batch:{input:10,output:40}}})
+const {
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeGenerator,
+  isWithinTokenLimit,
+  countTokens,
+  encodeChat,
+  encodeChatGenerator,
+  vocabularySize,
+  setMergeCacheSize,
+  clearMergeCache,
+  estimateCost,
+} = api
+export {
+  clearMergeCache,
+  countTokens,
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeChat,
+  encodeChatGenerator,
+  encodeGenerator,
+  estimateCost,
+  isWithinTokenLimit,
+  setMergeCacheSize,
+  vocabularySize,
+}
+// eslint-disable-next-line import/no-default-export
+export default api

package/src/model/o3-pro.ts

@@ -0,0 +1,40 @@
+/* eslint-disable import/extensions */
+import bpeRanks from '../bpeRanks/o200k_base.js'
+import { GptEncoding } from '../GptEncoding.js'
+
+export * from '../constants.js'
+export * from '../specialTokens.js'
+// prettier-ignore
+const api = GptEncoding.getEncodingApiForModel('o3-pro', () => bpeRanks, {name:"o3-pro-2025-06-10",slug:"o3-pro-2025-06-10",performance:5,latency:1,modalities:{input:["text","image"],output:["text"]},context_window:200000,max_output_tokens:100000,knowledge_cutoff:new Date(1717200000000),supported_features:["structured_outputs","function_calling","image_input"],supported_endpoints:["responses","batch"],reasoning_tokens:true,price_data:{main:{input:20,output:80},batch:{input:10,output:40}}})
+const {
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeGenerator,
+  isWithinTokenLimit,
+  countTokens,
+  encodeChat,
+  encodeChatGenerator,
+  vocabularySize,
+  setMergeCacheSize,
+  clearMergeCache,
+  estimateCost,
+} = api
+export {
+  clearMergeCache,
+  countTokens,
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeChat,
+  encodeChatGenerator,
+  encodeGenerator,
+  estimateCost,
+  isWithinTokenLimit,
+  setMergeCacheSize,
+  vocabularySize,
+}
+// eslint-disable-next-line import/no-default-export
+export default api

package/src/model/o3.ts

@@ -5,7 +5,7 @@ import { GptEncoding } from '../GptEncoding.js'
 export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
-const api = GptEncoding.getEncodingApiForModel('o3', () => bpeRanks, {name:"o3-2025-04-16",slug:"o3-2025-04-16",performance:5,latency:1,modalities:{input:["text","image"],output:["text"]},context_window:200000,max_output_tokens:100000,knowledge_cutoff:new Date(1717200000000),supported_features:["streaming","structured_outputs","file_search","function_calling","file_uploads","image_input","prompt_caching","evals","stored_completions"],supported_endpoints:["chat_completions","responses","batch"],reasoning_tokens:true,price_data:{main:{input:15,cached_output:7.5,output:60},batch:{input:7.5,output:30}}})
+const api = GptEncoding.getEncodingApiForModel('o3', () => bpeRanks, {name:"o3-2025-04-16",slug:"o3-2025-04-16",performance:5,latency:1,modalities:{input:["text","image"],output:["text"]},context_window:200000,max_output_tokens:100000,knowledge_cutoff:new Date(1717200000000),supported_features:["streaming","structured_outputs","file_search","function_calling","file_uploads","image_input","prompt_caching","evals","stored_completions"],supported_endpoints:["chat_completions","responses","batch"],reasoning_tokens:true,price_data:{main:{input:2,cached_output:.5,output:8},batch:{input:1,output:4}}})
 const {
   decode,
   decodeAsyncGenerator,

package/src/models.gen.ts

@@ -3343,6 +3343,98 @@ export {o3_mini_2025_01_31_spec as 'o3-mini-2025-01-31'}
 // alias:
 export { o3_mini_2025_01_31_spec as 'o3-mini' };
 
+const o3_pro_config = {
+  name: 'o3-pro',
+  slug: 'o3-pro',
+  current_snapshot: 'o3-pro-2025-06-10',
+  tagline: 'Version of o3 with more compute for better responses',
+  description: 'The o-series of models are trained with reinforcement learning to think \nbefore they answer and perform complex reasoning. The o3-pro model uses more \ncompute to think harder and provide consistently better answers.\n\no3-pro is available in the [Responses API only](/docs/api-reference/responses)\nto enable support for multi-turn model interactions before responding to API \nrequests, and other advanced API features in the future. Since o3-pro is designed \nto tackle tough problems, some requests may take several minutes to finish. \nTo avoid timeouts, try using [background mode](/docs/guides/background).\n',
+  type: 'reasoning',
+  snapshots: [
+    'o3-pro-2025-06-10',
+  ],
+  compare_prices: [
+    'o3',
+    'o3-mini',
+  ],
+  supported_tools: [
+    'function_calling',
+    'file_search',
+    'image_generation',
+    'mcp',
+  ],
+  rate_limits: {
+    tier_1: {
+      rpm: 500,
+      tpm: 3e4,
+      batch_queue_limit: 9e4,
+    },
+    tier_2: {
+      rpm: 5e3,
+      tpm: 45e4,
+      batch_queue_limit: 135e4,
+    },
+    tier_3: {
+      rpm: 5e3,
+      tpm: 8e5,
+      batch_queue_limit: 5e7,
+    },
+    tier_4: {
+      rpm: 1e4,
+      tpm: 2e6,
+      batch_queue_limit: 2e8,
+    },
+    tier_5: {
+      rpm: 1e4,
+      tpm: 3e7,
+      batch_queue_limit: 5e9,
+    },
+  },
+} as const satisfies ModelConfig
+
+const o3_pro_2025_06_10_spec = {
+  name: 'o3-pro-2025-06-10',
+  slug: 'o3-pro-2025-06-10',
+  performance: 5,
+  latency: 1,
+  modalities: {
+    input: [
+      'text',
+      'image',
+    ],
+    output: [
+      'text',
+    ],
+  },
+  context_window: 2e5,
+  max_output_tokens: 1e5,
+  knowledge_cutoff: new Date(17172e8),
+  supported_features: [
+    'structured_outputs',
+    'function_calling',
+    'image_input',
+  ],
+  supported_endpoints: [
+    'responses',
+    'batch',
+  ],
+  reasoning_tokens: true,
+  price_data: {
+    main: {
+      input: 20,
+      output: 80,
+    },
+    batch: {
+      input: 10,
+      output: 40,
+    },
+  },
+} as const satisfies ModelSpec
+export {o3_pro_2025_06_10_spec as 'o3-pro-2025-06-10'}
+
+// alias:
+export { o3_pro_2025_06_10_spec as 'o3-pro' };
+
 const o3_config = {
   name: 'o3',
   slug: 'o3',
@@ -3429,13 +3521,13 @@ const o3_2025_04_16_spec = {
   reasoning_tokens: true,
   price_data: {
     main: {
-      input: 15,
-      cached_output: 7.5,
-      output: 60,
+      input: 2,
+      cached_output: .5,
+      output: 8,
     },
     batch: {
-      input: 7.5,
-      output: 30,
+      input: 1,
+      output: 4,
     },
   },
 } as const satisfies ModelSpec

package/src/modelsChatEnabled.gen.ts

@@ -1 +1 @@
-export const chatEnabledModels = ["chatgpt-4o-latest","codex-mini-latest","computer-use-preview","computer-use-preview-2025-03-11","gpt-3.5","gpt-3.5-0301","gpt-3.5-turbo","gpt-3.5-turbo-0125","gpt-3.5-turbo-0613","gpt-3.5-turbo-1106","gpt-3.5-turbo-16k-0613","gpt-3.5-turbo-instruct","gpt-4","gpt-4-0125-preview","gpt-4-0314","gpt-4-0613","gpt-4-1106-preview","gpt-4-1106-vision-preview","gpt-4-32k","gpt-4-turbo","gpt-4-turbo-2024-04-09","gpt-4-turbo-preview","gpt-4.1","gpt-4.1-2025-04-14","gpt-4.1-mini","gpt-4.1-mini-2025-04-14","gpt-4.1-nano","gpt-4.1-nano-2025-04-14","gpt-4.5-preview","gpt-4.5-preview-2025-02-27","gpt-4o","gpt-4o-2024-05-13","gpt-4o-2024-08-06","gpt-4o-2024-11-20","gpt-4o-audio-preview","gpt-4o-audio-preview-2024-10-01","gpt-4o-audio-preview-2024-12-17","gpt-4o-audio-preview-2025-06-03","gpt-4o-mini","gpt-4o-mini-2024-07-18","gpt-4o-mini-audio-preview","gpt-4o-mini-audio-preview-2024-12-17","gpt-4o-mini-search-preview","gpt-4o-mini-search-preview-2025-03-11","gpt-4o-search-preview","gpt-4o-search-preview-2025-03-11","o1","o1-2024-12-17","o1-mini","o1-mini-2024-09-12","o1-preview","o1-preview-2024-09-12","o1-pro","o1-pro-2025-03-19","o3","o3-2025-04-16","o3-mini","o3-mini-2025-01-31","o4-mini","o4-mini-2025-04-16"] as const;
+export const chatEnabledModels = ["chatgpt-4o-latest","codex-mini-latest","computer-use-preview","computer-use-preview-2025-03-11","gpt-3.5","gpt-3.5-0301","gpt-3.5-turbo","gpt-3.5-turbo-0125","gpt-3.5-turbo-0613","gpt-3.5-turbo-1106","gpt-3.5-turbo-16k-0613","gpt-3.5-turbo-instruct","gpt-4","gpt-4-0125-preview","gpt-4-0314","gpt-4-0613","gpt-4-1106-preview","gpt-4-1106-vision-preview","gpt-4-32k","gpt-4-turbo","gpt-4-turbo-2024-04-09","gpt-4-turbo-preview","gpt-4.1","gpt-4.1-2025-04-14","gpt-4.1-mini","gpt-4.1-mini-2025-04-14","gpt-4.1-nano","gpt-4.1-nano-2025-04-14","gpt-4.5-preview","gpt-4.5-preview-2025-02-27","gpt-4o","gpt-4o-2024-05-13","gpt-4o-2024-08-06","gpt-4o-2024-11-20","gpt-4o-audio-preview","gpt-4o-audio-preview-2024-10-01","gpt-4o-audio-preview-2024-12-17","gpt-4o-audio-preview-2025-06-03","gpt-4o-mini","gpt-4o-mini-2024-07-18","gpt-4o-mini-audio-preview","gpt-4o-mini-audio-preview-2024-12-17","gpt-4o-mini-search-preview","gpt-4o-mini-search-preview-2025-03-11","gpt-4o-search-preview","gpt-4o-search-preview-2025-03-11","o1","o1-2024-12-17","o1-mini","o1-mini-2024-09-12","o1-preview","o1-preview-2024-09-12","o1-pro","o1-pro-2025-03-19","o3","o3-2025-04-16","o3-mini","o3-mini-2025-01-31","o3-pro","o3-pro-2025-06-10","o4-mini","o4-mini-2025-04-16"] as const;