ai-token-estimator 1.0.2 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. package/README.md +110 -5
  2. package/dist/index.cjs +347 -18
  3. package/dist/index.d.cts +64 -4
  4. package/dist/index.d.ts +64 -4
  5. package/dist/index.js +343 -18
  6. package/package.json +6 -2
package/README.md CHANGED
@@ -39,6 +39,29 @@ console.log(getAvailableModels());
39
39
  // ['gpt-5.2', 'gpt-4o', 'claude-opus-4.5', 'gemini-3-pro', ...]
40
40
  ```
41
41
 
42
+ ## Exact OpenAI tokenization (BPE)
43
+
44
+ This package includes **exact tokenization for OpenAI models** using a tiktoken-compatible BPE tokenizer (via `gpt-tokenizer`).
45
+
46
+ Notes:
47
+ - Encodings are **lazy-loaded on first use** (one-time cost per encoding).
48
+ - Exact tokenization is **slower** than heuristic estimation; `estimate()` defaults to `'heuristic'` to keep existing behavior fast.
49
+ - `encode` / `decode` and `estimate({ tokenizer: 'openai_exact' })` require **Node.js** (uses `node:module` under the hood).
50
+
51
+ ```ts
52
+ import { encode, decode } from 'ai-token-estimator';
53
+
54
+ const text = 'Hello, world!';
55
+ const tokens = encode(text, { model: 'gpt-5.1' }); // exact OpenAI token IDs
56
+ const roundTrip = decode(tokens, { model: 'gpt-5.1' });
57
+
58
+ console.log(tokens.length);
59
+ console.log(roundTrip); // "Hello, world!"
60
+ ```
61
+
62
+ Supported encodings:
63
+ `r50k_base`, `p50k_base`, `p50k_edit`, `cl100k_base`, `o200k_base`, `o200k_harmony`
64
+
42
65
  ## API Reference
43
66
 
44
67
  ### `estimate(input: EstimateInput): EstimateOutput`
@@ -52,6 +75,7 @@ interface EstimateInput {
52
75
  text: string; // The text to estimate tokens for
53
76
  model: string; // Model ID (e.g., 'gpt-4o', 'claude-opus-4.5')
54
77
  rounding?: 'ceil' | 'round' | 'floor'; // Rounding strategy (default: 'ceil')
78
+ tokenizer?: 'heuristic' | 'openai_exact' | 'auto'; // Token counting strategy (default: 'heuristic')
55
79
  }
56
80
  ```
57
81
 
@@ -64,13 +88,36 @@ interface EstimateOutput {
64
88
  estimatedTokens: number; // Estimated token count (integer)
65
89
  estimatedInputCost: number; // Estimated cost in USD
66
90
  charsPerToken: number; // The ratio used for this model
91
+ tokenizerMode?: 'heuristic' | 'openai_exact' | 'auto'; // Which strategy was used
92
+ encodingUsed?: string; // OpenAI encoding when using exact tokenization
67
93
  }
68
94
  ```
69
95
 
96
+ ### `countTokens(input: TokenCountInput): TokenCountOutput`
97
+
98
+ Counts tokens for a given model:
99
+ - OpenAI models: **exact** BPE tokenization
100
+ - Other providers: heuristic estimate
101
+
102
+ ```ts
103
+ import { countTokens } from 'ai-token-estimator';
104
+
105
+ const result = countTokens({ text: 'Hello, world!', model: 'gpt-5.1' });
106
+ // { tokens: 4, exact: true, encoding: 'o200k_base' }
107
+ ```
108
+
70
109
  ### `getAvailableModels(): string[]`
71
110
 
72
111
  Returns an array of all supported model IDs.
73
112
 
113
+ ### `encode(text: string, options?: EncodeOptions): number[]`
114
+
115
+ Encodes text into **OpenAI token IDs** using tiktoken-compatible BPE tokenization.
116
+
117
+ ### `decode(tokens: Iterable<number>, options?: { encoding?: OpenAIEncoding; model?: string }): string`
118
+
119
+ Decodes OpenAI token IDs back into text using the selected encoding/model.
120
+
74
121
  ### `getModelConfig(model: string): ModelConfig`
75
122
 
76
123
  Returns the configuration for a specific model. Throws if the model is not found.
@@ -108,6 +155,14 @@ This package counts Unicode code points, not UTF-16 code units. This means:
108
155
  - Accented characters count correctly
109
156
  - Most source code characters count as 1
110
157
 
158
+ ## Benchmarks (repo only)
159
+
160
+ This repository includes a small benchmark script to compare heuristic vs exact OpenAI tokenization:
161
+
162
+ ```bash
163
+ npm run benchmark:tokenizer
164
+ ```
165
+
111
166
  <!-- SUPPORTED_MODELS_START -->
112
167
  ## Supported Models
113
168
 
@@ -117,20 +172,70 @@ This package counts Unicode code points, not UTF-16 code units. This means:
117
172
 
118
173
  | Model | Chars/Token | Input Cost (per 1M tokens) |
119
174
  |-------|-------------|---------------------------|
120
- | gpt-4.1 | 4 | $3.00 |
175
+ | babbage-002 | 4 | $0.40 |
176
+ | chatgpt-4o-latest | 4 | $5.00 |
177
+ | chatgpt-image-latest | 4 | $5.00 |
178
+ | codex-mini-latest | 4 | $1.50 |
179
+ | computer-use-preview | 4 | $3.00 |
180
+ | davinci-002 | 4 | $2.00 |
181
+ | gpt-3.5-0301 | 4 | $1.50 |
182
+ | gpt-3.5-turbo | 4 | $0.50 |
183
+ | gpt-3.5-turbo-0125 | 4 | $0.50 |
184
+ | gpt-3.5-turbo-0613 | 4 | $1.50 |
185
+ | gpt-3.5-turbo-1106 | 4 | $1.00 |
186
+ | gpt-3.5-turbo-16k-0613 | 4 | $3.00 |
187
+ | gpt-3.5-turbo-instruct | 4 | $1.50 |
188
+ | gpt-4-0125-preview | 4 | $10.00 |
189
+ | gpt-4-0314 | 4 | $30.00 |
190
+ | gpt-4-0613 | 4 | $30.00 |
191
+ | gpt-4-1106-preview | 4 | $10.00 |
192
+ | gpt-4-1106-vision-preview | 4 | $10.00 |
193
+ | gpt-4-32k | 4 | $60.00 |
194
+ | gpt-4-turbo-2024-04-09 | 4 | $10.00 |
195
+ | gpt-4.1 | 4 | $2.00 |
121
196
  | gpt-4.1-mini | 4 | $0.40 |
122
197
  | gpt-4.1-nano | 4 | $0.10 |
123
198
  | gpt-4o | 4 | $2.50 |
199
+ | gpt-4o-2024-05-13 | 4 | $5.00 |
200
+ | gpt-4o-audio-preview | 4 | $2.50 |
124
201
  | gpt-4o-mini | 4 | $0.15 |
202
+ | gpt-4o-mini-audio-preview | 4 | $0.15 |
203
+ | gpt-4o-mini-realtime-preview | 4 | $0.60 |
204
+ | gpt-4o-mini-search-preview | 4 | $0.15 |
205
+ | gpt-4o-realtime-preview | 4 | $5.00 |
206
+ | gpt-4o-search-preview | 4 | $2.50 |
207
+ | gpt-5 | 4 | $1.25 |
208
+ | gpt-5-chat-latest | 4 | $1.25 |
209
+ | gpt-5-codex | 4 | $1.25 |
125
210
  | gpt-5-mini | 4 | $0.25 |
211
+ | gpt-5-nano | 4 | $0.05 |
212
+ | gpt-5-pro | 4 | $15.00 |
213
+ | gpt-5-search-api | 4 | $1.25 |
214
+ | gpt-5.1 | 4 | $1.25 |
215
+ | gpt-5.1-chat-latest | 4 | $1.25 |
216
+ | gpt-5.1-codex | 4 | $1.25 |
217
+ | gpt-5.1-codex-max | 4 | $1.25 |
218
+ | gpt-5.1-codex-mini | 4 | $0.25 |
126
219
  | gpt-5.2 | 4 | $1.75 |
220
+ | gpt-5.2-chat-latest | 4 | $1.75 |
221
+ | gpt-5.2-codex | 4 | $1.75 |
127
222
  | gpt-5.2-pro | 4 | $21.00 |
223
+ | gpt-audio | 4 | $2.50 |
224
+ | gpt-audio-mini | 4 | $0.60 |
225
+ | gpt-image-1 | 4 | $5.00 |
226
+ | gpt-image-1-mini | 4 | $2.00 |
227
+ | gpt-image-1.5 | 4 | $5.00 |
128
228
  | gpt-realtime | 4 | $4.00 |
129
229
  | gpt-realtime-mini | 4 | $0.60 |
130
230
  | o1 | 4 | $15.00 |
231
+ | o1-mini | 4 | $1.10 |
131
232
  | o1-pro | 4 | $150.00 |
132
233
  | o3 | 4 | $2.00 |
133
- | o4-mini | 4 | $4.00 |
234
+ | o3-deep-research | 4 | $10.00 |
235
+ | o3-mini | 4 | $1.10 |
236
+ | o3-pro | 4 | $20.00 |
237
+ | o4-mini | 4 | $1.10 |
238
+ | o4-mini-deep-research | 4 | $2.00 |
134
239
 
135
240
  ### Anthropic Claude Models
136
241
 
@@ -164,13 +269,13 @@ This package counts Unicode code points, not UTF-16 code units. This means:
164
269
  | gemini-3-flash | 4 | $0.50 |
165
270
  | gemini-3-pro | 4 | $2.00 |
166
271
 
167
- *Last updated: 2025-12-25*
272
+ *Last updated: 2026-01-14*
168
273
  <!-- SUPPORTED_MODELS_END -->
169
274
 
170
275
  ## Pricing Updates
171
276
 
172
277
  Model pricing is automatically updated weekly via GitHub Actions. The update script fetches the latest prices directly from:
173
- - [OpenAI Pricing](https://openai.com/api/pricing/)
278
+ - [OpenAI Pricing](https://platform.openai.com/docs/pricing)
174
279
  - [Anthropic Pricing](https://www.anthropic.com/pricing)
175
280
  - [Google AI Pricing](https://ai.google.dev/gemini-api/docs/pricing)
176
281
 
@@ -178,7 +283,7 @@ You can check when prices were last updated:
178
283
 
179
284
  ```typescript
180
285
  import { LAST_UPDATED } from 'ai-token-estimator';
181
- console.log(LAST_UPDATED); // '2025-12-25'
286
+ console.log(LAST_UPDATED); // e.g. '2026-01-14'
182
287
  ```
183
288
 
184
289
  ## License
package/dist/index.cjs CHANGED
@@ -22,6 +22,9 @@ var index_exports = {};
22
22
  __export(index_exports, {
23
23
  DEFAULT_MODELS: () => DEFAULT_MODELS,
24
24
  LAST_UPDATED: () => LAST_UPDATED,
25
+ countTokens: () => countTokens,
26
+ decode: () => decode,
27
+ encode: () => encode,
25
28
  estimate: () => estimate,
26
29
  getAvailableModels: () => getAvailableModels,
27
30
  getModelConfig: () => getModelConfig
@@ -29,44 +32,224 @@ __export(index_exports, {
29
32
  module.exports = __toCommonJS(index_exports);
30
33
 
31
34
  // src/models.ts
32
- var LAST_UPDATED = "2025-12-25";
35
+ var LAST_UPDATED = "2026-01-14";
33
36
  var models = {
34
37
  // ===================
35
38
  // OpenAI Models
36
39
  // ===================
37
40
  // OpenAI uses ~4 chars per token for English text
38
- "gpt-4.1": {
41
+ "babbage-002": {
42
+ charsPerToken: 4,
43
+ inputCostPerMillion: 0.4
44
+ },
45
+ "chatgpt-4o-latest": {
46
+ charsPerToken: 4,
47
+ inputCostPerMillion: 5
48
+ },
49
+ "chatgpt-image-latest": {
50
+ charsPerToken: 4,
51
+ inputCostPerMillion: 5
52
+ },
53
+ "codex-mini-latest": {
54
+ charsPerToken: 4,
55
+ inputCostPerMillion: 1.5
56
+ },
57
+ "computer-use-preview": {
58
+ charsPerToken: 4,
59
+ inputCostPerMillion: 3
60
+ },
61
+ "davinci-002": {
62
+ charsPerToken: 4,
63
+ inputCostPerMillion: 2
64
+ },
65
+ "gpt-3.5-0301": {
66
+ charsPerToken: 4,
67
+ inputCostPerMillion: 1.5
68
+ },
69
+ "gpt-3.5-turbo": {
70
+ charsPerToken: 4,
71
+ inputCostPerMillion: 0.5
72
+ },
73
+ "gpt-3.5-turbo-0125": {
74
+ charsPerToken: 4,
75
+ inputCostPerMillion: 0.5
76
+ },
77
+ "gpt-3.5-turbo-0613": {
78
+ charsPerToken: 4,
79
+ inputCostPerMillion: 1.5
80
+ },
81
+ "gpt-3.5-turbo-1106": {
82
+ charsPerToken: 4,
83
+ inputCostPerMillion: 1
84
+ },
85
+ "gpt-3.5-turbo-16k-0613": {
39
86
  charsPerToken: 4,
40
87
  inputCostPerMillion: 3
41
88
  },
89
+ "gpt-3.5-turbo-instruct": {
90
+ charsPerToken: 4,
91
+ inputCostPerMillion: 1.5
92
+ },
93
+ "gpt-4-0125-preview": {
94
+ charsPerToken: 4,
95
+ inputCostPerMillion: 10
96
+ },
97
+ "gpt-4-0314": {
98
+ charsPerToken: 4,
99
+ inputCostPerMillion: 30
100
+ },
101
+ "gpt-4-0613": {
102
+ charsPerToken: 4,
103
+ inputCostPerMillion: 30
104
+ },
105
+ "gpt-4-1106-preview": {
106
+ charsPerToken: 4,
107
+ inputCostPerMillion: 10
108
+ },
109
+ "gpt-4-1106-vision-preview": {
110
+ charsPerToken: 4,
111
+ inputCostPerMillion: 10
112
+ },
113
+ "gpt-4-32k": {
114
+ charsPerToken: 4,
115
+ inputCostPerMillion: 60
116
+ },
117
+ "gpt-4-turbo-2024-04-09": {
118
+ charsPerToken: 4,
119
+ inputCostPerMillion: 10
120
+ },
121
+ "gpt-4.1": {
122
+ charsPerToken: 4,
123
+ inputCostPerMillion: 2
124
+ },
42
125
  "gpt-4.1-mini": {
43
126
  charsPerToken: 4,
44
- inputCostPerMillion: 0.8
127
+ inputCostPerMillion: 0.4
45
128
  },
46
129
  "gpt-4.1-nano": {
47
130
  charsPerToken: 4,
48
- inputCostPerMillion: 0.2
131
+ inputCostPerMillion: 0.1
49
132
  },
50
133
  "gpt-4o": {
51
134
  charsPerToken: 4,
52
135
  inputCostPerMillion: 2.5
53
136
  },
137
+ "gpt-4o-2024-05-13": {
138
+ charsPerToken: 4,
139
+ inputCostPerMillion: 5
140
+ },
141
+ "gpt-4o-audio-preview": {
142
+ charsPerToken: 4,
143
+ inputCostPerMillion: 2.5
144
+ },
54
145
  "gpt-4o-mini": {
55
146
  charsPerToken: 4,
56
147
  inputCostPerMillion: 0.15
57
148
  },
149
+ "gpt-4o-mini-audio-preview": {
150
+ charsPerToken: 4,
151
+ inputCostPerMillion: 0.15
152
+ },
153
+ "gpt-4o-mini-realtime-preview": {
154
+ charsPerToken: 4,
155
+ inputCostPerMillion: 0.6
156
+ },
157
+ "gpt-4o-mini-search-preview": {
158
+ charsPerToken: 4,
159
+ inputCostPerMillion: 0.15
160
+ },
161
+ "gpt-4o-realtime-preview": {
162
+ charsPerToken: 4,
163
+ inputCostPerMillion: 5
164
+ },
165
+ "gpt-4o-search-preview": {
166
+ charsPerToken: 4,
167
+ inputCostPerMillion: 2.5
168
+ },
169
+ "gpt-5": {
170
+ charsPerToken: 4,
171
+ inputCostPerMillion: 1.25
172
+ },
58
173
  "gpt-5-mini": {
59
174
  charsPerToken: 4,
60
175
  inputCostPerMillion: 0.25
61
176
  },
177
+ "gpt-5-nano": {
178
+ charsPerToken: 4,
179
+ inputCostPerMillion: 0.05
180
+ },
181
+ "gpt-5-pro": {
182
+ charsPerToken: 4,
183
+ inputCostPerMillion: 15
184
+ },
185
+ "gpt-5-search-api": {
186
+ charsPerToken: 4,
187
+ inputCostPerMillion: 1.25
188
+ },
189
+ "gpt-5.1": {
190
+ charsPerToken: 4,
191
+ inputCostPerMillion: 1.25
192
+ },
193
+ "gpt-5.1-chat-latest": {
194
+ charsPerToken: 4,
195
+ inputCostPerMillion: 1.25
196
+ },
197
+ "gpt-5.1-codex": {
198
+ charsPerToken: 4,
199
+ inputCostPerMillion: 1.25
200
+ },
201
+ "gpt-5.1-codex-max": {
202
+ charsPerToken: 4,
203
+ inputCostPerMillion: 1.25
204
+ },
205
+ "gpt-5.1-codex-mini": {
206
+ charsPerToken: 4,
207
+ inputCostPerMillion: 0.25
208
+ },
209
+ "gpt-5-chat-latest": {
210
+ charsPerToken: 4,
211
+ inputCostPerMillion: 1.25
212
+ },
213
+ "gpt-5-codex": {
214
+ charsPerToken: 4,
215
+ inputCostPerMillion: 1.25
216
+ },
62
217
  "gpt-5.2": {
63
218
  charsPerToken: 4,
64
219
  inputCostPerMillion: 1.75
65
220
  },
221
+ "gpt-5.2-chat-latest": {
222
+ charsPerToken: 4,
223
+ inputCostPerMillion: 1.75
224
+ },
225
+ "gpt-5.2-codex": {
226
+ charsPerToken: 4,
227
+ inputCostPerMillion: 1.75
228
+ },
66
229
  "gpt-5.2-pro": {
67
230
  charsPerToken: 4,
68
231
  inputCostPerMillion: 21
69
232
  },
233
+ "gpt-audio": {
234
+ charsPerToken: 4,
235
+ inputCostPerMillion: 2.5
236
+ },
237
+ "gpt-audio-mini": {
238
+ charsPerToken: 4,
239
+ inputCostPerMillion: 0.6
240
+ },
241
+ "gpt-image-1": {
242
+ charsPerToken: 4,
243
+ inputCostPerMillion: 5
244
+ },
245
+ "gpt-image-1-mini": {
246
+ charsPerToken: 4,
247
+ inputCostPerMillion: 2
248
+ },
249
+ "gpt-image-1.5": {
250
+ charsPerToken: 4,
251
+ inputCostPerMillion: 5
252
+ },
70
253
  "gpt-realtime": {
71
254
  charsPerToken: 4,
72
255
  inputCostPerMillion: 4
@@ -79,6 +262,10 @@ var models = {
79
262
  charsPerToken: 4,
80
263
  inputCostPerMillion: 15
81
264
  },
265
+ "o1-mini": {
266
+ charsPerToken: 4,
267
+ inputCostPerMillion: 1.1
268
+ },
82
269
  "o1-pro": {
83
270
  charsPerToken: 4,
84
271
  inputCostPerMillion: 150
@@ -87,9 +274,25 @@ var models = {
87
274
  charsPerToken: 4,
88
275
  inputCostPerMillion: 2
89
276
  },
277
+ "o3-deep-research": {
278
+ charsPerToken: 4,
279
+ inputCostPerMillion: 10
280
+ },
281
+ "o3-mini": {
282
+ charsPerToken: 4,
283
+ inputCostPerMillion: 1.1
284
+ },
285
+ "o3-pro": {
286
+ charsPerToken: 4,
287
+ inputCostPerMillion: 20
288
+ },
90
289
  "o4-mini": {
91
290
  charsPerToken: 4,
92
- inputCostPerMillion: 4
291
+ inputCostPerMillion: 1.1
292
+ },
293
+ "o4-mini-deep-research": {
294
+ charsPerToken: 4,
295
+ inputCostPerMillion: 2
93
296
  },
94
297
  // ===================
95
298
  // Anthropic Models
@@ -204,6 +407,79 @@ function getAvailableModels() {
204
407
  return Object.keys(DEFAULT_MODELS);
205
408
  }
206
409
 
410
+ // src/openai-bpe.ts
411
+ var import_node_module = require("module");
412
+ var import_constants = require("gpt-tokenizer/constants");
413
+ var import_mapping = require("gpt-tokenizer/mapping");
414
+ var import_meta = {};
415
+ var requireBase = typeof __filename === "string" && __filename.length > 0 ? __filename : import_meta.url;
416
+ var NODE_REQUIRE = (0, import_node_module.createRequire)(requireBase);
417
+ var ENCODING_MODULES = {
418
+ r50k_base: "gpt-tokenizer/cjs/encoding/r50k_base",
419
+ p50k_base: "gpt-tokenizer/cjs/encoding/p50k_base",
420
+ p50k_edit: "gpt-tokenizer/cjs/encoding/p50k_edit",
421
+ cl100k_base: "gpt-tokenizer/cjs/encoding/cl100k_base",
422
+ o200k_base: "gpt-tokenizer/cjs/encoding/o200k_base",
423
+ o200k_harmony: "gpt-tokenizer/cjs/encoding/o200k_harmony"
424
+ };
425
+ var encodingApiCache = /* @__PURE__ */ new Map();
426
+ function getEncodingApi(encoding) {
427
+ const cached = encodingApiCache.get(encoding);
428
+ if (cached) return cached;
429
+ const modulePath = ENCODING_MODULES[encoding];
430
+ const mod = NODE_REQUIRE(modulePath);
431
+ const api = { encode: mod.encode, decode: mod.decode };
432
+ encodingApiCache.set(encoding, api);
433
+ return api;
434
+ }
435
+ function resolveEncoding(selector) {
436
+ if (selector?.encoding) {
437
+ return selector.encoding;
438
+ }
439
+ const model = selector?.model?.trim();
440
+ if (model) {
441
+ const mapped = import_mapping.modelToEncodingMap[model];
442
+ if (mapped) {
443
+ return mapped;
444
+ }
445
+ }
446
+ return import_mapping.DEFAULT_ENCODING;
447
+ }
448
+ function getOpenAIEncoding(selector) {
449
+ return resolveEncoding(selector);
450
+ }
451
+ function toGptTokenizerEncodeOptions(allowSpecial) {
452
+ const mode = allowSpecial ?? "none_raise";
453
+ switch (mode) {
454
+ case "all":
455
+ return {
456
+ allowedSpecial: import_constants.ALL_SPECIAL_TOKENS,
457
+ disallowedSpecial: /* @__PURE__ */ new Set()
458
+ };
459
+ case "none":
460
+ return {
461
+ allowedSpecial: /* @__PURE__ */ new Set(),
462
+ disallowedSpecial: /* @__PURE__ */ new Set()
463
+ };
464
+ case "none_raise":
465
+ default:
466
+ return {
467
+ disallowedSpecial: import_constants.ALL_SPECIAL_TOKENS
468
+ };
469
+ }
470
+ }
471
+ function encode(text, options) {
472
+ const encoding = resolveEncoding(options);
473
+ const api = getEncodingApi(encoding);
474
+ const encodeOptions = toGptTokenizerEncodeOptions(options?.allowSpecial);
475
+ return api.encode(text, encodeOptions);
476
+ }
477
+ function decode(tokens, options) {
478
+ const encoding = resolveEncoding(options);
479
+ const api = getEncodingApi(encoding);
480
+ return api.decode(tokens);
481
+ }
482
+
207
483
  // src/estimator.ts
208
484
  function countCodePoints(text) {
209
485
  let count = 0;
@@ -213,21 +489,43 @@ function countCodePoints(text) {
213
489
  return count;
214
490
  }
215
491
  function estimate(input) {
216
- const { text, model, rounding = "ceil" } = input;
492
+ const { text, model, rounding = "ceil", tokenizer = "heuristic" } = input;
217
493
  const config = getModelConfig(model);
218
494
  const characterCount = countCodePoints(text);
219
- const rawTokens = characterCount / config.charsPerToken;
495
+ const isNonOpenAIModel2 = model.startsWith("claude-") || model.startsWith("gemini-");
220
496
  let estimatedTokens;
221
- switch (rounding) {
222
- case "floor":
223
- estimatedTokens = Math.floor(rawTokens);
224
- break;
225
- case "round":
226
- estimatedTokens = Math.round(rawTokens);
227
- break;
228
- case "ceil":
229
- default:
230
- estimatedTokens = Math.ceil(rawTokens);
497
+ let tokenizerModeUsed = "heuristic";
498
+ let encodingUsed;
499
+ const shouldTryExact = tokenizer === "openai_exact" || tokenizer === "auto";
500
+ if (shouldTryExact && !isNonOpenAIModel2) {
501
+ try {
502
+ estimatedTokens = encode(text, { model, allowSpecial: "none" }).length;
503
+ tokenizerModeUsed = "openai_exact";
504
+ encodingUsed = getOpenAIEncoding({ model });
505
+ } catch (error) {
506
+ if (tokenizer === "openai_exact") {
507
+ throw error;
508
+ }
509
+ }
510
+ } else if (tokenizer === "openai_exact" && isNonOpenAIModel2) {
511
+ throw new Error(
512
+ `Tokenizer mode "openai_exact" requested for non-OpenAI model: "${model}"`
513
+ );
514
+ }
515
+ if (estimatedTokens === void 0) {
516
+ const rawTokens = characterCount / config.charsPerToken;
517
+ switch (rounding) {
518
+ case "floor":
519
+ estimatedTokens = Math.floor(rawTokens);
520
+ break;
521
+ case "round":
522
+ estimatedTokens = Math.round(rawTokens);
523
+ break;
524
+ case "ceil":
525
+ default:
526
+ estimatedTokens = Math.ceil(rawTokens);
527
+ }
528
+ tokenizerModeUsed = "heuristic";
231
529
  }
232
530
  const estimatedInputCost = estimatedTokens * config.inputCostPerMillion / 1e6;
233
531
  return {
@@ -235,13 +533,44 @@ function estimate(input) {
235
533
  characterCount,
236
534
  estimatedTokens,
237
535
  estimatedInputCost,
238
- charsPerToken: config.charsPerToken
536
+ charsPerToken: config.charsPerToken,
537
+ tokenizerMode: tokenizerModeUsed,
538
+ encodingUsed
239
539
  };
240
540
  }
541
+
542
+ // src/token-counter.ts
543
+ function isNonOpenAIModel(model) {
544
+ return model.startsWith("claude-") || model.startsWith("gemini-");
545
+ }
546
+ function countTokens(input) {
547
+ const { text, model } = input;
548
+ if (isNonOpenAIModel(model)) {
549
+ return {
550
+ tokens: estimate({ text, model }).estimatedTokens,
551
+ exact: false
552
+ };
553
+ }
554
+ try {
555
+ return {
556
+ tokens: encode(text, { model, allowSpecial: "none" }).length,
557
+ exact: true,
558
+ encoding: getOpenAIEncoding({ model })
559
+ };
560
+ } catch {
561
+ return {
562
+ tokens: estimate({ text, model }).estimatedTokens,
563
+ exact: false
564
+ };
565
+ }
566
+ }
241
567
  // Annotate the CommonJS export names for ESM import in node:
242
568
  0 && (module.exports = {
243
569
  DEFAULT_MODELS,
244
570
  LAST_UPDATED,
571
+ countTokens,
572
+ decode,
573
+ encode,
245
574
  estimate,
246
575
  getAvailableModels,
247
576
  getModelConfig
package/dist/index.d.cts CHANGED
@@ -7,6 +7,7 @@ interface ModelConfig {
7
7
  /** Cost in USD per 1 million input tokens */
8
8
  inputCostPerMillion: number;
9
9
  }
10
+ type TokenizerMode = 'heuristic' | 'openai_exact' | 'auto';
10
11
  /**
11
12
  * Input parameters for the estimate function.
12
13
  */
@@ -17,6 +18,13 @@ interface EstimateInput {
17
18
  model: string;
18
19
  /** Rounding strategy for token count (default: 'ceil') */
19
20
  rounding?: 'ceil' | 'round' | 'floor';
21
+ /**
22
+ * Token counting strategy.
23
+ * - `heuristic` (default): use chars-per-token ratios
24
+ * - `openai_exact`: use OpenAI BPE tokenization (throws if non-OpenAI model)
25
+ * - `auto`: use OpenAI BPE for OpenAI models, otherwise heuristic
26
+ */
27
+ tokenizer?: TokenizerMode;
20
28
  }
21
29
  /**
22
30
  * Output from the estimate function.
@@ -32,6 +40,10 @@ interface EstimateOutput {
32
40
  estimatedInputCost: number;
33
41
  /** The chars-per-token ratio used */
34
42
  charsPerToken: number;
43
+ /** Which tokenizer strategy was used */
44
+ tokenizerMode?: TokenizerMode;
45
+ /** OpenAI encoding used when tokenizerMode is `openai_exact` */
46
+ encodingUsed?: string;
35
47
  }
36
48
 
37
49
  /**
@@ -57,16 +69,16 @@ declare function estimate(input: EstimateInput): EstimateOutput;
57
69
  * Default model configurations.
58
70
  *
59
71
  * AUTO-GENERATED FILE - DO NOT EDIT MANUALLY
60
- * Last updated: 2025-12-25
72
+ * Last updated: 2026-01-14
61
73
  *
62
74
  * Sources:
63
- * - OpenAI: https://openai.com/api/pricing/
75
+ * - OpenAI: https://platform.openai.com/docs/pricing
64
76
  * - Anthropic: https://www.anthropic.com/pricing
65
77
  * - Google: https://ai.google.dev/gemini-api/docs/pricing
66
78
  *
67
79
  * This file is automatically updated weekly by GitHub Actions.
68
80
  */
69
- declare const LAST_UPDATED = "2025-12-25";
81
+ declare const LAST_UPDATED = "2026-01-14";
70
82
  declare const DEFAULT_MODELS: Readonly<Record<string, Readonly<ModelConfig>>>;
71
83
  /**
72
84
  * Get configuration for a specific model.
@@ -81,4 +93,52 @@ declare function getModelConfig(model: string): ModelConfig;
81
93
  */
82
94
  declare function getAvailableModels(): string[];
83
95
 
84
- export { DEFAULT_MODELS, type EstimateInput, type EstimateOutput, LAST_UPDATED, type ModelConfig, estimate, getAvailableModels, getModelConfig };
96
+ type OpenAIEncoding = 'r50k_base' | 'p50k_base' | 'p50k_edit' | 'cl100k_base' | 'o200k_base' | 'o200k_harmony';
97
+ type SpecialTokenHandling = 'all' | 'none' | 'none_raise';
98
+ interface EncodeOptions {
99
+ /**
100
+ * Explicit OpenAI encoding override.
101
+ * When provided, this takes precedence over `model`.
102
+ */
103
+ encoding?: OpenAIEncoding;
104
+ /**
105
+ * OpenAI model ID used to select the appropriate encoding.
106
+ */
107
+ model?: string;
108
+ /**
109
+ * How special tokens are handled.
110
+ * - `none_raise` (default): throw if special tokens appear
111
+ * - `none`: treat special tokens as regular text
112
+ * - `all`: allow special tokens and encode them as special token IDs
113
+ */
114
+ allowSpecial?: SpecialTokenHandling;
115
+ }
116
+ /**
117
+ * Encode text into OpenAI token IDs using tiktoken-compatible BPE encoding.
118
+ *
119
+ * This is exact tokenization for OpenAI models (unlike heuristic estimators).
120
+ */
121
+ declare function encode(text: string, options?: EncodeOptions): number[];
122
+ /**
123
+ * Decode OpenAI token IDs into text using tiktoken-compatible BPE encoding.
124
+ */
125
+ declare function decode(tokens: Iterable<number>, options?: Pick<EncodeOptions, 'encoding' | 'model'>): string;
126
+
127
+ interface TokenCountInput {
128
+ text: string;
129
+ model: string;
130
+ }
131
+ interface TokenCountOutput {
132
+ tokens: number;
133
+ exact: boolean;
134
+ encoding?: OpenAIEncoding;
135
+ }
136
+ /**
137
+ * Count tokens for a given model.
138
+ *
139
+ * - OpenAI models: exact BPE tokenization
140
+ * - Other providers: heuristic estimate (chars-per-token)
141
+ */
142
+ declare function countTokens(input: TokenCountInput): TokenCountOutput;
143
+
144
+ export { DEFAULT_MODELS, type EncodeOptions, type EstimateInput, type EstimateOutput, LAST_UPDATED, type ModelConfig, type OpenAIEncoding, type SpecialTokenHandling, type TokenCountInput, type TokenCountOutput, type TokenizerMode, countTokens, decode, encode, estimate, getAvailableModels, getModelConfig };
package/dist/index.d.ts CHANGED
@@ -7,6 +7,7 @@ interface ModelConfig {
7
7
  /** Cost in USD per 1 million input tokens */
8
8
  inputCostPerMillion: number;
9
9
  }
10
+ type TokenizerMode = 'heuristic' | 'openai_exact' | 'auto';
10
11
  /**
11
12
  * Input parameters for the estimate function.
12
13
  */
@@ -17,6 +18,13 @@ interface EstimateInput {
17
18
  model: string;
18
19
  /** Rounding strategy for token count (default: 'ceil') */
19
20
  rounding?: 'ceil' | 'round' | 'floor';
21
+ /**
22
+ * Token counting strategy.
23
+ * - `heuristic` (default): use chars-per-token ratios
24
+ * - `openai_exact`: use OpenAI BPE tokenization (throws if non-OpenAI model)
25
+ * - `auto`: use OpenAI BPE for OpenAI models, otherwise heuristic
26
+ */
27
+ tokenizer?: TokenizerMode;
20
28
  }
21
29
  /**
22
30
  * Output from the estimate function.
@@ -32,6 +40,10 @@ interface EstimateOutput {
32
40
  estimatedInputCost: number;
33
41
  /** The chars-per-token ratio used */
34
42
  charsPerToken: number;
43
+ /** Which tokenizer strategy was used */
44
+ tokenizerMode?: TokenizerMode;
45
+ /** OpenAI encoding used when tokenizerMode is `openai_exact` */
46
+ encodingUsed?: string;
35
47
  }
36
48
 
37
49
  /**
@@ -57,16 +69,16 @@ declare function estimate(input: EstimateInput): EstimateOutput;
57
69
  * Default model configurations.
58
70
  *
59
71
  * AUTO-GENERATED FILE - DO NOT EDIT MANUALLY
60
- * Last updated: 2025-12-25
72
+ * Last updated: 2026-01-14
61
73
  *
62
74
  * Sources:
63
- * - OpenAI: https://openai.com/api/pricing/
75
+ * - OpenAI: https://platform.openai.com/docs/pricing
64
76
  * - Anthropic: https://www.anthropic.com/pricing
65
77
  * - Google: https://ai.google.dev/gemini-api/docs/pricing
66
78
  *
67
79
  * This file is automatically updated weekly by GitHub Actions.
68
80
  */
69
- declare const LAST_UPDATED = "2025-12-25";
81
+ declare const LAST_UPDATED = "2026-01-14";
70
82
  declare const DEFAULT_MODELS: Readonly<Record<string, Readonly<ModelConfig>>>;
71
83
  /**
72
84
  * Get configuration for a specific model.
@@ -81,4 +93,52 @@ declare function getModelConfig(model: string): ModelConfig;
81
93
  */
82
94
  declare function getAvailableModels(): string[];
83
95
 
84
- export { DEFAULT_MODELS, type EstimateInput, type EstimateOutput, LAST_UPDATED, type ModelConfig, estimate, getAvailableModels, getModelConfig };
96
+ type OpenAIEncoding = 'r50k_base' | 'p50k_base' | 'p50k_edit' | 'cl100k_base' | 'o200k_base' | 'o200k_harmony';
97
+ type SpecialTokenHandling = 'all' | 'none' | 'none_raise';
98
+ interface EncodeOptions {
99
+ /**
100
+ * Explicit OpenAI encoding override.
101
+ * When provided, this takes precedence over `model`.
102
+ */
103
+ encoding?: OpenAIEncoding;
104
+ /**
105
+ * OpenAI model ID used to select the appropriate encoding.
106
+ */
107
+ model?: string;
108
+ /**
109
+ * How special tokens are handled.
110
+ * - `none_raise` (default): throw if special tokens appear
111
+ * - `none`: treat special tokens as regular text
112
+ * - `all`: allow special tokens and encode them as special token IDs
113
+ */
114
+ allowSpecial?: SpecialTokenHandling;
115
+ }
116
+ /**
117
+ * Encode text into OpenAI token IDs using tiktoken-compatible BPE encoding.
118
+ *
119
+ * This is exact tokenization for OpenAI models (unlike heuristic estimators).
120
+ */
121
+ declare function encode(text: string, options?: EncodeOptions): number[];
122
+ /**
123
+ * Decode OpenAI token IDs into text using tiktoken-compatible BPE encoding.
124
+ */
125
+ declare function decode(tokens: Iterable<number>, options?: Pick<EncodeOptions, 'encoding' | 'model'>): string;
126
+
127
+ interface TokenCountInput {
128
+ text: string;
129
+ model: string;
130
+ }
131
+ interface TokenCountOutput {
132
+ tokens: number;
133
+ exact: boolean;
134
+ encoding?: OpenAIEncoding;
135
+ }
136
+ /**
137
+ * Count tokens for a given model.
138
+ *
139
+ * - OpenAI models: exact BPE tokenization
140
+ * - Other providers: heuristic estimate (chars-per-token)
141
+ */
142
+ declare function countTokens(input: TokenCountInput): TokenCountOutput;
143
+
144
+ export { DEFAULT_MODELS, type EncodeOptions, type EstimateInput, type EstimateOutput, LAST_UPDATED, type ModelConfig, type OpenAIEncoding, type SpecialTokenHandling, type TokenCountInput, type TokenCountOutput, type TokenizerMode, countTokens, decode, encode, estimate, getAvailableModels, getModelConfig };
package/dist/index.js CHANGED
@@ -1,42 +1,222 @@
1
1
  // src/models.ts
2
- var LAST_UPDATED = "2025-12-25";
2
+ var LAST_UPDATED = "2026-01-14";
3
3
  var models = {
4
4
  // ===================
5
5
  // OpenAI Models
6
6
  // ===================
7
7
  // OpenAI uses ~4 chars per token for English text
8
- "gpt-4.1": {
8
+ "babbage-002": {
9
+ charsPerToken: 4,
10
+ inputCostPerMillion: 0.4
11
+ },
12
+ "chatgpt-4o-latest": {
13
+ charsPerToken: 4,
14
+ inputCostPerMillion: 5
15
+ },
16
+ "chatgpt-image-latest": {
17
+ charsPerToken: 4,
18
+ inputCostPerMillion: 5
19
+ },
20
+ "codex-mini-latest": {
21
+ charsPerToken: 4,
22
+ inputCostPerMillion: 1.5
23
+ },
24
+ "computer-use-preview": {
25
+ charsPerToken: 4,
26
+ inputCostPerMillion: 3
27
+ },
28
+ "davinci-002": {
29
+ charsPerToken: 4,
30
+ inputCostPerMillion: 2
31
+ },
32
+ "gpt-3.5-0301": {
33
+ charsPerToken: 4,
34
+ inputCostPerMillion: 1.5
35
+ },
36
+ "gpt-3.5-turbo": {
37
+ charsPerToken: 4,
38
+ inputCostPerMillion: 0.5
39
+ },
40
+ "gpt-3.5-turbo-0125": {
41
+ charsPerToken: 4,
42
+ inputCostPerMillion: 0.5
43
+ },
44
+ "gpt-3.5-turbo-0613": {
45
+ charsPerToken: 4,
46
+ inputCostPerMillion: 1.5
47
+ },
48
+ "gpt-3.5-turbo-1106": {
49
+ charsPerToken: 4,
50
+ inputCostPerMillion: 1
51
+ },
52
+ "gpt-3.5-turbo-16k-0613": {
9
53
  charsPerToken: 4,
10
54
  inputCostPerMillion: 3
11
55
  },
56
+ "gpt-3.5-turbo-instruct": {
57
+ charsPerToken: 4,
58
+ inputCostPerMillion: 1.5
59
+ },
60
+ "gpt-4-0125-preview": {
61
+ charsPerToken: 4,
62
+ inputCostPerMillion: 10
63
+ },
64
+ "gpt-4-0314": {
65
+ charsPerToken: 4,
66
+ inputCostPerMillion: 30
67
+ },
68
+ "gpt-4-0613": {
69
+ charsPerToken: 4,
70
+ inputCostPerMillion: 30
71
+ },
72
+ "gpt-4-1106-preview": {
73
+ charsPerToken: 4,
74
+ inputCostPerMillion: 10
75
+ },
76
+ "gpt-4-1106-vision-preview": {
77
+ charsPerToken: 4,
78
+ inputCostPerMillion: 10
79
+ },
80
+ "gpt-4-32k": {
81
+ charsPerToken: 4,
82
+ inputCostPerMillion: 60
83
+ },
84
+ "gpt-4-turbo-2024-04-09": {
85
+ charsPerToken: 4,
86
+ inputCostPerMillion: 10
87
+ },
88
+ "gpt-4.1": {
89
+ charsPerToken: 4,
90
+ inputCostPerMillion: 2
91
+ },
12
92
  "gpt-4.1-mini": {
13
93
  charsPerToken: 4,
14
- inputCostPerMillion: 0.8
94
+ inputCostPerMillion: 0.4
15
95
  },
16
96
  "gpt-4.1-nano": {
17
97
  charsPerToken: 4,
18
- inputCostPerMillion: 0.2
98
+ inputCostPerMillion: 0.1
19
99
  },
20
100
  "gpt-4o": {
21
101
  charsPerToken: 4,
22
102
  inputCostPerMillion: 2.5
23
103
  },
104
+ "gpt-4o-2024-05-13": {
105
+ charsPerToken: 4,
106
+ inputCostPerMillion: 5
107
+ },
108
+ "gpt-4o-audio-preview": {
109
+ charsPerToken: 4,
110
+ inputCostPerMillion: 2.5
111
+ },
24
112
  "gpt-4o-mini": {
25
113
  charsPerToken: 4,
26
114
  inputCostPerMillion: 0.15
27
115
  },
116
+ "gpt-4o-mini-audio-preview": {
117
+ charsPerToken: 4,
118
+ inputCostPerMillion: 0.15
119
+ },
120
+ "gpt-4o-mini-realtime-preview": {
121
+ charsPerToken: 4,
122
+ inputCostPerMillion: 0.6
123
+ },
124
+ "gpt-4o-mini-search-preview": {
125
+ charsPerToken: 4,
126
+ inputCostPerMillion: 0.15
127
+ },
128
+ "gpt-4o-realtime-preview": {
129
+ charsPerToken: 4,
130
+ inputCostPerMillion: 5
131
+ },
132
+ "gpt-4o-search-preview": {
133
+ charsPerToken: 4,
134
+ inputCostPerMillion: 2.5
135
+ },
136
+ "gpt-5": {
137
+ charsPerToken: 4,
138
+ inputCostPerMillion: 1.25
139
+ },
28
140
  "gpt-5-mini": {
29
141
  charsPerToken: 4,
30
142
  inputCostPerMillion: 0.25
31
143
  },
144
+ "gpt-5-nano": {
145
+ charsPerToken: 4,
146
+ inputCostPerMillion: 0.05
147
+ },
148
+ "gpt-5-pro": {
149
+ charsPerToken: 4,
150
+ inputCostPerMillion: 15
151
+ },
152
+ "gpt-5-search-api": {
153
+ charsPerToken: 4,
154
+ inputCostPerMillion: 1.25
155
+ },
156
+ "gpt-5.1": {
157
+ charsPerToken: 4,
158
+ inputCostPerMillion: 1.25
159
+ },
160
+ "gpt-5.1-chat-latest": {
161
+ charsPerToken: 4,
162
+ inputCostPerMillion: 1.25
163
+ },
164
+ "gpt-5.1-codex": {
165
+ charsPerToken: 4,
166
+ inputCostPerMillion: 1.25
167
+ },
168
+ "gpt-5.1-codex-max": {
169
+ charsPerToken: 4,
170
+ inputCostPerMillion: 1.25
171
+ },
172
+ "gpt-5.1-codex-mini": {
173
+ charsPerToken: 4,
174
+ inputCostPerMillion: 0.25
175
+ },
176
+ "gpt-5-chat-latest": {
177
+ charsPerToken: 4,
178
+ inputCostPerMillion: 1.25
179
+ },
180
+ "gpt-5-codex": {
181
+ charsPerToken: 4,
182
+ inputCostPerMillion: 1.25
183
+ },
32
184
  "gpt-5.2": {
33
185
  charsPerToken: 4,
34
186
  inputCostPerMillion: 1.75
35
187
  },
188
+ "gpt-5.2-chat-latest": {
189
+ charsPerToken: 4,
190
+ inputCostPerMillion: 1.75
191
+ },
192
+ "gpt-5.2-codex": {
193
+ charsPerToken: 4,
194
+ inputCostPerMillion: 1.75
195
+ },
36
196
  "gpt-5.2-pro": {
37
197
  charsPerToken: 4,
38
198
  inputCostPerMillion: 21
39
199
  },
200
+ "gpt-audio": {
201
+ charsPerToken: 4,
202
+ inputCostPerMillion: 2.5
203
+ },
204
+ "gpt-audio-mini": {
205
+ charsPerToken: 4,
206
+ inputCostPerMillion: 0.6
207
+ },
208
+ "gpt-image-1": {
209
+ charsPerToken: 4,
210
+ inputCostPerMillion: 5
211
+ },
212
+ "gpt-image-1-mini": {
213
+ charsPerToken: 4,
214
+ inputCostPerMillion: 2
215
+ },
216
+ "gpt-image-1.5": {
217
+ charsPerToken: 4,
218
+ inputCostPerMillion: 5
219
+ },
40
220
  "gpt-realtime": {
41
221
  charsPerToken: 4,
42
222
  inputCostPerMillion: 4
@@ -49,6 +229,10 @@ var models = {
49
229
  charsPerToken: 4,
50
230
  inputCostPerMillion: 15
51
231
  },
232
+ "o1-mini": {
233
+ charsPerToken: 4,
234
+ inputCostPerMillion: 1.1
235
+ },
52
236
  "o1-pro": {
53
237
  charsPerToken: 4,
54
238
  inputCostPerMillion: 150
@@ -57,9 +241,25 @@ var models = {
57
241
  charsPerToken: 4,
58
242
  inputCostPerMillion: 2
59
243
  },
244
+ "o3-deep-research": {
245
+ charsPerToken: 4,
246
+ inputCostPerMillion: 10
247
+ },
248
+ "o3-mini": {
249
+ charsPerToken: 4,
250
+ inputCostPerMillion: 1.1
251
+ },
252
+ "o3-pro": {
253
+ charsPerToken: 4,
254
+ inputCostPerMillion: 20
255
+ },
60
256
  "o4-mini": {
61
257
  charsPerToken: 4,
62
- inputCostPerMillion: 4
258
+ inputCostPerMillion: 1.1
259
+ },
260
+ "o4-mini-deep-research": {
261
+ charsPerToken: 4,
262
+ inputCostPerMillion: 2
63
263
  },
64
264
  // ===================
65
265
  // Anthropic Models
@@ -174,6 +374,78 @@ function getAvailableModels() {
174
374
  return Object.keys(DEFAULT_MODELS);
175
375
  }
176
376
 
377
+ // src/openai-bpe.ts
378
+ import { createRequire } from "module";
379
+ import { ALL_SPECIAL_TOKENS } from "gpt-tokenizer/constants";
380
+ import { DEFAULT_ENCODING, modelToEncodingMap } from "gpt-tokenizer/mapping";
381
+ var requireBase = typeof __filename === "string" && __filename.length > 0 ? __filename : import.meta.url;
382
+ var NODE_REQUIRE = createRequire(requireBase);
383
+ var ENCODING_MODULES = {
384
+ r50k_base: "gpt-tokenizer/cjs/encoding/r50k_base",
385
+ p50k_base: "gpt-tokenizer/cjs/encoding/p50k_base",
386
+ p50k_edit: "gpt-tokenizer/cjs/encoding/p50k_edit",
387
+ cl100k_base: "gpt-tokenizer/cjs/encoding/cl100k_base",
388
+ o200k_base: "gpt-tokenizer/cjs/encoding/o200k_base",
389
+ o200k_harmony: "gpt-tokenizer/cjs/encoding/o200k_harmony"
390
+ };
391
+ var encodingApiCache = /* @__PURE__ */ new Map();
392
+ function getEncodingApi(encoding) {
393
+ const cached = encodingApiCache.get(encoding);
394
+ if (cached) return cached;
395
+ const modulePath = ENCODING_MODULES[encoding];
396
+ const mod = NODE_REQUIRE(modulePath);
397
+ const api = { encode: mod.encode, decode: mod.decode };
398
+ encodingApiCache.set(encoding, api);
399
+ return api;
400
+ }
401
+ function resolveEncoding(selector) {
402
+ if (selector?.encoding) {
403
+ return selector.encoding;
404
+ }
405
+ const model = selector?.model?.trim();
406
+ if (model) {
407
+ const mapped = modelToEncodingMap[model];
408
+ if (mapped) {
409
+ return mapped;
410
+ }
411
+ }
412
+ return DEFAULT_ENCODING;
413
+ }
414
+ function getOpenAIEncoding(selector) {
415
+ return resolveEncoding(selector);
416
+ }
417
+ function toGptTokenizerEncodeOptions(allowSpecial) {
418
+ const mode = allowSpecial ?? "none_raise";
419
+ switch (mode) {
420
+ case "all":
421
+ return {
422
+ allowedSpecial: ALL_SPECIAL_TOKENS,
423
+ disallowedSpecial: /* @__PURE__ */ new Set()
424
+ };
425
+ case "none":
426
+ return {
427
+ allowedSpecial: /* @__PURE__ */ new Set(),
428
+ disallowedSpecial: /* @__PURE__ */ new Set()
429
+ };
430
+ case "none_raise":
431
+ default:
432
+ return {
433
+ disallowedSpecial: ALL_SPECIAL_TOKENS
434
+ };
435
+ }
436
+ }
437
+ function encode(text, options) {
438
+ const encoding = resolveEncoding(options);
439
+ const api = getEncodingApi(encoding);
440
+ const encodeOptions = toGptTokenizerEncodeOptions(options?.allowSpecial);
441
+ return api.encode(text, encodeOptions);
442
+ }
443
+ function decode(tokens, options) {
444
+ const encoding = resolveEncoding(options);
445
+ const api = getEncodingApi(encoding);
446
+ return api.decode(tokens);
447
+ }
448
+
177
449
  // src/estimator.ts
178
450
  function countCodePoints(text) {
179
451
  let count = 0;
@@ -183,21 +455,43 @@ function countCodePoints(text) {
183
455
  return count;
184
456
  }
185
457
  function estimate(input) {
186
- const { text, model, rounding = "ceil" } = input;
458
+ const { text, model, rounding = "ceil", tokenizer = "heuristic" } = input;
187
459
  const config = getModelConfig(model);
188
460
  const characterCount = countCodePoints(text);
189
- const rawTokens = characterCount / config.charsPerToken;
461
+ const isNonOpenAIModel2 = model.startsWith("claude-") || model.startsWith("gemini-");
190
462
  let estimatedTokens;
191
- switch (rounding) {
192
- case "floor":
193
- estimatedTokens = Math.floor(rawTokens);
194
- break;
195
- case "round":
196
- estimatedTokens = Math.round(rawTokens);
197
- break;
198
- case "ceil":
199
- default:
200
- estimatedTokens = Math.ceil(rawTokens);
463
+ let tokenizerModeUsed = "heuristic";
464
+ let encodingUsed;
465
+ const shouldTryExact = tokenizer === "openai_exact" || tokenizer === "auto";
466
+ if (shouldTryExact && !isNonOpenAIModel2) {
467
+ try {
468
+ estimatedTokens = encode(text, { model, allowSpecial: "none" }).length;
469
+ tokenizerModeUsed = "openai_exact";
470
+ encodingUsed = getOpenAIEncoding({ model });
471
+ } catch (error) {
472
+ if (tokenizer === "openai_exact") {
473
+ throw error;
474
+ }
475
+ }
476
+ } else if (tokenizer === "openai_exact" && isNonOpenAIModel2) {
477
+ throw new Error(
478
+ `Tokenizer mode "openai_exact" requested for non-OpenAI model: "${model}"`
479
+ );
480
+ }
481
+ if (estimatedTokens === void 0) {
482
+ const rawTokens = characterCount / config.charsPerToken;
483
+ switch (rounding) {
484
+ case "floor":
485
+ estimatedTokens = Math.floor(rawTokens);
486
+ break;
487
+ case "round":
488
+ estimatedTokens = Math.round(rawTokens);
489
+ break;
490
+ case "ceil":
491
+ default:
492
+ estimatedTokens = Math.ceil(rawTokens);
493
+ }
494
+ tokenizerModeUsed = "heuristic";
201
495
  }
202
496
  const estimatedInputCost = estimatedTokens * config.inputCostPerMillion / 1e6;
203
497
  return {
@@ -205,12 +499,43 @@ function estimate(input) {
205
499
  characterCount,
206
500
  estimatedTokens,
207
501
  estimatedInputCost,
208
- charsPerToken: config.charsPerToken
502
+ charsPerToken: config.charsPerToken,
503
+ tokenizerMode: tokenizerModeUsed,
504
+ encodingUsed
209
505
  };
210
506
  }
507
+
508
+ // src/token-counter.ts
509
+ function isNonOpenAIModel(model) {
510
+ return model.startsWith("claude-") || model.startsWith("gemini-");
511
+ }
512
+ function countTokens(input) {
513
+ const { text, model } = input;
514
+ if (isNonOpenAIModel(model)) {
515
+ return {
516
+ tokens: estimate({ text, model }).estimatedTokens,
517
+ exact: false
518
+ };
519
+ }
520
+ try {
521
+ return {
522
+ tokens: encode(text, { model, allowSpecial: "none" }).length,
523
+ exact: true,
524
+ encoding: getOpenAIEncoding({ model })
525
+ };
526
+ } catch {
527
+ return {
528
+ tokens: estimate({ text, model }).estimatedTokens,
529
+ exact: false
530
+ };
531
+ }
532
+ }
211
533
  export {
212
534
  DEFAULT_MODELS,
213
535
  LAST_UPDATED,
536
+ countTokens,
537
+ decode,
538
+ encode,
214
539
  estimate,
215
540
  getAvailableModels,
216
541
  getModelConfig
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "ai-token-estimator",
3
- "version": "1.0.2",
3
+ "version": "1.1.0",
4
4
  "description": "Estimate token counts and costs for LLM API calls",
5
5
  "type": "module",
6
6
  "main": "./dist/index.cjs",
@@ -23,13 +23,17 @@
23
23
  "LICENSE",
24
24
  "README.md"
25
25
  ],
26
+ "dependencies": {
27
+ "gpt-tokenizer": "^3.4.0"
28
+ },
26
29
  "scripts": {
27
30
  "build": "tsup src/index.ts --format cjs,esm --dts",
28
31
  "test": "vitest run",
29
32
  "test:watch": "vitest",
30
33
  "lint": "eslint src tests",
31
34
  "prepublishOnly": "npm run lint && npm run test && npm run build",
32
- "update-pricing": "tsx scripts/update-pricing.ts"
35
+ "update-pricing": "tsx scripts/update-pricing.ts",
36
+ "benchmark:tokenizer": "tsx benchmark/tokenizer.ts"
33
37
  },
34
38
  "keywords": [
35
39
  "llm",