@goliapkg/tiktoken-wasm 2.1.1 → 3.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/package.json +1 -1
  2. package/tiktoken_wasm.d.ts +13 -3
  3. package/tiktoken_wasm.js +13 -3
  4. package/tiktoken_wasm_bg.wasm +0 -0
package/package.json CHANGED
@@ -2,7 +2,7 @@
2
2
  "name": "@goliapkg/tiktoken-wasm",
3
3
  "type": "module",
4
4
  "description": "WASM bindings for the tiktoken BPE tokenizer",
5
- "version": "2.1.1",
5
+ "version": "3.1.1",
6
6
  "license": "MIT",
7
7
  "repository": {
8
8
  "type": "git",
package/tiktoken_wasm.d.ts CHANGED
@@ -44,8 +44,9 @@ export class Encoding {
44
44
  }
45
45
 
46
46
  /**
47
- * Get an encoding for an OpenAI model name (e.g. `"gpt-4o"`, `"o3-mini"`).
47
+ * Get an encoding for a model name (e.g. `"gpt-4o"`, `"o3-mini"`, `"llama-4"`, `"deepseek-r1"`).
48
48
  *
49
+ * Supports models from OpenAI, Meta, DeepSeek, Qwen, and Mistral.
49
50
  * Automatically resolves the model name to the correct encoding.
50
51
  * Throws `Error` for unknown model names.
51
52
  */
@@ -54,7 +55,7 @@ export function encodingForModel(model: string): Encoding;
54
55
  /**
55
56
  * Estimate cost in USD for a given model, input token count, and output token count.
56
57
  *
57
- * Supports OpenAI, Anthropic Claude, and Google Gemini models.
58
+ * Supports OpenAI, Anthropic Claude, Google Gemini, Meta Llama, DeepSeek, Qwen, and Mistral models.
58
59
  * Throws `Error` for unknown model ids.
59
60
  */
60
61
  export function estimateCost(model_id: string, input_tokens: number, output_tokens: number): number;
@@ -62,7 +63,16 @@ export function estimateCost(model_id: string, input_tokens: number, output_toke
62
63
  /**
63
64
  * Get an encoding by name.
64
65
  *
65
- * Supported: `"cl100k_base"`, `"o200k_base"`, `"p50k_base"`, `"p50k_edit"`, `"r50k_base"`.
66
+ * Supported encodings:
67
+ * - `"cl100k_base"` — GPT-4, GPT-3.5-turbo
68
+ * - `"o200k_base"` — GPT-4o, GPT-4.1, o1, o3
69
+ * - `"p50k_base"` — text-davinci-002/003
70
+ * - `"p50k_edit"` — text-davinci-edit
71
+ * - `"r50k_base"` — GPT-3 (davinci, curie, etc.)
72
+ * - `"llama3"` — Meta Llama 3/4
73
+ * - `"deepseek_v3"` — DeepSeek V3/R1
74
+ * - `"qwen2"` — Qwen 2/2.5/3
75
+ * - `"mistral_v3"` — Mistral/Codestral/Pixtral
66
76
  *
67
77
  * Throws `Error` for unknown encoding names.
68
78
  */
package/tiktoken_wasm.js CHANGED
@@ -110,8 +110,9 @@ export class Encoding {
110
110
  if (Symbol.dispose) Encoding.prototype[Symbol.dispose] = Encoding.prototype.free;
111
111
 
112
112
  /**
113
- * Get an encoding for an OpenAI model name (e.g. `"gpt-4o"`, `"o3-mini"`).
113
+ * Get an encoding for a model name (e.g. `"gpt-4o"`, `"o3-mini"`, `"llama-4"`, `"deepseek-r1"`).
114
114
  *
115
+ * Supports models from OpenAI, Meta, DeepSeek, Qwen, and Mistral.
115
116
  * Automatically resolves the model name to the correct encoding.
116
117
  * Throws `Error` for unknown model names.
117
118
  * @param {string} model
@@ -130,7 +131,7 @@ export function encodingForModel(model) {
130
131
  /**
131
132
  * Estimate cost in USD for a given model, input token count, and output token count.
132
133
  *
133
- * Supports OpenAI, Anthropic Claude, and Google Gemini models.
134
+ * Supports OpenAI, Anthropic Claude, Google Gemini, Meta Llama, DeepSeek, Qwen, and Mistral models.
134
135
  * Throws `Error` for unknown model ids.
135
136
  * @param {string} model_id
136
137
  * @param {number} input_tokens
@@ -150,7 +151,16 @@ export function estimateCost(model_id, input_tokens, output_tokens) {
150
151
  /**
151
152
  * Get an encoding by name.
152
153
  *
153
- * Supported: `"cl100k_base"`, `"o200k_base"`, `"p50k_base"`, `"p50k_edit"`, `"r50k_base"`.
154
+ * Supported encodings:
155
+ * - `"cl100k_base"` — GPT-4, GPT-3.5-turbo
156
+ * - `"o200k_base"` — GPT-4o, GPT-4.1, o1, o3
157
+ * - `"p50k_base"` — text-davinci-002/003
158
+ * - `"p50k_edit"` — text-davinci-edit
159
+ * - `"r50k_base"` — GPT-3 (davinci, curie, etc.)
160
+ * - `"llama3"` — Meta Llama 3/4
161
+ * - `"deepseek_v3"` — DeepSeek V3/R1
162
+ * - `"qwen2"` — Qwen 2/2.5/3
163
+ * - `"mistral_v3"` — Mistral/Codestral/Pixtral
154
164
  *
155
165
  * Throws `Error` for unknown encoding names.
156
166
  * @param {string} name
package/tiktoken_wasm_bg.wasm CHANGED
Binary file