@goliapkg/tiktoken-wasm 2.1.1 → 3.1.0

package/README.md

@@ -1,14 +1,14 @@
-# @goliapkg/tokenrs-wasm
+# @goliapkg/tiktoken-wasm
 
-[![npm](https://img.shields.io/npm/v/@goliapkg/tokenrs-wasm?style=flat-square&logo=npm)](https://www.npmjs.com/package/@goliapkg/tokenrs-wasm)
-[![License](https://img.shields.io/npm/l/@goliapkg/tokenrs-wasm?style=flat-square)](../LICENSE)
+[![npm](https://img.shields.io/npm/v/@goliapkg/tiktoken-wasm?style=flat-square&logo=npm)](https://www.npmjs.com/package/@goliapkg/tiktoken-wasm)
+[![License](https://img.shields.io/npm/l/@goliapkg/tiktoken-wasm?style=flat-square)](../LICENSE)
 
-WebAssembly bindings for the [tiktoken](https://crates.io/crates/tiktoken) BPE tokenizer — run OpenAI-compatible tokenization directly in the browser or Node.js with near-native performance.
+WebAssembly bindings for the [tiktoken](https://crates.io/crates/tiktoken) BPE tokenizer — run multi-provider tokenization directly in the browser or Node.js with near-native performance.
 
 ## Install
 
 ```bash
-npm install @goliapkg/tokenrs-wasm
+npm install @goliapkg/tiktoken-wasm
 ```
 
 ## Build from source
@@ -35,7 +35,7 @@ import init, {
   estimateCost,
   getModelInfo,
   type Encoding,
-} from '@goliapkg/tokenrs-wasm'
+} from '@goliapkg/tiktoken-wasm'
 
 // initialize WASM module (required once, before any other calls)
 await init()
@@ -46,8 +46,10 @@ const tokens: Uint32Array = enc.encode('hello world')
 const text: string = enc.decode(tokens)   // "hello world"
 const count: number = enc.count('hello world')  // 2
 
-// by model name
+// by model name — supports OpenAI, Meta, DeepSeek, Qwen, Mistral
 const enc2 = encodingForModel('gpt-4o')
+const enc3 = encodingForModel('llama-4-scout')
+const enc4 = encodingForModel('deepseek-r1')
 
 // cost estimation (USD)
 const cost: number = estimateCost('gpt-4o', 1000, 500)
@@ -59,6 +61,8 @@ const info = getModelInfo('claude-opus-4')
 // free WASM memory when done
 enc.free()
 enc2.free()
+enc3.free()
+enc4.free()
 ```
 
 ### Bundler Configuration
@@ -99,11 +103,20 @@ module.exports = {
 
 ### `getEncoding(name: string): Encoding`
 
-Get a tokenizer by encoding name. Supported: `cl100k_base`, `o200k_base`, `p50k_base`, `p50k_edit`, `r50k_base`.
+Get a tokenizer by encoding name. Supported:
+- `cl100k_base` — GPT-4, GPT-3.5-turbo
+- `o200k_base` — GPT-4o, GPT-4.1, o1, o3
+- `p50k_base` — text-davinci-002/003
+- `p50k_edit` — text-davinci-edit
+- `r50k_base` — GPT-3 (davinci, curie, etc.)
+- `llama3` — Meta Llama 3/4
+- `deepseek_v3` — DeepSeek V3/R1
+- `qwen2` — Qwen 2/2.5/3
+- `mistral_v3` — Mistral/Codestral/Pixtral
 
 ### `encodingForModel(model: string): Encoding`
 
-Get a tokenizer by OpenAI model name (e.g. `gpt-4o`, `o3-mini`, `gpt-3.5-turbo`).
+Get a tokenizer by model name (e.g. `gpt-4o`, `llama-4-scout`, `deepseek-r1`, `qwen3-235b`).
 
 ### `Encoding`
 
@@ -118,7 +131,7 @@ Get a tokenizer by OpenAI model name (e.g. `gpt-4o`, `o3-mini`, `gpt-3.5-turbo`)
 
 ### `estimateCost(modelId, inputTokens, outputTokens): number`
 
-Estimate API cost in USD. Supports OpenAI, Anthropic Claude, and Google Gemini models.
+Estimate API cost in USD. Supports 57 models across 7 providers.
 
 ### `getModelInfo(modelId): object`
 
@@ -128,9 +141,13 @@ Get model metadata: pricing, context window, max output tokens.
 
 | Provider | Models |
 |----------|--------|
-| OpenAI | gpt-4o, gpt-4o-mini, o1, o3, o4-mini, gpt-4-turbo, gpt-4, gpt-3.5-turbo, embeddings |
-| Anthropic | claude-opus-4, claude-sonnet-4, claude-3.5-haiku, claude-3.5-sonnet, claude-3-opus, claude-3-haiku |
+| OpenAI | gpt-4.1, gpt-4.1-mini, gpt-4.1-nano, gpt-4o, gpt-4o-mini, o3, o3-pro, o3-mini, o4-mini, o1, gpt-4-turbo, gpt-4, gpt-3.5-turbo, embeddings |
+| Anthropic | claude-opus-4, claude-sonnet-4, claude-4.5-sonnet, claude-4.5-haiku, claude-4.6-sonnet, claude-4.6-opus, claude-4.6-haiku, claude-3.5-haiku, claude-3.5-sonnet, claude-3-opus, claude-3-haiku |
 | Google | gemini-2.5-pro, gemini-2.5-flash, gemini-2.0-flash, gemini-1.5-pro, gemini-1.5-flash |
+| Meta | llama-4-scout, llama-4-maverick, llama-3.3-70b, llama-3.1-405b, llama-3.1-70b, llama-3.1-8b |
+| DeepSeek | deepseek-r1, deepseek-v3, deepseek-chat |
+| Qwen | qwen3-235b, qwen3-32b, qwen3-30b-a3b, qwen3-14b, qwen-2.5-72b, qwen-2.5-coder-32b, qwen-turbo |
+| Mistral | mistral-large, mistral-medium, mistral-small, codestral, pixtral-large |
 
 ## Demo
 

package/package.json

@@ -2,7 +2,7 @@
   "name": "@goliapkg/tiktoken-wasm",
   "type": "module",
   "description": "WASM bindings for the tiktoken BPE tokenizer",
-  "version": "2.1.1",
+  "version": "3.1.0",
   "license": "MIT",
   "repository": {
     "type": "git",

package/tiktoken_wasm.d.ts

@@ -44,8 +44,9 @@ export class Encoding {
 }
 
 /**
- * Get an encoding for an OpenAI model name (e.g. `"gpt-4o"`, `"o3-mini"`).
+ * Get an encoding for a model name (e.g. `"gpt-4o"`, `"o3-mini"`, `"llama-4"`, `"deepseek-r1"`).
  *
+ * Supports models from OpenAI, Meta, DeepSeek, Qwen, and Mistral.
  * Automatically resolves the model name to the correct encoding.
  * Throws `Error` for unknown model names.
  */
@@ -54,7 +55,7 @@ export function encodingForModel(model: string): Encoding;
 /**
  * Estimate cost in USD for a given model, input token count, and output token count.
  *
- * Supports OpenAI, Anthropic Claude, and Google Gemini models.
+ * Supports OpenAI, Anthropic Claude, Google Gemini, Meta Llama, DeepSeek, Qwen, and Mistral models.
  * Throws `Error` for unknown model ids.
  */
 export function estimateCost(model_id: string, input_tokens: number, output_tokens: number): number;
@@ -62,7 +63,16 @@ export function estimateCost(model_id: string, input_tokens: number, output_toke
 /**
  * Get an encoding by name.
  *
- * Supported: `"cl100k_base"`, `"o200k_base"`, `"p50k_base"`, `"p50k_edit"`, `"r50k_base"`.
+ * Supported encodings:
+ * - `"cl100k_base"` — GPT-4, GPT-3.5-turbo
+ * - `"o200k_base"` — GPT-4o, GPT-4.1, o1, o3
+ * - `"p50k_base"` — text-davinci-002/003
+ * - `"p50k_edit"` — text-davinci-edit
+ * - `"r50k_base"` — GPT-3 (davinci, curie, etc.)
+ * - `"llama3"` — Meta Llama 3/4
+ * - `"deepseek_v3"` — DeepSeek V3/R1
+ * - `"qwen2"` — Qwen 2/2.5/3
+ * - `"mistral_v3"` — Mistral/Codestral/Pixtral
  *
  * Throws `Error` for unknown encoding names.
  */

package/tiktoken_wasm.js

@@ -110,8 +110,9 @@ export class Encoding {
 if (Symbol.dispose) Encoding.prototype[Symbol.dispose] = Encoding.prototype.free;
 
 /**
- * Get an encoding for an OpenAI model name (e.g. `"gpt-4o"`, `"o3-mini"`).
+ * Get an encoding for a model name (e.g. `"gpt-4o"`, `"o3-mini"`, `"llama-4"`, `"deepseek-r1"`).
  *
+ * Supports models from OpenAI, Meta, DeepSeek, Qwen, and Mistral.
  * Automatically resolves the model name to the correct encoding.
  * Throws `Error` for unknown model names.
  * @param {string} model
@@ -130,7 +131,7 @@ export function encodingForModel(model) {
 /**
  * Estimate cost in USD for a given model, input token count, and output token count.
  *
- * Supports OpenAI, Anthropic Claude, and Google Gemini models.
+ * Supports OpenAI, Anthropic Claude, Google Gemini, Meta Llama, DeepSeek, Qwen, and Mistral models.
  * Throws `Error` for unknown model ids.
  * @param {string} model_id
  * @param {number} input_tokens
@@ -150,7 +151,16 @@ export function estimateCost(model_id, input_tokens, output_tokens) {
 /**
  * Get an encoding by name.
  *
- * Supported: `"cl100k_base"`, `"o200k_base"`, `"p50k_base"`, `"p50k_edit"`, `"r50k_base"`.
+ * Supported encodings:
+ * - `"cl100k_base"` — GPT-4, GPT-3.5-turbo
+ * - `"o200k_base"` — GPT-4o, GPT-4.1, o1, o3
+ * - `"p50k_base"` — text-davinci-002/003
+ * - `"p50k_edit"` — text-davinci-edit
+ * - `"r50k_base"` — GPT-3 (davinci, curie, etc.)
+ * - `"llama3"` — Meta Llama 3/4
+ * - `"deepseek_v3"` — DeepSeek V3/R1
+ * - `"qwen2"` — Qwen 2/2.5/3
+ * - `"mistral_v3"` — Mistral/Codestral/Pixtral
  *
  * Throws `Error` for unknown encoding names.
  * @param {string} name