@goliapkg/tiktoken-wasm 3.2.0 → 3.2.2

package/README.md

@@ -137,7 +137,7 @@ Get a tokenizer by encoding name. Supported:
 
 ### `encodingForModel(model: string): Encoding`
 
-Get a tokenizer by model name (e.g. `gpt-4o`, `llama-4-scout`, `deepseek-r1`, `qwen3-235b`).
+Get a tokenizer by model name (e.g. `gpt-4o`, `llama-4-scout`, `deepseek-r1`, `qwen3-max`).
 
 ### `modelToEncoding(model: string): string | null`
 
@@ -189,17 +189,27 @@ Filter models by provider: `"OpenAI"`, `"Anthropic"`, `"Google"`, `"Meta"`, `"De
 
 | Provider | Models |
 |----------|--------|
-| OpenAI | gpt-4.1, gpt-4.1-mini, gpt-4.1-nano, gpt-4o, gpt-4o-mini, o3, o3-pro, o3-mini, o4-mini, o1, gpt-4-turbo, gpt-4, gpt-3.5-turbo, embeddings |
-| Anthropic | claude-opus-4, claude-sonnet-4, claude-4.5-sonnet, claude-4.5-haiku, claude-4.6-sonnet, claude-4.6-opus, claude-4.6-haiku, claude-3.5-haiku, claude-3.5-sonnet, claude-3-opus, claude-3-haiku |
-| Google | gemini-2.5-pro, gemini-2.5-flash, gemini-2.0-flash, gemini-1.5-pro, gemini-1.5-flash |
-| Meta | llama-4-scout, llama-4-maverick, llama-3.3-70b, llama-3.1-405b, llama-3.1-70b, llama-3.1-8b |
-| DeepSeek | deepseek-r1, deepseek-v3, deepseek-chat |
-| Qwen | qwen3-235b, qwen3-32b, qwen3-30b-a3b, qwen3-14b, qwen-2.5-72b, qwen-2.5-coder-32b, qwen-turbo |
-| Mistral | mistral-large, mistral-medium, mistral-small, codestral, pixtral-large |
+| OpenAI | gpt-4.1, gpt-4.1-mini, gpt-4.1-nano, gpt-4o, gpt-4o-mini, o1, o1-mini, o1-pro, o3, o3-pro, o3-mini, o4-mini, gpt-4-turbo, gpt-4, gpt-3.5-turbo, text-embedding-3-small, text-embedding-3-large, text-embedding-ada-002 |
+| Anthropic | claude-opus-4.6, claude-sonnet-4.6, claude-haiku-4.5, claude-opus-4.5, claude-sonnet-4.5, claude-opus-4, claude-sonnet-4, claude-3.5-haiku, claude-3.5-sonnet, claude-3-opus, claude-3-haiku |
+| Google | gemini-2.5-pro, gemini-2.5-flash, gemini-2.0-flash, gemini-1.5-pro, gemini-1.5-flash, text-embedding-004 |
+| Meta | llama-4-scout, llama-4-maverick, llama-3.1-405b, llama-3.1-70b, llama-3.1-8b, llama-3.3-70b |
+| DeepSeek | deepseek-v3, deepseek-r1 |
+| Alibaba | qwen3-max, qwen3-plus, qwen3-coder, qwen3-8b, qwen2.5-72b, qwen2.5-32b, qwen2.5-7b |
+| Mistral | mistral-large, mistral-medium, mistral-small, mistral-nemo, codestral, pixtral-large, mixtral-8x7b |
+
+## Notes
+
+### Initialization
+
+Call `await init()` once before any other API calls. This loads and compiles the WASM module. Subsequent calls are a no-op.
+
+### Memory Management
+
+`Encoding` instances hold references to globally cached data and are lightweight. Calling `.free()` releases the JS wrapper — the underlying encoding data remains cached for reuse. In short-lived scripts you can skip `.free()`; in long-running apps, call it when you're done with the instance.
 
 ## Demo
 
-See [`examples/react-app`](../examples/react-app/) for a complete Vite + React demo application.
+See [`examples/react-app`](../tiktoken/examples/react-app/) for a complete Vite + React demo application.
 
 ## License
 

package/package.json

@@ -2,7 +2,7 @@
   "name": "@goliapkg/tiktoken-wasm",
   "type": "module",
   "description": "WASM bindings for the tiktoken BPE tokenizer",
-  "version": "3.2.0",
+  "version": "3.2.2",
   "license": "MIT",
   "repository": {
     "type": "git",
@@ -11,11 +11,13 @@
   "files": [
     "tiktoken_wasm_bg.wasm",
     "tiktoken_wasm.js",
+    "tiktoken_wasm_bg.js",
     "tiktoken_wasm.d.ts"
   ],
   "main": "tiktoken_wasm.js",
   "types": "tiktoken_wasm.d.ts",
   "sideEffects": [
+    "./tiktoken_wasm.js",
     "./snippets/*"
   ]
 }

package/tiktoken_wasm.d.ts

@@ -131,7 +131,7 @@ export function getModelInfo(model_id: string): ModelInfo;
  *
  * Returns an array of strings: `["cl100k_base", "o200k_base", ...]`
  */
-export function listEncodings(): any[];
+export function listEncodings(): string[];
 
 /**
  * Map a model name to its encoding name without loading the encoding.
@@ -147,63 +147,3 @@ export function modelToEncoding(model: string): string | undefined;
  * Returns an empty array for unknown providers.
  */
 export function modelsByProvider(provider: string): ModelInfo[];
-
-export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module;
-
-export interface InitOutput {
-    readonly memory: WebAssembly.Memory;
-    readonly __wbg_encoding_free: (a: number, b: number) => void;
-    readonly __wbg_modelinfo_free: (a: number, b: number) => void;
-    readonly allModels: () => [number, number];
-    readonly encodingForModel: (a: number, b: number) => [number, number, number];
-    readonly encoding_count: (a: number, b: number, c: number) => number;
-    readonly encoding_countWithSpecialTokens: (a: number, b: number, c: number) => number;
-    readonly encoding_decode: (a: number, b: number, c: number) => [number, number];
-    readonly encoding_encode: (a: number, b: number, c: number) => [number, number];
-    readonly encoding_encodeWithSpecialTokens: (a: number, b: number, c: number) => [number, number];
-    readonly encoding_name: (a: number) => [number, number];
-    readonly encoding_numSpecialTokens: (a: number) => number;
-    readonly encoding_vocabSize: (a: number) => number;
-    readonly estimateCost: (a: number, b: number, c: number, d: number) => [number, number, number];
-    readonly getEncoding: (a: number, b: number) => [number, number, number];
-    readonly getModelInfo: (a: number, b: number) => [number, number, number];
-    readonly listEncodings: () => [number, number];
-    readonly modelToEncoding: (a: number, b: number) => [number, number];
-    readonly modelinfo_cachedInputPer1m: (a: number) => [number, number];
-    readonly modelinfo_contextWindow: (a: number) => number;
-    readonly modelinfo_id: (a: number) => [number, number];
-    readonly modelinfo_inputPer1m: (a: number) => number;
-    readonly modelinfo_maxOutput: (a: number) => number;
-    readonly modelinfo_outputPer1m: (a: number) => number;
-    readonly modelinfo_provider: (a: number) => [number, number];
-    readonly modelsByProvider: (a: number, b: number) => [number, number];
-    readonly __wbindgen_externrefs: WebAssembly.Table;
-    readonly __externref_drop_slice: (a: number, b: number) => void;
-    readonly __wbindgen_free: (a: number, b: number, c: number) => void;
-    readonly __wbindgen_malloc: (a: number, b: number) => number;
-    readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
-    readonly __externref_table_dealloc: (a: number) => void;
-    readonly __wbindgen_start: () => void;
-}
-
-export type SyncInitInput = BufferSource | WebAssembly.Module;
-
-/**
- * Instantiates the given `module`, which can either be bytes or
- * a precompiled `WebAssembly.Module`.
- *
- * @param {{ module: SyncInitInput }} module - Passing `SyncInitInput` directly is deprecated.
- *
- * @returns {InitOutput}
- */
-export function initSync(module: { module: SyncInitInput } | SyncInitInput): InitOutput;
-
-/**
- * If `module_or_path` is {RequestInfo} or {URL}, makes a request and
- * for everything else, calls `WebAssembly.instantiate` directly.
- *
- * @param {{ module_or_path: InitInput | Promise<InitInput> }} module_or_path - Passing `InitInput` directly is deprecated.
- *
- * @returns {Promise<InitOutput>}
- */
-export default function __wbg_init (module_or_path?: { module_or_path: InitInput | Promise<InitInput> } | InitInput | Promise<InitInput>): Promise<InitOutput>;