@goliapkg/tiktoken-wasm 2.1.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 GOLIA株式会社
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,141 @@
+# @goliapkg/tokenrs-wasm
+
+[![npm](https://img.shields.io/npm/v/@goliapkg/tokenrs-wasm?style=flat-square&logo=npm)](https://www.npmjs.com/package/@goliapkg/tokenrs-wasm)
+[![License](https://img.shields.io/npm/l/@goliapkg/tokenrs-wasm?style=flat-square)](../LICENSE)
+
+WebAssembly bindings for the [tiktoken](https://crates.io/crates/tiktoken) BPE tokenizer — run OpenAI-compatible tokenization directly in the browser or Node.js with near-native performance.
+
+## Install
+
+```bash
+npm install @goliapkg/tokenrs-wasm
+```
+
+## Build from source
+
+```bash
+# requires wasm-pack: cargo install wasm-pack
+cd tiktoken-wasm
+wasm-pack build --target web --release
+```
+
+Output is in `pkg/` — a complete npm-ready package containing:
+- `tiktoken_wasm.js` — ES module with WASM loader
+- `tiktoken_wasm_bg.wasm` — compiled WASM binary (~3 MB)
+- `tiktoken_wasm.d.ts` — TypeScript type definitions
+
+## Usage
+
+### ES Module (Browser / Vite / webpack)
+
+```typescript
+import init, {
+  getEncoding,
+  encodingForModel,
+  estimateCost,
+  getModelInfo,
+  type Encoding,
+} from '@goliapkg/tokenrs-wasm'
+
+// initialize WASM module (required once, before any other calls)
+await init()
+
+// encode / decode
+const enc: Encoding = getEncoding('cl100k_base')
+const tokens: Uint32Array = enc.encode('hello world')
+const text: string = enc.decode(tokens)   // "hello world"
+const count: number = enc.count('hello world')  // 2
+
+// by model name
+const enc2 = encodingForModel('gpt-4o')
+
+// cost estimation (USD)
+const cost: number = estimateCost('gpt-4o', 1000, 500)
+
+// model metadata
+const info = getModelInfo('claude-opus-4')
+// { id, provider, input_per_1m, output_per_1m, cached_input_per_1m, context_window, max_output }
+
+// free WASM memory when done
+enc.free()
+enc2.free()
+```
+
+### Bundler Configuration
+
+**Vite** — add plugins to `vite.config.ts`:
+
+```typescript
+import wasm from 'vite-plugin-wasm'
+import topLevelAwait from 'vite-plugin-top-level-await'
+
+export default defineConfig({
+  plugins: [wasm(), topLevelAwait()],
+})
+```
+
+**webpack 5** — enable WASM experiments in `webpack.config.js`:
+
+```javascript
+module.exports = {
+  experiments: {
+    asyncWebAssembly: true,
+  },
+}
+```
+
+**Next.js** — add to `next.config.js`:
+
+```javascript
+module.exports = {
+  webpack: (config) => {
+    config.experiments = { ...config.experiments, asyncWebAssembly: true }
+    return config
+  },
+}
+```
+
+## API Reference
+
+### `getEncoding(name: string): Encoding`
+
+Get a tokenizer by encoding name. Supported: `cl100k_base`, `o200k_base`, `p50k_base`, `p50k_edit`, `r50k_base`.
+
+### `encodingForModel(model: string): Encoding`
+
+Get a tokenizer by OpenAI model name (e.g. `gpt-4o`, `o3-mini`, `gpt-3.5-turbo`).
+
+### `Encoding`
+
+| Method | Returns | Description |
+|--------|---------|-------------|
+| `encode(text)` | `Uint32Array` | Encode text to token ids |
+| `encodeWithSpecialTokens(text)` | `Uint32Array` | Encode with special token recognition |
+| `decode(tokens)` | `string` | Decode token ids to text |
+| `count(text)` | `number` | Count tokens (faster than `encode().length`) |
+| `name` | `string` | Encoding name (getter) |
+| `free()` | `void` | Release WASM memory |
+
+### `estimateCost(modelId, inputTokens, outputTokens): number`
+
+Estimate API cost in USD. Supports OpenAI, Anthropic Claude, and Google Gemini models.
+
+### `getModelInfo(modelId): object`
+
+Get model metadata: pricing, context window, max output tokens.
+
+## Supported Models (pricing)
+
+| Provider | Models |
+|----------|--------|
+| OpenAI | gpt-4o, gpt-4o-mini, o1, o3, o4-mini, gpt-4-turbo, gpt-4, gpt-3.5-turbo, embeddings |
+| Anthropic | claude-opus-4, claude-sonnet-4, claude-3.5-haiku, claude-3.5-sonnet, claude-3-opus, claude-3-haiku |
+| Google | gemini-2.5-pro, gemini-2.5-flash, gemini-2.0-flash, gemini-1.5-pro, gemini-1.5-flash |
+
+## Demo
+
+See [`examples/react-app`](../examples/react-app/) for a complete Vite + React demo application.
+
+## License
+
+[MIT](../LICENSE)

package/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "@goliapkg/tiktoken-wasm",
+  "type": "module",
+  "description": "WASM bindings for the tiktoken BPE tokenizer",
+  "version": "2.1.1",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/goliajp/airs"
+  },
+  "files": [
+    "tiktoken_wasm_bg.wasm",
+    "tiktoken_wasm.js",
+    "tiktoken_wasm.d.ts"
+  ],
+  "main": "tiktoken_wasm.js",
+  "types": "tiktoken_wasm.d.ts",
+  "sideEffects": [
+    "./snippets/*"
+  ]
+}

package/tiktoken_wasm.d.ts

@@ -0,0 +1,123 @@
+/* tslint:disable */
+/* eslint-disable */
+
+/**
+ * WASM wrapper around a tiktoken encoding instance.
+ *
+ * Created via [`get_encoding`] or [`encoding_for_model`].
+ * Call `.free()` when done to release WASM memory.
+ */
+export class Encoding {
+    private constructor();
+    free(): void;
+    [Symbol.dispose](): void;
+    /**
+     * Count tokens without building the full token id array.
+     *
+     * Faster than `encode(text).length` for cases where you only need the count.
+     */
+    count(text: string): number;
+    /**
+     * Decode token ids back to a UTF-8 string.
+     *
+     * Uses lossy UTF-8 conversion — invalid byte sequences are replaced with U+FFFD.
+     */
+    decode(tokens: Uint32Array): string;
+    /**
+     * Encode text into token ids (returns `Uint32Array` in JS).
+     *
+     * Special tokens like `<|endoftext|>` are treated as ordinary text.
+     * Use `encodeWithSpecialTokens()` to recognize them.
+     */
+    encode(text: string): Uint32Array;
+    /**
+     * Encode text into token ids, recognizing special tokens.
+     *
+     * Special tokens (e.g. `<|endoftext|>`) are encoded as their designated ids
+     * instead of being split into sub-word pieces.
+     */
+    encodeWithSpecialTokens(text: string): Uint32Array;
+    /**
+     * Get the encoding name (e.g. `"cl100k_base"`).
+     */
+    readonly name: string;
+}
+
+/**
+ * Get an encoding for an OpenAI model name (e.g. `"gpt-4o"`, `"o3-mini"`).
+ *
+ * Automatically resolves the model name to the correct encoding.
+ * Throws `Error` for unknown model names.
+ */
+export function encodingForModel(model: string): Encoding;
+
+/**
+ * Estimate cost in USD for a given model, input token count, and output token count.
+ *
+ * Supports OpenAI, Anthropic Claude, and Google Gemini models.
+ * Throws `Error` for unknown model ids.
+ */
+export function estimateCost(model_id: string, input_tokens: number, output_tokens: number): number;
+
+/**
+ * Get an encoding by name.
+ *
+ * Supported: `"cl100k_base"`, `"o200k_base"`, `"p50k_base"`, `"p50k_edit"`, `"r50k_base"`.
+ *
+ * Throws `Error` for unknown encoding names.
+ */
+export function getEncoding(name: string): Encoding;
+
+/**
+ * Get model pricing and metadata as a JS object.
+ *
+ * Returns an object with: `id`, `provider`, `input_per_1m`, `output_per_1m`,
+ * `cached_input_per_1m`, `context_window`, `max_output`.
+ *
+ * Throws `Error` for unknown model ids.
+ */
+export function getModelInfo(model_id: string): any;
+
+export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module;
+
+export interface InitOutput {
+    readonly memory: WebAssembly.Memory;
+    readonly __wbg_encoding_free: (a: number, b: number) => void;
+    readonly encodingForModel: (a: number, b: number) => [number, number, number];
+    readonly encoding_count: (a: number, b: number, c: number) => number;
+    readonly encoding_decode: (a: number, b: number, c: number) => [number, number];
+    readonly encoding_encode: (a: number, b: number, c: number) => [number, number];
+    readonly encoding_encodeWithSpecialTokens: (a: number, b: number, c: number) => [number, number];
+    readonly encoding_name: (a: number) => [number, number];
+    readonly estimateCost: (a: number, b: number, c: number, d: number) => [number, number, number];
+    readonly getEncoding: (a: number, b: number) => [number, number, number];
+    readonly getModelInfo: (a: number, b: number) => [number, number, number];
+    readonly __wbindgen_malloc: (a: number, b: number) => number;
+    readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
+    readonly __wbindgen_externrefs: WebAssembly.Table;
+    readonly __externref_table_dealloc: (a: number) => void;
+    readonly __wbindgen_free: (a: number, b: number, c: number) => void;
+    readonly __wbindgen_start: () => void;
+}
+
+export type SyncInitInput = BufferSource | WebAssembly.Module;
+
+/**
+ * Instantiates the given `module`, which can either be bytes or
+ * a precompiled `WebAssembly.Module`.
+ *
+ * @param {{ module: SyncInitInput }} module - Passing `SyncInitInput` directly is deprecated.
+ *
+ * @returns {InitOutput}
+ */
+export function initSync(module: { module: SyncInitInput } | SyncInitInput): InitOutput;
+
+/**
+ * If `module_or_path` is {RequestInfo} or {URL}, makes a request and
+ * for everything else, calls `WebAssembly.instantiate` directly.
+ *
+ * @param {{ module_or_path: InitInput | Promise<InitInput> }} module_or_path - Passing `InitInput` directly is deprecated.
+ *
+ * @returns {Promise<InitOutput>}
+ */
+export default function __wbg_init (module_or_path?: { module_or_path: InitInput | Promise<InitInput> } | InitInput | Promise<InitInput>): Promise<InitOutput>;

package/tiktoken_wasm.js

@@ -0,0 +1,448 @@
+/* @ts-self-types="./tiktoken_wasm.d.ts" */
+
+/**
+ * WASM wrapper around a tiktoken encoding instance.
+ *
+ * Created via [`get_encoding`] or [`encoding_for_model`].
+ * Call `.free()` when done to release WASM memory.
+ */
+export class Encoding {
+    static __wrap(ptr) {
+        ptr = ptr >>> 0;
+        const obj = Object.create(Encoding.prototype);
+        obj.__wbg_ptr = ptr;
+        EncodingFinalization.register(obj, obj.__wbg_ptr, obj);
+        return obj;
+    }
+    __destroy_into_raw() {
+        const ptr = this.__wbg_ptr;
+        this.__wbg_ptr = 0;
+        EncodingFinalization.unregister(this);
+        return ptr;
+    }
+    free() {
+        const ptr = this.__destroy_into_raw();
+        wasm.__wbg_encoding_free(ptr, 0);
+    }
+    /**
+     * Count tokens without building the full token id array.
+     *
+     * Faster than `encode(text).length` for cases where you only need the count.
+     * @param {string} text
+     * @returns {number}
+     */
+    count(text) {
+        const ptr0 = passStringToWasm0(text, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        const len0 = WASM_VECTOR_LEN;
+        const ret = wasm.encoding_count(this.__wbg_ptr, ptr0, len0);
+        return ret >>> 0;
+    }
+    /**
+     * Decode token ids back to a UTF-8 string.
+     *
+     * Uses lossy UTF-8 conversion — invalid byte sequences are replaced with U+FFFD.
+     * @param {Uint32Array} tokens
+     * @returns {string}
+     */
+    decode(tokens) {
+        let deferred2_0;
+        let deferred2_1;
+        try {
+            const ptr0 = passArray32ToWasm0(tokens, wasm.__wbindgen_malloc);
+            const len0 = WASM_VECTOR_LEN;
+            const ret = wasm.encoding_decode(this.__wbg_ptr, ptr0, len0);
+            deferred2_0 = ret[0];
+            deferred2_1 = ret[1];
+            return getStringFromWasm0(ret[0], ret[1]);
+        } finally {
+            wasm.__wbindgen_free(deferred2_0, deferred2_1, 1);
+        }
+    }
+    /**
+     * Encode text into token ids (returns `Uint32Array` in JS).
+     *
+     * Special tokens like `<|endoftext|>` are treated as ordinary text.
+     * Use `encodeWithSpecialTokens()` to recognize them.
+     * @param {string} text
+     * @returns {Uint32Array}
+     */
+    encode(text) {
+        const ptr0 = passStringToWasm0(text, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        const len0 = WASM_VECTOR_LEN;
+        const ret = wasm.encoding_encode(this.__wbg_ptr, ptr0, len0);
+        var v2 = getArrayU32FromWasm0(ret[0], ret[1]).slice();
+        wasm.__wbindgen_free(ret[0], ret[1] * 4, 4);
+        return v2;
+    }
+    /**
+     * Encode text into token ids, recognizing special tokens.
+     *
+     * Special tokens (e.g. `<|endoftext|>`) are encoded as their designated ids
+     * instead of being split into sub-word pieces.
+     * @param {string} text
+     * @returns {Uint32Array}
+     */
+    encodeWithSpecialTokens(text) {
+        const ptr0 = passStringToWasm0(text, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        const len0 = WASM_VECTOR_LEN;
+        const ret = wasm.encoding_encodeWithSpecialTokens(this.__wbg_ptr, ptr0, len0);
+        var v2 = getArrayU32FromWasm0(ret[0], ret[1]).slice();
+        wasm.__wbindgen_free(ret[0], ret[1] * 4, 4);
+        return v2;
+    }
+    /**
+     * Get the encoding name (e.g. `"cl100k_base"`).
+     * @returns {string}
+     */
+    get name() {
+        let deferred1_0;
+        let deferred1_1;
+        try {
+            const ret = wasm.encoding_name(this.__wbg_ptr);
+            deferred1_0 = ret[0];
+            deferred1_1 = ret[1];
+            return getStringFromWasm0(ret[0], ret[1]);
+        } finally {
+            wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
+        }
+    }
+}
+if (Symbol.dispose) Encoding.prototype[Symbol.dispose] = Encoding.prototype.free;
+
+/**
+ * Get an encoding for an OpenAI model name (e.g. `"gpt-4o"`, `"o3-mini"`).
+ *
+ * Automatically resolves the model name to the correct encoding.
+ * Throws `Error` for unknown model names.
+ * @param {string} model
+ * @returns {Encoding}
+ */
+export function encodingForModel(model) {
+    const ptr0 = passStringToWasm0(model, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+    const len0 = WASM_VECTOR_LEN;
+    const ret = wasm.encodingForModel(ptr0, len0);
+    if (ret[2]) {
+        throw takeFromExternrefTable0(ret[1]);
+    }
+    return Encoding.__wrap(ret[0]);
+}
+
+/**
+ * Estimate cost in USD for a given model, input token count, and output token count.
+ *
+ * Supports OpenAI, Anthropic Claude, and Google Gemini models.
+ * Throws `Error` for unknown model ids.
+ * @param {string} model_id
+ * @param {number} input_tokens
+ * @param {number} output_tokens
+ * @returns {number}
+ */
+export function estimateCost(model_id, input_tokens, output_tokens) {
+    const ptr0 = passStringToWasm0(model_id, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+    const len0 = WASM_VECTOR_LEN;
+    const ret = wasm.estimateCost(ptr0, len0, input_tokens, output_tokens);
+    if (ret[2]) {
+        throw takeFromExternrefTable0(ret[1]);
+    }
+    return ret[0];
+}
+
+/**
+ * Get an encoding by name.
+ *
+ * Supported: `"cl100k_base"`, `"o200k_base"`, `"p50k_base"`, `"p50k_edit"`, `"r50k_base"`.
+ *
+ * Throws `Error` for unknown encoding names.
+ * @param {string} name
+ * @returns {Encoding}
+ */
+export function getEncoding(name) {
+    const ptr0 = passStringToWasm0(name, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+    const len0 = WASM_VECTOR_LEN;
+    const ret = wasm.getEncoding(ptr0, len0);
+    if (ret[2]) {
+        throw takeFromExternrefTable0(ret[1]);
+    }
+    return Encoding.__wrap(ret[0]);
+}
+
+/**
+ * Get model pricing and metadata as a JS object.
+ *
+ * Returns an object with: `id`, `provider`, `input_per_1m`, `output_per_1m`,
+ * `cached_input_per_1m`, `context_window`, `max_output`.
+ *
+ * Throws `Error` for unknown model ids.
+ * @param {string} model_id
+ * @returns {any}
+ */
+export function getModelInfo(model_id) {
+    const ptr0 = passStringToWasm0(model_id, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+    const len0 = WASM_VECTOR_LEN;
+    const ret = wasm.getModelInfo(ptr0, len0);
+    if (ret[2]) {
+        throw takeFromExternrefTable0(ret[1]);
+    }
+    return takeFromExternrefTable0(ret[0]);
+}
+
+function __wbg_get_imports() {
+    const import0 = {
+        __proto__: null,
+        __wbg_Error_83742b46f01ce22d: function(arg0, arg1) {
+            const ret = Error(getStringFromWasm0(arg0, arg1));
+            return ret;
+        },
+        __wbg_String_8564e559799eccda: function(arg0, arg1) {
+            const ret = String(arg1);
+            const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+            const len1 = WASM_VECTOR_LEN;
+            getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true);
+            getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true);
+        },
+        __wbg___wbindgen_throw_6ddd609b62940d55: function(arg0, arg1) {
+            throw new Error(getStringFromWasm0(arg0, arg1));
+        },
+        __wbg_new_ab79df5bd7c26067: function() {
+            const ret = new Object();
+            return ret;
+        },
+        __wbg_set_6be42768c690e380: function(arg0, arg1, arg2) {
+            arg0[arg1] = arg2;
+        },
+        __wbindgen_cast_0000000000000001: function(arg0) {
+            // Cast intrinsic for `F64 -> Externref`.
+            const ret = arg0;
+            return ret;
+        },
+        __wbindgen_cast_0000000000000002: function(arg0, arg1) {
+            // Cast intrinsic for `Ref(String) -> Externref`.
+            const ret = getStringFromWasm0(arg0, arg1);
+            return ret;
+        },
+        __wbindgen_init_externref_table: function() {
+            const table = wasm.__wbindgen_externrefs;
+            const offset = table.grow(4);
+            table.set(0, undefined);
+            table.set(offset + 0, undefined);
+            table.set(offset + 1, null);
+            table.set(offset + 2, true);
+            table.set(offset + 3, false);
+        },
+    };
+    return {
+        __proto__: null,
+        "./tiktoken_wasm_bg.js": import0,
+    };
+}
+
+const EncodingFinalization = (typeof FinalizationRegistry === 'undefined')
+    ? { register: () => {}, unregister: () => {} }
+    : new FinalizationRegistry(ptr => wasm.__wbg_encoding_free(ptr >>> 0, 1));
+
+function getArrayU32FromWasm0(ptr, len) {
+    ptr = ptr >>> 0;
+    return getUint32ArrayMemory0().subarray(ptr / 4, ptr / 4 + len);
+}
+
+let cachedDataViewMemory0 = null;
+function getDataViewMemory0() {
+    if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) {
+        cachedDataViewMemory0 = new DataView(wasm.memory.buffer);
+    }
+    return cachedDataViewMemory0;
+}
+
+function getStringFromWasm0(ptr, len) {
+    ptr = ptr >>> 0;
+    return decodeText(ptr, len);
+}
+
+let cachedUint32ArrayMemory0 = null;
+function getUint32ArrayMemory0() {
+    if (cachedUint32ArrayMemory0 === null || cachedUint32ArrayMemory0.byteLength === 0) {
+        cachedUint32ArrayMemory0 = new Uint32Array(wasm.memory.buffer);
+    }
+    return cachedUint32ArrayMemory0;
+}
+
+let cachedUint8ArrayMemory0 = null;
+function getUint8ArrayMemory0() {
+    if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) {
+        cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer);
+    }
+    return cachedUint8ArrayMemory0;
+}
+
+function passArray32ToWasm0(arg, malloc) {
+    const ptr = malloc(arg.length * 4, 4) >>> 0;
+    getUint32ArrayMemory0().set(arg, ptr / 4);
+    WASM_VECTOR_LEN = arg.length;
+    return ptr;
+}
+
+function passStringToWasm0(arg, malloc, realloc) {
+    if (realloc === undefined) {
+        const buf = cachedTextEncoder.encode(arg);
+        const ptr = malloc(buf.length, 1) >>> 0;
+        getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf);
+        WASM_VECTOR_LEN = buf.length;
+        return ptr;
+    }
+
+    let len = arg.length;
+    let ptr = malloc(len, 1) >>> 0;
+
+    const mem = getUint8ArrayMemory0();
+
+    let offset = 0;
+
+    for (; offset < len; offset++) {
+        const code = arg.charCodeAt(offset);
+        if (code > 0x7F) break;
+        mem[ptr + offset] = code;
+    }
+    if (offset !== len) {
+        if (offset !== 0) {
+            arg = arg.slice(offset);
+        }
+        ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0;
+        const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len);
+        const ret = cachedTextEncoder.encodeInto(arg, view);
+
+        offset += ret.written;
+        ptr = realloc(ptr, len, offset, 1) >>> 0;
+    }
+
+    WASM_VECTOR_LEN = offset;
+    return ptr;
+}
+
+function takeFromExternrefTable0(idx) {
+    const value = wasm.__wbindgen_externrefs.get(idx);
+    wasm.__externref_table_dealloc(idx);
+    return value;
+}
+
+let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
+cachedTextDecoder.decode();
+const MAX_SAFARI_DECODE_BYTES = 2146435072;
+let numBytesDecoded = 0;
+function decodeText(ptr, len) {
+    numBytesDecoded += len;
+    if (numBytesDecoded >= MAX_SAFARI_DECODE_BYTES) {
+        cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
+        cachedTextDecoder.decode();
+        numBytesDecoded = len;
+    }
+    return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len));
+}
+
+const cachedTextEncoder = new TextEncoder();
+
+if (!('encodeInto' in cachedTextEncoder)) {
+    cachedTextEncoder.encodeInto = function (arg, view) {
+        const buf = cachedTextEncoder.encode(arg);
+        view.set(buf);
+        return {
+            read: arg.length,
+            written: buf.length
+        };
+    };
+}
+
+let WASM_VECTOR_LEN = 0;
+
+let wasmModule, wasm;
+function __wbg_finalize_init(instance, module) {
+    wasm = instance.exports;
+    wasmModule = module;
+    cachedDataViewMemory0 = null;
+    cachedUint32ArrayMemory0 = null;
+    cachedUint8ArrayMemory0 = null;
+    wasm.__wbindgen_start();
+    return wasm;
+}
+
+async function __wbg_load(module, imports) {
+    if (typeof Response === 'function' && module instanceof Response) {
+        if (typeof WebAssembly.instantiateStreaming === 'function') {
+            try {
+                return await WebAssembly.instantiateStreaming(module, imports);
+            } catch (e) {
+                const validResponse = module.ok && expectedResponseType(module.type);
+
+                if (validResponse && module.headers.get('Content-Type') !== 'application/wasm') {
+                    console.warn("`WebAssembly.instantiateStreaming` failed because your server does not serve Wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n", e);
+
+                } else { throw e; }
+            }
+        }
+
+        const bytes = await module.arrayBuffer();
+        return await WebAssembly.instantiate(bytes, imports);
+    } else {
+        const instance = await WebAssembly.instantiate(module, imports);
+
+        if (instance instanceof WebAssembly.Instance) {
+            return { instance, module };
+        } else {
+            return instance;
+        }
+    }
+
+    function expectedResponseType(type) {
+        switch (type) {
+            case 'basic': case 'cors': case 'default': return true;
+        }
+        return false;
+    }
+}
+
+function initSync(module) {
+    if (wasm !== undefined) return wasm;
+
+
+    if (module !== undefined) {
+        if (Object.getPrototypeOf(module) === Object.prototype) {
+            ({module} = module)
+        } else {
+            console.warn('using deprecated parameters for `initSync()`; pass a single object instead')
+        }
+    }
+
+    const imports = __wbg_get_imports();
+    if (!(module instanceof WebAssembly.Module)) {
+        module = new WebAssembly.Module(module);
+    }
+    const instance = new WebAssembly.Instance(module, imports);
+    return __wbg_finalize_init(instance, module);
+}
+
+async function __wbg_init(module_or_path) {
+    if (wasm !== undefined) return wasm;
+
+
+    if (module_or_path !== undefined) {
+        if (Object.getPrototypeOf(module_or_path) === Object.prototype) {
+            ({module_or_path} = module_or_path)
+        } else {
+            console.warn('using deprecated parameters for the initialization function; pass a single object instead')
+        }
+    }
+
+    if (module_or_path === undefined) {
+        module_or_path = new URL('tiktoken_wasm_bg.wasm', import.meta.url);
+    }
+    const imports = __wbg_get_imports();
+
+    if (typeof module_or_path === 'string' || (typeof Request === 'function' && module_or_path instanceof Request) || (typeof URL === 'function' && module_or_path instanceof URL)) {
+        module_or_path = fetch(module_or_path);
+    }
+
+    const { instance, module } = await __wbg_load(await module_or_path, imports);
+
+    return __wbg_finalize_init(instance, module);
+}
+
+export { initSync, __wbg_init as default };