@goliapkg/tiktoken-wasm 3.2.0 → 3.2.2

package/tiktoken_wasm.js

@@ -1,638 +1,9 @@
 /* @ts-self-types="./tiktoken_wasm.d.ts" */
 
-/**
- * WASM wrapper around a tiktoken encoding instance.
- *
- * Created via [`get_encoding`] or [`encoding_for_model`].
- * Call `.free()` when done to release WASM memory.
- */
-export class Encoding {
-    static __wrap(ptr) {
-        ptr = ptr >>> 0;
-        const obj = Object.create(Encoding.prototype);
-        obj.__wbg_ptr = ptr;
-        EncodingFinalization.register(obj, obj.__wbg_ptr, obj);
-        return obj;
-    }
-    __destroy_into_raw() {
-        const ptr = this.__wbg_ptr;
-        this.__wbg_ptr = 0;
-        EncodingFinalization.unregister(this);
-        return ptr;
-    }
-    free() {
-        const ptr = this.__destroy_into_raw();
-        wasm.__wbg_encoding_free(ptr, 0);
-    }
-    /**
-     * Count tokens without building the full token id array.
-     *
-     * Faster than `encode(text).length` for cases where you only need the count.
-     * @param {string} text
-     * @returns {number}
-     */
-    count(text) {
-        const ptr0 = passStringToWasm0(text, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
-        const len0 = WASM_VECTOR_LEN;
-        const ret = wasm.encoding_count(this.__wbg_ptr, ptr0, len0);
-        return ret >>> 0;
-    }
-    /**
-     * Count tokens, recognizing special tokens.
-     *
-     * Like `count()` but special tokens (e.g. `<|endoftext|>`) are counted
-     * as single tokens instead of being split into sub-word pieces.
-     * @param {string} text
-     * @returns {number}
-     */
-    countWithSpecialTokens(text) {
-        const ptr0 = passStringToWasm0(text, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
-        const len0 = WASM_VECTOR_LEN;
-        const ret = wasm.encoding_countWithSpecialTokens(this.__wbg_ptr, ptr0, len0);
-        return ret >>> 0;
-    }
-    /**
-     * Decode token ids back to a UTF-8 string.
-     *
-     * Uses lossy UTF-8 conversion — invalid byte sequences are replaced with U+FFFD.
-     * @param {Uint32Array} tokens
-     * @returns {string}
-     */
-    decode(tokens) {
-        let deferred2_0;
-        let deferred2_1;
-        try {
-            const ptr0 = passArray32ToWasm0(tokens, wasm.__wbindgen_malloc);
-            const len0 = WASM_VECTOR_LEN;
-            const ret = wasm.encoding_decode(this.__wbg_ptr, ptr0, len0);
-            deferred2_0 = ret[0];
-            deferred2_1 = ret[1];
-            return getStringFromWasm0(ret[0], ret[1]);
-        } finally {
-            wasm.__wbindgen_free(deferred2_0, deferred2_1, 1);
-        }
-    }
-    /**
-     * Encode text into token ids (returns `Uint32Array` in JS).
-     *
-     * Special tokens like `<|endoftext|>` are treated as ordinary text.
-     * Use `encodeWithSpecialTokens()` to recognize them.
-     * @param {string} text
-     * @returns {Uint32Array}
-     */
-    encode(text) {
-        const ptr0 = passStringToWasm0(text, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
-        const len0 = WASM_VECTOR_LEN;
-        const ret = wasm.encoding_encode(this.__wbg_ptr, ptr0, len0);
-        var v2 = getArrayU32FromWasm0(ret[0], ret[1]).slice();
-        wasm.__wbindgen_free(ret[0], ret[1] * 4, 4);
-        return v2;
-    }
-    /**
-     * Encode text into token ids, recognizing special tokens.
-     *
-     * Special tokens (e.g. `<|endoftext|>`) are encoded as their designated ids
-     * instead of being split into sub-word pieces.
-     * @param {string} text
-     * @returns {Uint32Array}
-     */
-    encodeWithSpecialTokens(text) {
-        const ptr0 = passStringToWasm0(text, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
-        const len0 = WASM_VECTOR_LEN;
-        const ret = wasm.encoding_encodeWithSpecialTokens(this.__wbg_ptr, ptr0, len0);
-        var v2 = getArrayU32FromWasm0(ret[0], ret[1]).slice();
-        wasm.__wbindgen_free(ret[0], ret[1] * 4, 4);
-        return v2;
-    }
-    /**
-     * Get the encoding name (e.g. `"cl100k_base"`).
-     * @returns {string}
-     */
-    get name() {
-        let deferred1_0;
-        let deferred1_1;
-        try {
-            const ret = wasm.encoding_name(this.__wbg_ptr);
-            deferred1_0 = ret[0];
-            deferred1_1 = ret[1];
-            return getStringFromWasm0(ret[0], ret[1]);
-        } finally {
-            wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
-        }
-    }
-    /**
-     * Get the number of special tokens in the vocabulary.
-     * @returns {number}
-     */
-    get numSpecialTokens() {
-        const ret = wasm.encoding_numSpecialTokens(this.__wbg_ptr);
-        return ret >>> 0;
-    }
-    /**
-     * Get the number of regular (non-special) tokens in the vocabulary.
-     * @returns {number}
-     */
-    get vocabSize() {
-        const ret = wasm.encoding_vocabSize(this.__wbg_ptr);
-        return ret >>> 0;
-    }
-}
-if (Symbol.dispose) Encoding.prototype[Symbol.dispose] = Encoding.prototype.free;
+import * as wasm from "./tiktoken_wasm_bg.wasm";
+import { __wbg_set_wasm } from "./tiktoken_wasm_bg.js";
+__wbg_set_wasm(wasm);
 
-/**
- * Model pricing and metadata.
- */
-export class ModelInfo {
-    static __wrap(ptr) {
-        ptr = ptr >>> 0;
-        const obj = Object.create(ModelInfo.prototype);
-        obj.__wbg_ptr = ptr;
-        ModelInfoFinalization.register(obj, obj.__wbg_ptr, obj);
-        return obj;
-    }
-    __destroy_into_raw() {
-        const ptr = this.__wbg_ptr;
-        this.__wbg_ptr = 0;
-        ModelInfoFinalization.unregister(this);
-        return ptr;
-    }
-    free() {
-        const ptr = this.__destroy_into_raw();
-        wasm.__wbg_modelinfo_free(ptr, 0);
-    }
-    /**
-     * @returns {number | undefined}
-     */
-    get cachedInputPer1m() {
-        const ret = wasm.modelinfo_cachedInputPer1m(this.__wbg_ptr);
-        return ret[0] === 0 ? undefined : ret[1];
-    }
-    /**
-     * @returns {number}
-     */
-    get contextWindow() {
-        const ret = wasm.modelinfo_contextWindow(this.__wbg_ptr);
-        return ret >>> 0;
-    }
-    /**
-     * @returns {string}
-     */
-    get id() {
-        let deferred1_0;
-        let deferred1_1;
-        try {
-            const ret = wasm.modelinfo_id(this.__wbg_ptr);
-            deferred1_0 = ret[0];
-            deferred1_1 = ret[1];
-            return getStringFromWasm0(ret[0], ret[1]);
-        } finally {
-            wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
-        }
-    }
-    /**
-     * @returns {number}
-     */
-    get inputPer1m() {
-        const ret = wasm.modelinfo_inputPer1m(this.__wbg_ptr);
-        return ret;
-    }
-    /**
-     * @returns {number}
-     */
-    get maxOutput() {
-        const ret = wasm.modelinfo_maxOutput(this.__wbg_ptr);
-        return ret >>> 0;
-    }
-    /**
-     * @returns {number}
-     */
-    get outputPer1m() {
-        const ret = wasm.modelinfo_outputPer1m(this.__wbg_ptr);
-        return ret;
-    }
-    /**
-     * @returns {string}
-     */
-    get provider() {
-        let deferred1_0;
-        let deferred1_1;
-        try {
-            const ret = wasm.modelinfo_provider(this.__wbg_ptr);
-            deferred1_0 = ret[0];
-            deferred1_1 = ret[1];
-            return getStringFromWasm0(ret[0], ret[1]);
-        } finally {
-            wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
-        }
-    }
-}
-if (Symbol.dispose) ModelInfo.prototype[Symbol.dispose] = ModelInfo.prototype.free;
-
-/**
- * List all supported models with pricing info.
- *
- * Returns an array of `ModelInfo` objects.
- * @returns {ModelInfo[]}
- */
-export function allModels() {
-    const ret = wasm.allModels();
-    var v1 = getArrayJsValueFromWasm0(ret[0], ret[1]).slice();
-    wasm.__wbindgen_free(ret[0], ret[1] * 4, 4);
-    return v1;
-}
-
-/**
- * Get an encoding for a model name (e.g. `"gpt-4o"`, `"o3-mini"`, `"llama-4"`, `"deepseek-r1"`).
- *
- * Supports models from OpenAI, Meta, DeepSeek, Qwen, and Mistral.
- * Automatically resolves the model name to the correct encoding.
- * Throws `Error` for unknown model names.
- * @param {string} model
- * @returns {Encoding}
- */
-export function encodingForModel(model) {
-    const ptr0 = passStringToWasm0(model, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
-    const len0 = WASM_VECTOR_LEN;
-    const ret = wasm.encodingForModel(ptr0, len0);
-    if (ret[2]) {
-        throw takeFromExternrefTable0(ret[1]);
-    }
-    return Encoding.__wrap(ret[0]);
-}
-
-/**
- * Estimate cost in USD for a given model, input token count, and output token count.
- *
- * Supports OpenAI, Anthropic Claude, Google Gemini, Meta Llama, DeepSeek, Qwen, and Mistral models.
- * Throws `Error` for unknown model ids.
- * @param {string} model_id
- * @param {number} input_tokens
- * @param {number} output_tokens
- * @returns {number}
- */
-export function estimateCost(model_id, input_tokens, output_tokens) {
-    const ptr0 = passStringToWasm0(model_id, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
-    const len0 = WASM_VECTOR_LEN;
-    const ret = wasm.estimateCost(ptr0, len0, input_tokens, output_tokens);
-    if (ret[2]) {
-        throw takeFromExternrefTable0(ret[1]);
-    }
-    return ret[0];
-}
-
-/**
- * Get an encoding by name.
- *
- * Supported encodings:
- * - `"cl100k_base"` — GPT-4, GPT-3.5-turbo
- * - `"o200k_base"` — GPT-4o, GPT-4.1, o1, o3
- * - `"p50k_base"` — text-davinci-002/003
- * - `"p50k_edit"` — text-davinci-edit
- * - `"r50k_base"` — GPT-3 (davinci, curie, etc.)
- * - `"llama3"` — Meta Llama 3/4
- * - `"deepseek_v3"` — DeepSeek V3/R1
- * - `"qwen2"` — Qwen 2/2.5/3
- * - `"mistral_v3"` — Mistral/Codestral/Pixtral
- *
- * Throws `Error` for unknown encoding names.
- * @param {string} name
- * @returns {Encoding}
- */
-export function getEncoding(name) {
-    const ptr0 = passStringToWasm0(name, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
-    const len0 = WASM_VECTOR_LEN;
-    const ret = wasm.getEncoding(ptr0, len0);
-    if (ret[2]) {
-        throw takeFromExternrefTable0(ret[1]);
-    }
-    return Encoding.__wrap(ret[0]);
-}
-
-/**
- * Get model pricing and metadata.
- *
- * Returns a typed object with: `id`, `provider`, `inputPer1m`, `outputPer1m`,
- * `cachedInputPer1m`, `contextWindow`, `maxOutput`.
- *
- * Throws `Error` for unknown model ids.
- * @param {string} model_id
- * @returns {ModelInfo}
- */
-export function getModelInfo(model_id) {
-    const ptr0 = passStringToWasm0(model_id, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
-    const len0 = WASM_VECTOR_LEN;
-    const ret = wasm.getModelInfo(ptr0, len0);
-    if (ret[2]) {
-        throw takeFromExternrefTable0(ret[1]);
-    }
-    return ModelInfo.__wrap(ret[0]);
-}
-
-/**
- * List all available encoding names.
- *
- * Returns an array of strings: `["cl100k_base", "o200k_base", ...]`
- * @returns {any[]}
- */
-export function listEncodings() {
-    const ret = wasm.listEncodings();
-    var v1 = getArrayJsValueFromWasm0(ret[0], ret[1]).slice();
-    wasm.__wbindgen_free(ret[0], ret[1] * 4, 4);
-    return v1;
-}
-
-/**
- * Map a model name to its encoding name without loading the encoding.
- *
- * Returns the encoding name string (e.g. `"o200k_base"`) or `null` for unknown models.
- * @param {string} model
- * @returns {string | undefined}
- */
-export function modelToEncoding(model) {
-    const ptr0 = passStringToWasm0(model, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
-    const len0 = WASM_VECTOR_LEN;
-    const ret = wasm.modelToEncoding(ptr0, len0);
-    let v2;
-    if (ret[0] !== 0) {
-        v2 = getStringFromWasm0(ret[0], ret[1]).slice();
-        wasm.__wbindgen_free(ret[0], ret[1] * 1, 1);
-    }
-    return v2;
-}
-
-/**
- * List models filtered by provider name.
- *
- * Provider names: `"OpenAI"`, `"Anthropic"`, `"Google"`, `"Meta"`, `"DeepSeek"`, `"Alibaba"`, `"Mistral"`.
- * Returns an empty array for unknown providers.
- * @param {string} provider
- * @returns {ModelInfo[]}
- */
-export function modelsByProvider(provider) {
-    const ptr0 = passStringToWasm0(provider, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
-    const len0 = WASM_VECTOR_LEN;
-    const ret = wasm.modelsByProvider(ptr0, len0);
-    var v2 = getArrayJsValueFromWasm0(ret[0], ret[1]).slice();
-    wasm.__wbindgen_free(ret[0], ret[1] * 4, 4);
-    return v2;
-}
-
-function __wbg_get_imports() {
-    const import0 = {
-        __proto__: null,
-        __wbg_Error_83742b46f01ce22d: function(arg0, arg1) {
-            const ret = Error(getStringFromWasm0(arg0, arg1));
-            return ret;
-        },
-        __wbg___wbindgen_throw_6ddd609b62940d55: function(arg0, arg1) {
-            throw new Error(getStringFromWasm0(arg0, arg1));
-        },
-        __wbg_modelinfo_new: function(arg0) {
-            const ret = ModelInfo.__wrap(arg0);
-            return ret;
-        },
-        __wbindgen_cast_0000000000000001: function(arg0, arg1) {
-            // Cast intrinsic for `Ref(String) -> Externref`.
-            const ret = getStringFromWasm0(arg0, arg1);
-            return ret;
-        },
-        __wbindgen_init_externref_table: function() {
-            const table = wasm.__wbindgen_externrefs;
-            const offset = table.grow(4);
-            table.set(0, undefined);
-            table.set(offset + 0, undefined);
-            table.set(offset + 1, null);
-            table.set(offset + 2, true);
-            table.set(offset + 3, false);
-        },
-    };
-    return {
-        __proto__: null,
-        "./tiktoken_wasm_bg.js": import0,
-    };
-}
-
-const EncodingFinalization = (typeof FinalizationRegistry === 'undefined')
-    ? { register: () => {}, unregister: () => {} }
-    : new FinalizationRegistry(ptr => wasm.__wbg_encoding_free(ptr >>> 0, 1));
-const ModelInfoFinalization = (typeof FinalizationRegistry === 'undefined')
-    ? { register: () => {}, unregister: () => {} }
-    : new FinalizationRegistry(ptr => wasm.__wbg_modelinfo_free(ptr >>> 0, 1));
-
-function getArrayJsValueFromWasm0(ptr, len) {
-    ptr = ptr >>> 0;
-    const mem = getDataViewMemory0();
-    const result = [];
-    for (let i = ptr; i < ptr + 4 * len; i += 4) {
-        result.push(wasm.__wbindgen_externrefs.get(mem.getUint32(i, true)));
-    }
-    wasm.__externref_drop_slice(ptr, len);
-    return result;
-}
-
-function getArrayU32FromWasm0(ptr, len) {
-    ptr = ptr >>> 0;
-    return getUint32ArrayMemory0().subarray(ptr / 4, ptr / 4 + len);
-}
-
-let cachedDataViewMemory0 = null;
-function getDataViewMemory0() {
-    if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) {
-        cachedDataViewMemory0 = new DataView(wasm.memory.buffer);
-    }
-    return cachedDataViewMemory0;
-}
-
-function getStringFromWasm0(ptr, len) {
-    ptr = ptr >>> 0;
-    return decodeText(ptr, len);
-}
-
-let cachedUint32ArrayMemory0 = null;
-function getUint32ArrayMemory0() {
-    if (cachedUint32ArrayMemory0 === null || cachedUint32ArrayMemory0.byteLength === 0) {
-        cachedUint32ArrayMemory0 = new Uint32Array(wasm.memory.buffer);
-    }
-    return cachedUint32ArrayMemory0;
-}
-
-let cachedUint8ArrayMemory0 = null;
-function getUint8ArrayMemory0() {
-    if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) {
-        cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer);
-    }
-    return cachedUint8ArrayMemory0;
-}
-
-function passArray32ToWasm0(arg, malloc) {
-    const ptr = malloc(arg.length * 4, 4) >>> 0;
-    getUint32ArrayMemory0().set(arg, ptr / 4);
-    WASM_VECTOR_LEN = arg.length;
-    return ptr;
-}
-
-function passStringToWasm0(arg, malloc, realloc) {
-    if (realloc === undefined) {
-        const buf = cachedTextEncoder.encode(arg);
-        const ptr = malloc(buf.length, 1) >>> 0;
-        getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf);
-        WASM_VECTOR_LEN = buf.length;
-        return ptr;
-    }
-
-    let len = arg.length;
-    let ptr = malloc(len, 1) >>> 0;
-
-    const mem = getUint8ArrayMemory0();
-
-    let offset = 0;
-
-    for (; offset < len; offset++) {
-        const code = arg.charCodeAt(offset);
-        if (code > 0x7F) break;
-        mem[ptr + offset] = code;
-    }
-    if (offset !== len) {
-        if (offset !== 0) {
-            arg = arg.slice(offset);
-        }
-        ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0;
-        const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len);
-        const ret = cachedTextEncoder.encodeInto(arg, view);
-
-        offset += ret.written;
-        ptr = realloc(ptr, len, offset, 1) >>> 0;
-    }
-
-    WASM_VECTOR_LEN = offset;
-    return ptr;
-}
-
-function takeFromExternrefTable0(idx) {
-    const value = wasm.__wbindgen_externrefs.get(idx);
-    wasm.__externref_table_dealloc(idx);
-    return value;
-}
-
-let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
-cachedTextDecoder.decode();
-const MAX_SAFARI_DECODE_BYTES = 2146435072;
-let numBytesDecoded = 0;
-function decodeText(ptr, len) {
-    numBytesDecoded += len;
-    if (numBytesDecoded >= MAX_SAFARI_DECODE_BYTES) {
-        cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
-        cachedTextDecoder.decode();
-        numBytesDecoded = len;
-    }
-    return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len));
-}
-
-const cachedTextEncoder = new TextEncoder();
-
-if (!('encodeInto' in cachedTextEncoder)) {
-    cachedTextEncoder.encodeInto = function (arg, view) {
-        const buf = cachedTextEncoder.encode(arg);
-        view.set(buf);
-        return {
-            read: arg.length,
-            written: buf.length
-        };
-    };
-}
-
-let WASM_VECTOR_LEN = 0;
-
-let wasmModule, wasm;
-function __wbg_finalize_init(instance, module) {
-    wasm = instance.exports;
-    wasmModule = module;
-    cachedDataViewMemory0 = null;
-    cachedUint32ArrayMemory0 = null;
-    cachedUint8ArrayMemory0 = null;
-    wasm.__wbindgen_start();
-    return wasm;
-}
-
-async function __wbg_load(module, imports) {
-    if (typeof Response === 'function' && module instanceof Response) {
-        if (typeof WebAssembly.instantiateStreaming === 'function') {
-            try {
-                return await WebAssembly.instantiateStreaming(module, imports);
-            } catch (e) {
-                const validResponse = module.ok && expectedResponseType(module.type);
-
-                if (validResponse && module.headers.get('Content-Type') !== 'application/wasm') {
-                    console.warn("`WebAssembly.instantiateStreaming` failed because your server does not serve Wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n", e);
-
-                } else { throw e; }
-            }
-        }
-
-        const bytes = await module.arrayBuffer();
-        return await WebAssembly.instantiate(bytes, imports);
-    } else {
-        const instance = await WebAssembly.instantiate(module, imports);
-
-        if (instance instanceof WebAssembly.Instance) {
-            return { instance, module };
-        } else {
-            return instance;
-        }
-    }
-
-    function expectedResponseType(type) {
-        switch (type) {
-            case 'basic': case 'cors': case 'default': return true;
-        }
-        return false;
-    }
-}
-
-function initSync(module) {
-    if (wasm !== undefined) return wasm;
-
-
-    if (module !== undefined) {
-        if (Object.getPrototypeOf(module) === Object.prototype) {
-            ({module} = module)
-        } else {
-            console.warn('using deprecated parameters for `initSync()`; pass a single object instead')
-        }
-    }
-
-    const imports = __wbg_get_imports();
-    if (!(module instanceof WebAssembly.Module)) {
-        module = new WebAssembly.Module(module);
-    }
-    const instance = new WebAssembly.Instance(module, imports);
-    return __wbg_finalize_init(instance, module);
-}
-
-async function __wbg_init(module_or_path) {
-    if (wasm !== undefined) return wasm;
-
-
-    if (module_or_path !== undefined) {
-        if (Object.getPrototypeOf(module_or_path) === Object.prototype) {
-            ({module_or_path} = module_or_path)
-        } else {
-            console.warn('using deprecated parameters for the initialization function; pass a single object instead')
-        }
-    }
-
-    if (module_or_path === undefined) {
-        module_or_path = new URL('tiktoken_wasm_bg.wasm', import.meta.url);
-    }
-    const imports = __wbg_get_imports();
-
-    if (typeof module_or_path === 'string' || (typeof Request === 'function' && module_or_path instanceof Request) || (typeof URL === 'function' && module_or_path instanceof URL)) {
-        module_or_path = fetch(module_or_path);
-    }
-
-    const { instance, module } = await __wbg_load(await module_or_path, imports);
-
-    return __wbg_finalize_init(instance, module);
-}
-
-export { initSync, __wbg_init as default };
+export {
+    Encoding, ModelInfo, allModels, encodingForModel, estimateCost, getEncoding, getModelInfo, listEncodings, modelToEncoding, modelsByProvider
+} from "./tiktoken_wasm_bg.js";