@goliapkg/tiktoken-wasm 3.2.1 → 3.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -8
- package/package.json +4 -5
- package/tiktoken_wasm.d.ts +0 -60
- package/tiktoken_wasm.js +6 -635
- package/tiktoken_wasm_bg.js +646 -0
- package/tiktoken_wasm_bg.wasm +0 -0
|
@@ -0,0 +1,646 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WASM wrapper around a tiktoken encoding instance.
|
|
3
|
+
*
|
|
4
|
+
* Created via [`get_encoding`] or [`encoding_for_model`].
|
|
5
|
+
* Call `.free()` when done to release WASM memory.
|
|
6
|
+
*/
|
|
7
|
+
export class Encoding {
|
|
8
|
+
static __wrap(ptr) {
|
|
9
|
+
ptr = ptr >>> 0;
|
|
10
|
+
const obj = Object.create(Encoding.prototype);
|
|
11
|
+
obj.__wbg_ptr = ptr;
|
|
12
|
+
EncodingFinalization.register(obj, obj.__wbg_ptr, obj);
|
|
13
|
+
return obj;
|
|
14
|
+
}
|
|
15
|
+
__destroy_into_raw() {
|
|
16
|
+
const ptr = this.__wbg_ptr;
|
|
17
|
+
this.__wbg_ptr = 0;
|
|
18
|
+
EncodingFinalization.unregister(this);
|
|
19
|
+
return ptr;
|
|
20
|
+
}
|
|
21
|
+
free() {
|
|
22
|
+
const ptr = this.__destroy_into_raw();
|
|
23
|
+
wasm.__wbg_encoding_free(ptr, 0);
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Count tokens without building the full token id array.
|
|
27
|
+
*
|
|
28
|
+
* Faster than `encode(text).length` for cases where you only need the count.
|
|
29
|
+
* @param {string} text
|
|
30
|
+
* @returns {number}
|
|
31
|
+
*/
|
|
32
|
+
count(text) {
|
|
33
|
+
const ptr0 = passStringToWasm0(text, wasm.__wbindgen_export2, wasm.__wbindgen_export3);
|
|
34
|
+
const len0 = WASM_VECTOR_LEN;
|
|
35
|
+
const ret = wasm.encoding_count(this.__wbg_ptr, ptr0, len0);
|
|
36
|
+
return ret >>> 0;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Count tokens, recognizing special tokens.
|
|
40
|
+
*
|
|
41
|
+
* Like `count()` but special tokens (e.g. `<|endoftext|>`) are counted
|
|
42
|
+
* as single tokens instead of being split into sub-word pieces.
|
|
43
|
+
* @param {string} text
|
|
44
|
+
* @returns {number}
|
|
45
|
+
*/
|
|
46
|
+
countWithSpecialTokens(text) {
|
|
47
|
+
const ptr0 = passStringToWasm0(text, wasm.__wbindgen_export2, wasm.__wbindgen_export3);
|
|
48
|
+
const len0 = WASM_VECTOR_LEN;
|
|
49
|
+
const ret = wasm.encoding_countWithSpecialTokens(this.__wbg_ptr, ptr0, len0);
|
|
50
|
+
return ret >>> 0;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Decode token ids back to a UTF-8 string.
|
|
54
|
+
*
|
|
55
|
+
* Uses lossy UTF-8 conversion — invalid byte sequences are replaced with U+FFFD.
|
|
56
|
+
* @param {Uint32Array} tokens
|
|
57
|
+
* @returns {string}
|
|
58
|
+
*/
|
|
59
|
+
decode(tokens) {
|
|
60
|
+
let deferred2_0;
|
|
61
|
+
let deferred2_1;
|
|
62
|
+
try {
|
|
63
|
+
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
|
|
64
|
+
const ptr0 = passArray32ToWasm0(tokens, wasm.__wbindgen_export2);
|
|
65
|
+
const len0 = WASM_VECTOR_LEN;
|
|
66
|
+
wasm.encoding_decode(retptr, this.__wbg_ptr, ptr0, len0);
|
|
67
|
+
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
|
|
68
|
+
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
|
|
69
|
+
deferred2_0 = r0;
|
|
70
|
+
deferred2_1 = r1;
|
|
71
|
+
return getStringFromWasm0(r0, r1);
|
|
72
|
+
} finally {
|
|
73
|
+
wasm.__wbindgen_add_to_stack_pointer(16);
|
|
74
|
+
wasm.__wbindgen_export(deferred2_0, deferred2_1, 1);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Encode text into token ids (returns `Uint32Array` in JS).
|
|
79
|
+
*
|
|
80
|
+
* Special tokens like `<|endoftext|>` are treated as ordinary text.
|
|
81
|
+
* Use `encodeWithSpecialTokens()` to recognize them.
|
|
82
|
+
* @param {string} text
|
|
83
|
+
* @returns {Uint32Array}
|
|
84
|
+
*/
|
|
85
|
+
encode(text) {
|
|
86
|
+
try {
|
|
87
|
+
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
|
|
88
|
+
const ptr0 = passStringToWasm0(text, wasm.__wbindgen_export2, wasm.__wbindgen_export3);
|
|
89
|
+
const len0 = WASM_VECTOR_LEN;
|
|
90
|
+
wasm.encoding_encode(retptr, this.__wbg_ptr, ptr0, len0);
|
|
91
|
+
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
|
|
92
|
+
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
|
|
93
|
+
var v2 = getArrayU32FromWasm0(r0, r1).slice();
|
|
94
|
+
wasm.__wbindgen_export(r0, r1 * 4, 4);
|
|
95
|
+
return v2;
|
|
96
|
+
} finally {
|
|
97
|
+
wasm.__wbindgen_add_to_stack_pointer(16);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Encode text into token ids, recognizing special tokens.
|
|
102
|
+
*
|
|
103
|
+
* Special tokens (e.g. `<|endoftext|>`) are encoded as their designated ids
|
|
104
|
+
* instead of being split into sub-word pieces.
|
|
105
|
+
* @param {string} text
|
|
106
|
+
* @returns {Uint32Array}
|
|
107
|
+
*/
|
|
108
|
+
encodeWithSpecialTokens(text) {
|
|
109
|
+
try {
|
|
110
|
+
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
|
|
111
|
+
const ptr0 = passStringToWasm0(text, wasm.__wbindgen_export2, wasm.__wbindgen_export3);
|
|
112
|
+
const len0 = WASM_VECTOR_LEN;
|
|
113
|
+
wasm.encoding_encodeWithSpecialTokens(retptr, this.__wbg_ptr, ptr0, len0);
|
|
114
|
+
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
|
|
115
|
+
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
|
|
116
|
+
var v2 = getArrayU32FromWasm0(r0, r1).slice();
|
|
117
|
+
wasm.__wbindgen_export(r0, r1 * 4, 4);
|
|
118
|
+
return v2;
|
|
119
|
+
} finally {
|
|
120
|
+
wasm.__wbindgen_add_to_stack_pointer(16);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Get the encoding name (e.g. `"cl100k_base"`).
|
|
125
|
+
* @returns {string}
|
|
126
|
+
*/
|
|
127
|
+
get name() {
|
|
128
|
+
let deferred1_0;
|
|
129
|
+
let deferred1_1;
|
|
130
|
+
try {
|
|
131
|
+
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
|
|
132
|
+
wasm.encoding_name(retptr, this.__wbg_ptr);
|
|
133
|
+
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
|
|
134
|
+
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
|
|
135
|
+
deferred1_0 = r0;
|
|
136
|
+
deferred1_1 = r1;
|
|
137
|
+
return getStringFromWasm0(r0, r1);
|
|
138
|
+
} finally {
|
|
139
|
+
wasm.__wbindgen_add_to_stack_pointer(16);
|
|
140
|
+
wasm.__wbindgen_export(deferred1_0, deferred1_1, 1);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* Get the number of special tokens in the vocabulary.
|
|
145
|
+
* @returns {number}
|
|
146
|
+
*/
|
|
147
|
+
get numSpecialTokens() {
|
|
148
|
+
const ret = wasm.encoding_numSpecialTokens(this.__wbg_ptr);
|
|
149
|
+
return ret >>> 0;
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Get the number of regular (non-special) tokens in the vocabulary.
|
|
153
|
+
* @returns {number}
|
|
154
|
+
*/
|
|
155
|
+
get vocabSize() {
|
|
156
|
+
const ret = wasm.encoding_vocabSize(this.__wbg_ptr);
|
|
157
|
+
return ret >>> 0;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
if (Symbol.dispose) Encoding.prototype[Symbol.dispose] = Encoding.prototype.free;
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* Model pricing and metadata.
|
|
164
|
+
*/
|
|
165
|
+
export class ModelInfo {
|
|
166
|
+
static __wrap(ptr) {
|
|
167
|
+
ptr = ptr >>> 0;
|
|
168
|
+
const obj = Object.create(ModelInfo.prototype);
|
|
169
|
+
obj.__wbg_ptr = ptr;
|
|
170
|
+
ModelInfoFinalization.register(obj, obj.__wbg_ptr, obj);
|
|
171
|
+
return obj;
|
|
172
|
+
}
|
|
173
|
+
__destroy_into_raw() {
|
|
174
|
+
const ptr = this.__wbg_ptr;
|
|
175
|
+
this.__wbg_ptr = 0;
|
|
176
|
+
ModelInfoFinalization.unregister(this);
|
|
177
|
+
return ptr;
|
|
178
|
+
}
|
|
179
|
+
free() {
|
|
180
|
+
const ptr = this.__destroy_into_raw();
|
|
181
|
+
wasm.__wbg_modelinfo_free(ptr, 0);
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* @returns {number | undefined}
|
|
185
|
+
*/
|
|
186
|
+
get cachedInputPer1m() {
|
|
187
|
+
try {
|
|
188
|
+
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
|
|
189
|
+
wasm.modelinfo_cachedInputPer1m(retptr, this.__wbg_ptr);
|
|
190
|
+
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
|
|
191
|
+
var r2 = getDataViewMemory0().getFloat64(retptr + 8 * 1, true);
|
|
192
|
+
return r0 === 0 ? undefined : r2;
|
|
193
|
+
} finally {
|
|
194
|
+
wasm.__wbindgen_add_to_stack_pointer(16);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* @returns {number}
|
|
199
|
+
*/
|
|
200
|
+
get contextWindow() {
|
|
201
|
+
const ret = wasm.modelinfo_contextWindow(this.__wbg_ptr);
|
|
202
|
+
return ret >>> 0;
|
|
203
|
+
}
|
|
204
|
+
/**
|
|
205
|
+
* @returns {string}
|
|
206
|
+
*/
|
|
207
|
+
get id() {
|
|
208
|
+
let deferred1_0;
|
|
209
|
+
let deferred1_1;
|
|
210
|
+
try {
|
|
211
|
+
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
|
|
212
|
+
wasm.modelinfo_id(retptr, this.__wbg_ptr);
|
|
213
|
+
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
|
|
214
|
+
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
|
|
215
|
+
deferred1_0 = r0;
|
|
216
|
+
deferred1_1 = r1;
|
|
217
|
+
return getStringFromWasm0(r0, r1);
|
|
218
|
+
} finally {
|
|
219
|
+
wasm.__wbindgen_add_to_stack_pointer(16);
|
|
220
|
+
wasm.__wbindgen_export(deferred1_0, deferred1_1, 1);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
/**
|
|
224
|
+
* @returns {number}
|
|
225
|
+
*/
|
|
226
|
+
get inputPer1m() {
|
|
227
|
+
const ret = wasm.modelinfo_inputPer1m(this.__wbg_ptr);
|
|
228
|
+
return ret;
|
|
229
|
+
}
|
|
230
|
+
/**
|
|
231
|
+
* @returns {number}
|
|
232
|
+
*/
|
|
233
|
+
get maxOutput() {
|
|
234
|
+
const ret = wasm.modelinfo_maxOutput(this.__wbg_ptr);
|
|
235
|
+
return ret >>> 0;
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* @returns {number}
|
|
239
|
+
*/
|
|
240
|
+
get outputPer1m() {
|
|
241
|
+
const ret = wasm.modelinfo_outputPer1m(this.__wbg_ptr);
|
|
242
|
+
return ret;
|
|
243
|
+
}
|
|
244
|
+
/**
|
|
245
|
+
* @returns {string}
|
|
246
|
+
*/
|
|
247
|
+
get provider() {
|
|
248
|
+
let deferred1_0;
|
|
249
|
+
let deferred1_1;
|
|
250
|
+
try {
|
|
251
|
+
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
|
|
252
|
+
wasm.modelinfo_provider(retptr, this.__wbg_ptr);
|
|
253
|
+
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
|
|
254
|
+
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
|
|
255
|
+
deferred1_0 = r0;
|
|
256
|
+
deferred1_1 = r1;
|
|
257
|
+
return getStringFromWasm0(r0, r1);
|
|
258
|
+
} finally {
|
|
259
|
+
wasm.__wbindgen_add_to_stack_pointer(16);
|
|
260
|
+
wasm.__wbindgen_export(deferred1_0, deferred1_1, 1);
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
if (Symbol.dispose) ModelInfo.prototype[Symbol.dispose] = ModelInfo.prototype.free;
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* List all supported models with pricing info.
|
|
268
|
+
*
|
|
269
|
+
* Returns an array of `ModelInfo` objects.
|
|
270
|
+
* @returns {ModelInfo[]}
|
|
271
|
+
*/
|
|
272
|
+
export function allModels() {
|
|
273
|
+
try {
|
|
274
|
+
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
|
|
275
|
+
wasm.allModels(retptr);
|
|
276
|
+
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
|
|
277
|
+
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
|
|
278
|
+
var v1 = getArrayJsValueFromWasm0(r0, r1).slice();
|
|
279
|
+
wasm.__wbindgen_export(r0, r1 * 4, 4);
|
|
280
|
+
return v1;
|
|
281
|
+
} finally {
|
|
282
|
+
wasm.__wbindgen_add_to_stack_pointer(16);
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/**
|
|
287
|
+
* Get an encoding for a model name (e.g. `"gpt-4o"`, `"o3-mini"`, `"llama-4"`, `"deepseek-r1"`).
|
|
288
|
+
*
|
|
289
|
+
* Supports models from OpenAI, Meta, DeepSeek, Qwen, and Mistral.
|
|
290
|
+
* Automatically resolves the model name to the correct encoding.
|
|
291
|
+
* Throws `Error` for unknown model names.
|
|
292
|
+
* @param {string} model
|
|
293
|
+
* @returns {Encoding}
|
|
294
|
+
*/
|
|
295
|
+
export function encodingForModel(model) {
|
|
296
|
+
try {
|
|
297
|
+
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
|
|
298
|
+
const ptr0 = passStringToWasm0(model, wasm.__wbindgen_export2, wasm.__wbindgen_export3);
|
|
299
|
+
const len0 = WASM_VECTOR_LEN;
|
|
300
|
+
wasm.encodingForModel(retptr, ptr0, len0);
|
|
301
|
+
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
|
|
302
|
+
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
|
|
303
|
+
var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true);
|
|
304
|
+
if (r2) {
|
|
305
|
+
throw takeObject(r1);
|
|
306
|
+
}
|
|
307
|
+
return Encoding.__wrap(r0);
|
|
308
|
+
} finally {
|
|
309
|
+
wasm.__wbindgen_add_to_stack_pointer(16);
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
/**
|
|
314
|
+
* Estimate cost in USD for a given model, input token count, and output token count.
|
|
315
|
+
*
|
|
316
|
+
* Supports OpenAI, Anthropic Claude, Google Gemini, Meta Llama, DeepSeek, Qwen, and Mistral models.
|
|
317
|
+
* Throws `Error` for unknown model ids.
|
|
318
|
+
* @param {string} model_id
|
|
319
|
+
* @param {number} input_tokens
|
|
320
|
+
* @param {number} output_tokens
|
|
321
|
+
* @returns {number}
|
|
322
|
+
*/
|
|
323
|
+
export function estimateCost(model_id, input_tokens, output_tokens) {
|
|
324
|
+
try {
|
|
325
|
+
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
|
|
326
|
+
const ptr0 = passStringToWasm0(model_id, wasm.__wbindgen_export2, wasm.__wbindgen_export3);
|
|
327
|
+
const len0 = WASM_VECTOR_LEN;
|
|
328
|
+
wasm.estimateCost(retptr, ptr0, len0, input_tokens, output_tokens);
|
|
329
|
+
var r0 = getDataViewMemory0().getFloat64(retptr + 8 * 0, true);
|
|
330
|
+
var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true);
|
|
331
|
+
var r3 = getDataViewMemory0().getInt32(retptr + 4 * 3, true);
|
|
332
|
+
if (r3) {
|
|
333
|
+
throw takeObject(r2);
|
|
334
|
+
}
|
|
335
|
+
return r0;
|
|
336
|
+
} finally {
|
|
337
|
+
wasm.__wbindgen_add_to_stack_pointer(16);
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
/**
|
|
342
|
+
* Get an encoding by name.
|
|
343
|
+
*
|
|
344
|
+
* Supported encodings:
|
|
345
|
+
* - `"cl100k_base"` — GPT-4, GPT-3.5-turbo
|
|
346
|
+
* - `"o200k_base"` — GPT-4o, GPT-4.1, o1, o3
|
|
347
|
+
* - `"p50k_base"` — text-davinci-002/003
|
|
348
|
+
* - `"p50k_edit"` — text-davinci-edit
|
|
349
|
+
* - `"r50k_base"` — GPT-3 (davinci, curie, etc.)
|
|
350
|
+
* - `"llama3"` — Meta Llama 3/4
|
|
351
|
+
* - `"deepseek_v3"` — DeepSeek V3/R1
|
|
352
|
+
* - `"qwen2"` — Qwen 2/2.5/3
|
|
353
|
+
* - `"mistral_v3"` — Mistral/Codestral/Pixtral
|
|
354
|
+
*
|
|
355
|
+
* Throws `Error` for unknown encoding names.
|
|
356
|
+
* @param {string} name
|
|
357
|
+
* @returns {Encoding}
|
|
358
|
+
*/
|
|
359
|
+
export function getEncoding(name) {
|
|
360
|
+
try {
|
|
361
|
+
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
|
|
362
|
+
const ptr0 = passStringToWasm0(name, wasm.__wbindgen_export2, wasm.__wbindgen_export3);
|
|
363
|
+
const len0 = WASM_VECTOR_LEN;
|
|
364
|
+
wasm.getEncoding(retptr, ptr0, len0);
|
|
365
|
+
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
|
|
366
|
+
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
|
|
367
|
+
var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true);
|
|
368
|
+
if (r2) {
|
|
369
|
+
throw takeObject(r1);
|
|
370
|
+
}
|
|
371
|
+
return Encoding.__wrap(r0);
|
|
372
|
+
} finally {
|
|
373
|
+
wasm.__wbindgen_add_to_stack_pointer(16);
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
/**
|
|
378
|
+
* Get model pricing and metadata.
|
|
379
|
+
*
|
|
380
|
+
* Returns a typed object with: `id`, `provider`, `inputPer1m`, `outputPer1m`,
|
|
381
|
+
* `cachedInputPer1m`, `contextWindow`, `maxOutput`.
|
|
382
|
+
*
|
|
383
|
+
* Throws `Error` for unknown model ids.
|
|
384
|
+
* @param {string} model_id
|
|
385
|
+
* @returns {ModelInfo}
|
|
386
|
+
*/
|
|
387
|
+
export function getModelInfo(model_id) {
|
|
388
|
+
try {
|
|
389
|
+
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
|
|
390
|
+
const ptr0 = passStringToWasm0(model_id, wasm.__wbindgen_export2, wasm.__wbindgen_export3);
|
|
391
|
+
const len0 = WASM_VECTOR_LEN;
|
|
392
|
+
wasm.getModelInfo(retptr, ptr0, len0);
|
|
393
|
+
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
|
|
394
|
+
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
|
|
395
|
+
var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true);
|
|
396
|
+
if (r2) {
|
|
397
|
+
throw takeObject(r1);
|
|
398
|
+
}
|
|
399
|
+
return ModelInfo.__wrap(r0);
|
|
400
|
+
} finally {
|
|
401
|
+
wasm.__wbindgen_add_to_stack_pointer(16);
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
/**
|
|
406
|
+
* List all available encoding names.
|
|
407
|
+
*
|
|
408
|
+
* Returns an array of strings: `["cl100k_base", "o200k_base", ...]`
|
|
409
|
+
* @returns {string[]}
|
|
410
|
+
*/
|
|
411
|
+
export function listEncodings() {
|
|
412
|
+
try {
|
|
413
|
+
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
|
|
414
|
+
wasm.listEncodings(retptr);
|
|
415
|
+
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
|
|
416
|
+
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
|
|
417
|
+
var v1 = getArrayJsValueFromWasm0(r0, r1).slice();
|
|
418
|
+
wasm.__wbindgen_export(r0, r1 * 4, 4);
|
|
419
|
+
return v1;
|
|
420
|
+
} finally {
|
|
421
|
+
wasm.__wbindgen_add_to_stack_pointer(16);
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
/**
|
|
426
|
+
* Map a model name to its encoding name without loading the encoding.
|
|
427
|
+
*
|
|
428
|
+
* Returns the encoding name string (e.g. `"o200k_base"`) or `null` for unknown models.
|
|
429
|
+
* @param {string} model
|
|
430
|
+
* @returns {string | undefined}
|
|
431
|
+
*/
|
|
432
|
+
export function modelToEncoding(model) {
|
|
433
|
+
try {
|
|
434
|
+
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
|
|
435
|
+
const ptr0 = passStringToWasm0(model, wasm.__wbindgen_export2, wasm.__wbindgen_export3);
|
|
436
|
+
const len0 = WASM_VECTOR_LEN;
|
|
437
|
+
wasm.modelToEncoding(retptr, ptr0, len0);
|
|
438
|
+
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
|
|
439
|
+
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
|
|
440
|
+
let v2;
|
|
441
|
+
if (r0 !== 0) {
|
|
442
|
+
v2 = getStringFromWasm0(r0, r1).slice();
|
|
443
|
+
wasm.__wbindgen_export(r0, r1 * 1, 1);
|
|
444
|
+
}
|
|
445
|
+
return v2;
|
|
446
|
+
} finally {
|
|
447
|
+
wasm.__wbindgen_add_to_stack_pointer(16);
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
/**
|
|
452
|
+
* List models filtered by provider name.
|
|
453
|
+
*
|
|
454
|
+
* Provider names: `"OpenAI"`, `"Anthropic"`, `"Google"`, `"Meta"`, `"DeepSeek"`, `"Alibaba"`, `"Mistral"`.
|
|
455
|
+
* Returns an empty array for unknown providers.
|
|
456
|
+
* @param {string} provider
|
|
457
|
+
* @returns {ModelInfo[]}
|
|
458
|
+
*/
|
|
459
|
+
export function modelsByProvider(provider) {
|
|
460
|
+
try {
|
|
461
|
+
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
|
|
462
|
+
const ptr0 = passStringToWasm0(provider, wasm.__wbindgen_export2, wasm.__wbindgen_export3);
|
|
463
|
+
const len0 = WASM_VECTOR_LEN;
|
|
464
|
+
wasm.modelsByProvider(retptr, ptr0, len0);
|
|
465
|
+
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
|
|
466
|
+
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
|
|
467
|
+
var v2 = getArrayJsValueFromWasm0(r0, r1).slice();
|
|
468
|
+
wasm.__wbindgen_export(r0, r1 * 4, 4);
|
|
469
|
+
return v2;
|
|
470
|
+
} finally {
|
|
471
|
+
wasm.__wbindgen_add_to_stack_pointer(16);
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
export function __wbg_Error_83742b46f01ce22d(arg0, arg1) {
|
|
475
|
+
const ret = Error(getStringFromWasm0(arg0, arg1));
|
|
476
|
+
return addHeapObject(ret);
|
|
477
|
+
}
|
|
478
|
+
export function __wbg___wbindgen_throw_6ddd609b62940d55(arg0, arg1) {
|
|
479
|
+
throw new Error(getStringFromWasm0(arg0, arg1));
|
|
480
|
+
}
|
|
481
|
+
export function __wbg_modelinfo_new(arg0) {
|
|
482
|
+
const ret = ModelInfo.__wrap(arg0);
|
|
483
|
+
return addHeapObject(ret);
|
|
484
|
+
}
|
|
485
|
+
export function __wbindgen_cast_0000000000000001(arg0, arg1) {
|
|
486
|
+
// Cast intrinsic for `Ref(String) -> Externref`.
|
|
487
|
+
const ret = getStringFromWasm0(arg0, arg1);
|
|
488
|
+
return addHeapObject(ret);
|
|
489
|
+
}
|
|
490
|
+
const EncodingFinalization = (typeof FinalizationRegistry === 'undefined')
|
|
491
|
+
? { register: () => {}, unregister: () => {} }
|
|
492
|
+
: new FinalizationRegistry(ptr => wasm.__wbg_encoding_free(ptr >>> 0, 1));
|
|
493
|
+
const ModelInfoFinalization = (typeof FinalizationRegistry === 'undefined')
|
|
494
|
+
? { register: () => {}, unregister: () => {} }
|
|
495
|
+
: new FinalizationRegistry(ptr => wasm.__wbg_modelinfo_free(ptr >>> 0, 1));
|
|
496
|
+
|
|
497
|
+
function addHeapObject(obj) {
|
|
498
|
+
if (heap_next === heap.length) heap.push(heap.length + 1);
|
|
499
|
+
const idx = heap_next;
|
|
500
|
+
heap_next = heap[idx];
|
|
501
|
+
|
|
502
|
+
heap[idx] = obj;
|
|
503
|
+
return idx;
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
function dropObject(idx) {
|
|
507
|
+
if (idx < 1028) return;
|
|
508
|
+
heap[idx] = heap_next;
|
|
509
|
+
heap_next = idx;
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
function getArrayJsValueFromWasm0(ptr, len) {
|
|
513
|
+
ptr = ptr >>> 0;
|
|
514
|
+
const mem = getDataViewMemory0();
|
|
515
|
+
const result = [];
|
|
516
|
+
for (let i = ptr; i < ptr + 4 * len; i += 4) {
|
|
517
|
+
result.push(takeObject(mem.getUint32(i, true)));
|
|
518
|
+
}
|
|
519
|
+
return result;
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
function getArrayU32FromWasm0(ptr, len) {
|
|
523
|
+
ptr = ptr >>> 0;
|
|
524
|
+
return getUint32ArrayMemory0().subarray(ptr / 4, ptr / 4 + len);
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
let cachedDataViewMemory0 = null;
|
|
528
|
+
function getDataViewMemory0() {
|
|
529
|
+
if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) {
|
|
530
|
+
cachedDataViewMemory0 = new DataView(wasm.memory.buffer);
|
|
531
|
+
}
|
|
532
|
+
return cachedDataViewMemory0;
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
function getStringFromWasm0(ptr, len) {
|
|
536
|
+
ptr = ptr >>> 0;
|
|
537
|
+
return decodeText(ptr, len);
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
let cachedUint32ArrayMemory0 = null;
|
|
541
|
+
function getUint32ArrayMemory0() {
|
|
542
|
+
if (cachedUint32ArrayMemory0 === null || cachedUint32ArrayMemory0.byteLength === 0) {
|
|
543
|
+
cachedUint32ArrayMemory0 = new Uint32Array(wasm.memory.buffer);
|
|
544
|
+
}
|
|
545
|
+
return cachedUint32ArrayMemory0;
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
let cachedUint8ArrayMemory0 = null;
|
|
549
|
+
function getUint8ArrayMemory0() {
|
|
550
|
+
if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) {
|
|
551
|
+
cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer);
|
|
552
|
+
}
|
|
553
|
+
return cachedUint8ArrayMemory0;
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
function getObject(idx) { return heap[idx]; }
|
|
557
|
+
|
|
558
|
+
let heap = new Array(1024).fill(undefined);
|
|
559
|
+
heap.push(undefined, null, true, false);
|
|
560
|
+
|
|
561
|
+
let heap_next = heap.length;
|
|
562
|
+
|
|
563
|
+
function passArray32ToWasm0(arg, malloc) {
|
|
564
|
+
const ptr = malloc(arg.length * 4, 4) >>> 0;
|
|
565
|
+
getUint32ArrayMemory0().set(arg, ptr / 4);
|
|
566
|
+
WASM_VECTOR_LEN = arg.length;
|
|
567
|
+
return ptr;
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
function passStringToWasm0(arg, malloc, realloc) {
|
|
571
|
+
if (realloc === undefined) {
|
|
572
|
+
const buf = cachedTextEncoder.encode(arg);
|
|
573
|
+
const ptr = malloc(buf.length, 1) >>> 0;
|
|
574
|
+
getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf);
|
|
575
|
+
WASM_VECTOR_LEN = buf.length;
|
|
576
|
+
return ptr;
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
let len = arg.length;
|
|
580
|
+
let ptr = malloc(len, 1) >>> 0;
|
|
581
|
+
|
|
582
|
+
const mem = getUint8ArrayMemory0();
|
|
583
|
+
|
|
584
|
+
let offset = 0;
|
|
585
|
+
|
|
586
|
+
for (; offset < len; offset++) {
|
|
587
|
+
const code = arg.charCodeAt(offset);
|
|
588
|
+
if (code > 0x7F) break;
|
|
589
|
+
mem[ptr + offset] = code;
|
|
590
|
+
}
|
|
591
|
+
if (offset !== len) {
|
|
592
|
+
if (offset !== 0) {
|
|
593
|
+
arg = arg.slice(offset);
|
|
594
|
+
}
|
|
595
|
+
ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0;
|
|
596
|
+
const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len);
|
|
597
|
+
const ret = cachedTextEncoder.encodeInto(arg, view);
|
|
598
|
+
|
|
599
|
+
offset += ret.written;
|
|
600
|
+
ptr = realloc(ptr, len, offset, 1) >>> 0;
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
WASM_VECTOR_LEN = offset;
|
|
604
|
+
return ptr;
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
function takeObject(idx) {
|
|
608
|
+
const ret = getObject(idx);
|
|
609
|
+
dropObject(idx);
|
|
610
|
+
return ret;
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
|
|
614
|
+
cachedTextDecoder.decode();
|
|
615
|
+
const MAX_SAFARI_DECODE_BYTES = 2146435072;
|
|
616
|
+
let numBytesDecoded = 0;
|
|
617
|
+
function decodeText(ptr, len) {
|
|
618
|
+
numBytesDecoded += len;
|
|
619
|
+
if (numBytesDecoded >= MAX_SAFARI_DECODE_BYTES) {
|
|
620
|
+
cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
|
|
621
|
+
cachedTextDecoder.decode();
|
|
622
|
+
numBytesDecoded = len;
|
|
623
|
+
}
|
|
624
|
+
return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len));
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
const cachedTextEncoder = new TextEncoder();
|
|
628
|
+
|
|
629
|
+
if (!('encodeInto' in cachedTextEncoder)) {
|
|
630
|
+
cachedTextEncoder.encodeInto = function (arg, view) {
|
|
631
|
+
const buf = cachedTextEncoder.encode(arg);
|
|
632
|
+
view.set(buf);
|
|
633
|
+
return {
|
|
634
|
+
read: arg.length,
|
|
635
|
+
written: buf.length
|
|
636
|
+
};
|
|
637
|
+
};
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
let WASM_VECTOR_LEN = 0;
|
|
641
|
+
|
|
642
|
+
|
|
643
|
+
let wasm;
|
|
644
|
+
export function __wbg_set_wasm(val) {
|
|
645
|
+
wasm = val;
|
|
646
|
+
}
|
package/tiktoken_wasm_bg.wasm
CHANGED
|
Binary file
|