@usex/ternava 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/package.json +22 -5
- package/pkg-bundler/tern_engine_bg.wasm +0 -0
- package/pkg-node/tern_engine_bg.wasm +0 -0
- package/pkg-web/tern_engine_bg.wasm +0 -0
- package/scripts/check-package.mjs +7 -0
- package/pkg-node/tern_engine_bg.js +0 -320
- package/pkg-web/tern_engine_bg.js +0 -320
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,21 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.3.0](https://github.com/ali-master/ternava/compare/v0.2.0...v0.3.0) (2026-11-19)
|
|
4
|
+
|
|
5
|
+
### Features
|
|
6
|
+
|
|
7
|
+
* **agents:** add huggingface skills ([2100bc3](https://github.com/ali-master/ternava/commit/2100bc33665561dc6feb5b114b397fd4a69c722c))
|
|
8
|
+
* **agents:** add huggingface skills ([2f769b9](https://github.com/ali-master/ternava/commit/2f769b9b52eb738aaf251151c2b6d68fedc3255d))
|
|
9
|
+
* **eval:** add tool-search benchmark and checkpoint scoring ([cd94b3c](https://github.com/ali-master/ternava/commit/cd94b3cedf5bad4f6f957b6eb8f4bb441f9f6025))
|
|
10
|
+
* **training:** add v2 pilot config and cache-free domain round ([cd7d1a0](https://github.com/ali-master/ternava/commit/cd7d1a0a7473db9fcfce6a1f826c10012ae3c207))
|
|
11
|
+
|
|
12
|
+
### Bug Fixes
|
|
13
|
+
|
|
14
|
+
* **docs-search:** load the published bilingual model ([f09d670](https://github.com/ali-master/ternava/commit/f09d670f7500731729cf4f5d06c986e7ee854a92))
|
|
15
|
+
* **emoji-search:** rebuild vectors with 0.2.0 ([dd634a5](https://github.com/ali-master/ternava/commit/dd634a50fced1fdff673c596b55baf7d5b3d08d1))
|
|
16
|
+
* **tests:** unblock the branding check ([40e5038](https://github.com/ali-master/ternava/commit/40e5038cc3b041245b3867ba71dfa6d3ac914a4c))
|
|
17
|
+
* **tokenizer:** map Persian and Arabic-Indic digits to ASCII ([fe58b3b](https://github.com/ali-master/ternava/commit/fe58b3bec33f39bece0c2692e7e0cd989e8c30fb))
|
|
18
|
+
|
|
3
19
|
## [0.2.0](https://github.com/ali-master/ternava/compare/v0.1.1...v0.2.0) (2026-10-13)
|
|
4
20
|
|
|
5
21
|
### Features
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@usex/ternava",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "Ternava — experimental Persian/English semantic embeddings, 32k bilingual tokenizer and local WebAssembly inference for Node.js, Bun and browsers.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -59,10 +59,27 @@
|
|
|
59
59
|
"./package.json": "./package.json"
|
|
60
60
|
},
|
|
61
61
|
"files": [
|
|
62
|
-
"src/",
|
|
63
|
-
"
|
|
64
|
-
"
|
|
65
|
-
"
|
|
62
|
+
"src/index.js",
|
|
63
|
+
"src/index.mjs",
|
|
64
|
+
"src/index.browser.mjs",
|
|
65
|
+
"src/index.d.ts",
|
|
66
|
+
"pkg-node/tern_engine.js",
|
|
67
|
+
"pkg-node/tern_engine.d.ts",
|
|
68
|
+
"pkg-node/tern_engine_bg.wasm",
|
|
69
|
+
"pkg-node/tern_engine_bg.wasm.d.ts",
|
|
70
|
+
"pkg-bundler/package.json",
|
|
71
|
+
"pkg-bundler/tern_engine.js",
|
|
72
|
+
"pkg-bundler/tern_engine.d.ts",
|
|
73
|
+
"pkg-bundler/tern_engine_bg.js",
|
|
74
|
+
"pkg-bundler/tern_engine_bg.wasm",
|
|
75
|
+
"pkg-bundler/tern_engine_bg.wasm.d.ts",
|
|
76
|
+
"pkg-web/package.json",
|
|
77
|
+
"pkg-web/index.mjs",
|
|
78
|
+
"pkg-web/index.d.mts",
|
|
79
|
+
"pkg-web/tern_engine.js",
|
|
80
|
+
"pkg-web/tern_engine.d.ts",
|
|
81
|
+
"pkg-web/tern_engine_bg.wasm",
|
|
82
|
+
"pkg-web/tern_engine_bg.wasm.d.ts",
|
|
66
83
|
"README.md",
|
|
67
84
|
"CHANGELOG.md",
|
|
68
85
|
"LICENSE",
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -18,6 +18,13 @@ function checkExports(value) {
|
|
|
18
18
|
}
|
|
19
19
|
}
|
|
20
20
|
checkExports(manifest.exports);
|
|
21
|
+
let unpackedBytes = statSync(new URL('package.json', root)).size;
|
|
22
|
+
for (const path of manifest.files) {
|
|
23
|
+
const file = statSync(new URL(path, root));
|
|
24
|
+
assert.ok(file.isFile(), `Publish allowlist must contain individual files: ${path}`);
|
|
25
|
+
unpackedBytes += file.size;
|
|
26
|
+
}
|
|
27
|
+
assert.ok(unpackedBytes <= 25_000_000, `Package exceeds 25 MB unpacked budget: ${unpackedBytes} bytes`);
|
|
21
28
|
for (const target of ['node', 'bundler', 'web']) {
|
|
22
29
|
const bytes = readFileSync(new URL(`pkg-${target}/tern_engine_bg.wasm`, root));
|
|
23
30
|
assert.ok(WebAssembly.validate(bytes), `Invalid ${target} WASM`);
|
|
@@ -1,320 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Debug helper: returns a human-readable summary of the loaded model's
|
|
3
|
-
* header (format version, embedding format, dimensions, vocab size).
|
|
4
|
-
* @returns {string}
|
|
5
|
-
*/
|
|
6
|
-
export function config_summary() {
|
|
7
|
-
let deferred1_0;
|
|
8
|
-
let deferred1_1;
|
|
9
|
-
try {
|
|
10
|
-
const ret = wasm.config_summary();
|
|
11
|
-
deferred1_0 = ret[0];
|
|
12
|
-
deferred1_1 = ret[1];
|
|
13
|
-
return getStringFromWasm0(ret[0], ret[1]);
|
|
14
|
-
} finally {
|
|
15
|
-
wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
|
|
16
|
-
}
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
/**
|
|
20
|
-
* Primary entry point: text → 384-dim L2-normalized embedding.
|
|
21
|
-
*
|
|
22
|
-
* Tokenizes via the embedded BERT vocab, runs the full forward pass against
|
|
23
|
-
* the embedded `.bin`, returns the output vector.
|
|
24
|
-
* @param {string} text
|
|
25
|
-
* @returns {Float32Array}
|
|
26
|
-
*/
|
|
27
|
-
export function embed(text) {
|
|
28
|
-
const ptr0 = passStringToWasm0(text, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
|
29
|
-
const len0 = WASM_VECTOR_LEN;
|
|
30
|
-
const ret = wasm.embed(ptr0, len0);
|
|
31
|
-
var v2 = getArrayF32FromWasm0(ret[0], ret[1]).slice();
|
|
32
|
-
wasm.__wbindgen_free(ret[0], ret[1] * 4, 4);
|
|
33
|
-
return v2;
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
/**
|
|
37
|
-
* Identity of the exact packed model embedded in this build.
|
|
38
|
-
* @returns {string}
|
|
39
|
-
*/
|
|
40
|
-
export function model_sha256() {
|
|
41
|
-
let deferred1_0;
|
|
42
|
-
let deferred1_1;
|
|
43
|
-
try {
|
|
44
|
-
const ret = wasm.model_sha256();
|
|
45
|
-
deferred1_0 = ret[0];
|
|
46
|
-
deferred1_1 = ret[1];
|
|
47
|
-
return getStringFromWasm0(ret[0], ret[1]);
|
|
48
|
-
} finally {
|
|
49
|
-
wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
/**
|
|
54
|
-
* Debug helper: returns token IDs the tokenizer produced. Used by per-stage
|
|
55
|
-
* parity tests to confirm tokenization matches the Python `tokenizers` library.
|
|
56
|
-
* @param {string} text
|
|
57
|
-
* @returns {Uint32Array}
|
|
58
|
-
*/
|
|
59
|
-
export function tokenize(text) {
|
|
60
|
-
const ptr0 = passStringToWasm0(text, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
|
61
|
-
const len0 = WASM_VECTOR_LEN;
|
|
62
|
-
const ret = wasm.tokenize(ptr0, len0);
|
|
63
|
-
var v2 = getArrayU32FromWasm0(ret[0], ret[1]).slice();
|
|
64
|
-
wasm.__wbindgen_free(ret[0], ret[1] * 4, 4);
|
|
65
|
-
return v2;
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
/**
|
|
69
|
-
* Identity of the exact tokenizer embedded in this build.
|
|
70
|
-
* @returns {string}
|
|
71
|
-
*/
|
|
72
|
-
export function tokenizer_sha256() {
|
|
73
|
-
let deferred1_0;
|
|
74
|
-
let deferred1_1;
|
|
75
|
-
try {
|
|
76
|
-
const ret = wasm.tokenizer_sha256();
|
|
77
|
-
deferred1_0 = ret[0];
|
|
78
|
-
deferred1_1 = ret[1];
|
|
79
|
-
return getStringFromWasm0(ret[0], ret[1]);
|
|
80
|
-
} finally {
|
|
81
|
-
wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
export function __wbg___wbindgen_is_function_754e9f305ff6029e(arg0) {
|
|
85
|
-
const ret = typeof(arg0) === 'function';
|
|
86
|
-
return ret;
|
|
87
|
-
}
|
|
88
|
-
export function __wbg___wbindgen_is_object_56732c2bc353f41d(arg0) {
|
|
89
|
-
const val = arg0;
|
|
90
|
-
const ret = typeof(val) === 'object' && val !== null;
|
|
91
|
-
return ret;
|
|
92
|
-
}
|
|
93
|
-
export function __wbg___wbindgen_is_string_c236cabd84a4d769(arg0) {
|
|
94
|
-
const ret = typeof(arg0) === 'string';
|
|
95
|
-
return ret;
|
|
96
|
-
}
|
|
97
|
-
export function __wbg___wbindgen_is_undefined_67b456be8673d3d7(arg0) {
|
|
98
|
-
const ret = arg0 === undefined;
|
|
99
|
-
return ret;
|
|
100
|
-
}
|
|
101
|
-
export function __wbg___wbindgen_throw_1506f2235d1bdba0(arg0, arg1) {
|
|
102
|
-
throw new Error(getStringFromWasm0(arg0, arg1));
|
|
103
|
-
}
|
|
104
|
-
export function __wbg_call_9c758de292015997() { return handleError(function (arg0, arg1, arg2) {
|
|
105
|
-
const ret = arg0.call(arg1, arg2);
|
|
106
|
-
return ret;
|
|
107
|
-
}, arguments); }
|
|
108
|
-
export function __wbg_crypto_38df2bab126b63dc(arg0) {
|
|
109
|
-
const ret = arg0.crypto;
|
|
110
|
-
return ret;
|
|
111
|
-
}
|
|
112
|
-
export function __wbg_getRandomValues_c44a50d8cfdaebeb() { return handleError(function (arg0, arg1) {
|
|
113
|
-
arg0.getRandomValues(arg1);
|
|
114
|
-
}, arguments); }
|
|
115
|
-
export function __wbg_length_4a591ecaa01354d9(arg0) {
|
|
116
|
-
const ret = arg0.length;
|
|
117
|
-
return ret;
|
|
118
|
-
}
|
|
119
|
-
export function __wbg_msCrypto_bd5a034af96bcba6(arg0) {
|
|
120
|
-
const ret = arg0.msCrypto;
|
|
121
|
-
return ret;
|
|
122
|
-
}
|
|
123
|
-
export function __wbg_new_with_length_36a4998e27b014c5(arg0) {
|
|
124
|
-
const ret = new Uint8Array(arg0 >>> 0);
|
|
125
|
-
return ret;
|
|
126
|
-
}
|
|
127
|
-
export function __wbg_node_84ea875411254db1(arg0) {
|
|
128
|
-
const ret = arg0.node;
|
|
129
|
-
return ret;
|
|
130
|
-
}
|
|
131
|
-
export function __wbg_process_44c7a14e11e9f69e(arg0) {
|
|
132
|
-
const ret = arg0.process;
|
|
133
|
-
return ret;
|
|
134
|
-
}
|
|
135
|
-
export function __wbg_prototypesetcall_3249fc62a0fafa30(arg0, arg1, arg2) {
|
|
136
|
-
Uint8Array.prototype.set.call(getArrayU8FromWasm0(arg0, arg1), arg2);
|
|
137
|
-
}
|
|
138
|
-
export function __wbg_randomFillSync_6c25eac9869eb53c() { return handleError(function (arg0, arg1) {
|
|
139
|
-
arg0.randomFillSync(arg1);
|
|
140
|
-
}, arguments); }
|
|
141
|
-
export function __wbg_require_b4edbdcf3e2a1ef0() { return handleError(function () {
|
|
142
|
-
const ret = module.require;
|
|
143
|
-
return ret;
|
|
144
|
-
}, arguments); }
|
|
145
|
-
export function __wbg_static_accessor_GLOBAL_9d53f2689e622ca1() {
|
|
146
|
-
const ret = typeof global === 'undefined' ? null : global;
|
|
147
|
-
return isLikeNone(ret) ? 0 : addToExternrefTable0(ret);
|
|
148
|
-
}
|
|
149
|
-
export function __wbg_static_accessor_GLOBAL_THIS_a1a35cec07001a8a() {
|
|
150
|
-
const ret = typeof globalThis === 'undefined' ? null : globalThis;
|
|
151
|
-
return isLikeNone(ret) ? 0 : addToExternrefTable0(ret);
|
|
152
|
-
}
|
|
153
|
-
export function __wbg_static_accessor_SELF_4c59f6c7ea29a144() {
|
|
154
|
-
const ret = typeof self === 'undefined' ? null : self;
|
|
155
|
-
return isLikeNone(ret) ? 0 : addToExternrefTable0(ret);
|
|
156
|
-
}
|
|
157
|
-
export function __wbg_static_accessor_WINDOW_e70ae9f2eb052253() {
|
|
158
|
-
const ret = typeof window === 'undefined' ? null : window;
|
|
159
|
-
return isLikeNone(ret) ? 0 : addToExternrefTable0(ret);
|
|
160
|
-
}
|
|
161
|
-
export function __wbg_subarray_4aa221f6a4f5ab22(arg0, arg1, arg2) {
|
|
162
|
-
const ret = arg0.subarray(arg1 >>> 0, arg2 >>> 0);
|
|
163
|
-
return ret;
|
|
164
|
-
}
|
|
165
|
-
export function __wbg_versions_276b2795b1c6a219(arg0) {
|
|
166
|
-
const ret = arg0.versions;
|
|
167
|
-
return ret;
|
|
168
|
-
}
|
|
169
|
-
export function __wbindgen_cast_0000000000000001(arg0, arg1) {
|
|
170
|
-
// Cast intrinsic for `Ref(Slice(U8)) -> NamedExternref("Uint8Array")`.
|
|
171
|
-
const ret = getArrayU8FromWasm0(arg0, arg1);
|
|
172
|
-
return ret;
|
|
173
|
-
}
|
|
174
|
-
export function __wbindgen_cast_0000000000000002(arg0, arg1) {
|
|
175
|
-
// Cast intrinsic for `Ref(String) -> Externref`.
|
|
176
|
-
const ret = getStringFromWasm0(arg0, arg1);
|
|
177
|
-
return ret;
|
|
178
|
-
}
|
|
179
|
-
export function __wbindgen_init_externref_table() {
|
|
180
|
-
const table = wasm.__wbindgen_externrefs;
|
|
181
|
-
const offset = table.grow(4);
|
|
182
|
-
table.set(0, undefined);
|
|
183
|
-
table.set(offset + 0, undefined);
|
|
184
|
-
table.set(offset + 1, null);
|
|
185
|
-
table.set(offset + 2, true);
|
|
186
|
-
table.set(offset + 3, false);
|
|
187
|
-
}
|
|
188
|
-
function addToExternrefTable0(obj) {
|
|
189
|
-
const idx = wasm.__externref_table_alloc();
|
|
190
|
-
wasm.__wbindgen_externrefs.set(idx, obj);
|
|
191
|
-
return idx;
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
function getArrayF32FromWasm0(ptr, len) {
|
|
195
|
-
ptr = ptr >>> 0;
|
|
196
|
-
return getFloat32ArrayMemory0().subarray(ptr / 4, ptr / 4 + len);
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
function getArrayU32FromWasm0(ptr, len) {
|
|
200
|
-
ptr = ptr >>> 0;
|
|
201
|
-
return getUint32ArrayMemory0().subarray(ptr / 4, ptr / 4 + len);
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
function getArrayU8FromWasm0(ptr, len) {
|
|
205
|
-
ptr = ptr >>> 0;
|
|
206
|
-
return getUint8ArrayMemory0().subarray(ptr / 1, ptr / 1 + len);
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
let cachedFloat32ArrayMemory0 = null;
|
|
210
|
-
function getFloat32ArrayMemory0() {
|
|
211
|
-
if (cachedFloat32ArrayMemory0 === null || cachedFloat32ArrayMemory0.byteLength === 0) {
|
|
212
|
-
cachedFloat32ArrayMemory0 = new Float32Array(wasm.memory.buffer);
|
|
213
|
-
}
|
|
214
|
-
return cachedFloat32ArrayMemory0;
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
function getStringFromWasm0(ptr, len) {
|
|
218
|
-
return decodeText(ptr >>> 0, len);
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
let cachedUint32ArrayMemory0 = null;
|
|
222
|
-
function getUint32ArrayMemory0() {
|
|
223
|
-
if (cachedUint32ArrayMemory0 === null || cachedUint32ArrayMemory0.byteLength === 0) {
|
|
224
|
-
cachedUint32ArrayMemory0 = new Uint32Array(wasm.memory.buffer);
|
|
225
|
-
}
|
|
226
|
-
return cachedUint32ArrayMemory0;
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
let cachedUint8ArrayMemory0 = null;
|
|
230
|
-
function getUint8ArrayMemory0() {
|
|
231
|
-
if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) {
|
|
232
|
-
cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer);
|
|
233
|
-
}
|
|
234
|
-
return cachedUint8ArrayMemory0;
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
function handleError(f, args) {
|
|
238
|
-
try {
|
|
239
|
-
return f.apply(this, args);
|
|
240
|
-
} catch (e) {
|
|
241
|
-
const idx = addToExternrefTable0(e);
|
|
242
|
-
wasm.__wbindgen_exn_store(idx);
|
|
243
|
-
}
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
function isLikeNone(x) {
|
|
247
|
-
return x === undefined || x === null;
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
function passStringToWasm0(arg, malloc, realloc) {
|
|
251
|
-
if (realloc === undefined) {
|
|
252
|
-
const buf = cachedTextEncoder.encode(arg);
|
|
253
|
-
const ptr = malloc(buf.length, 1) >>> 0;
|
|
254
|
-
getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf);
|
|
255
|
-
WASM_VECTOR_LEN = buf.length;
|
|
256
|
-
return ptr;
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
let len = arg.length;
|
|
260
|
-
let ptr = malloc(len, 1) >>> 0;
|
|
261
|
-
|
|
262
|
-
const mem = getUint8ArrayMemory0();
|
|
263
|
-
|
|
264
|
-
let offset = 0;
|
|
265
|
-
|
|
266
|
-
for (; offset < len; offset++) {
|
|
267
|
-
const code = arg.charCodeAt(offset);
|
|
268
|
-
if (code > 0x7F) break;
|
|
269
|
-
mem[ptr + offset] = code;
|
|
270
|
-
}
|
|
271
|
-
if (offset !== len) {
|
|
272
|
-
if (offset !== 0) {
|
|
273
|
-
arg = arg.slice(offset);
|
|
274
|
-
}
|
|
275
|
-
ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0;
|
|
276
|
-
const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len);
|
|
277
|
-
const ret = cachedTextEncoder.encodeInto(arg, view);
|
|
278
|
-
|
|
279
|
-
offset += ret.written;
|
|
280
|
-
ptr = realloc(ptr, len, offset, 1) >>> 0;
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
WASM_VECTOR_LEN = offset;
|
|
284
|
-
return ptr;
|
|
285
|
-
}
|
|
286
|
-
|
|
287
|
-
let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
|
|
288
|
-
cachedTextDecoder.decode();
|
|
289
|
-
const MAX_SAFARI_DECODE_BYTES = 2146435072;
|
|
290
|
-
let numBytesDecoded = 0;
|
|
291
|
-
function decodeText(ptr, len) {
|
|
292
|
-
numBytesDecoded += len;
|
|
293
|
-
if (numBytesDecoded >= MAX_SAFARI_DECODE_BYTES) {
|
|
294
|
-
cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
|
|
295
|
-
cachedTextDecoder.decode();
|
|
296
|
-
numBytesDecoded = len;
|
|
297
|
-
}
|
|
298
|
-
return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len));
|
|
299
|
-
}
|
|
300
|
-
|
|
301
|
-
const cachedTextEncoder = new TextEncoder();
|
|
302
|
-
|
|
303
|
-
if (!('encodeInto' in cachedTextEncoder)) {
|
|
304
|
-
cachedTextEncoder.encodeInto = function (arg, view) {
|
|
305
|
-
const buf = cachedTextEncoder.encode(arg);
|
|
306
|
-
view.set(buf);
|
|
307
|
-
return {
|
|
308
|
-
read: arg.length,
|
|
309
|
-
written: buf.length
|
|
310
|
-
};
|
|
311
|
-
};
|
|
312
|
-
}
|
|
313
|
-
|
|
314
|
-
let WASM_VECTOR_LEN = 0;
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
let wasm;
|
|
318
|
-
export function __wbg_set_wasm(val) {
|
|
319
|
-
wasm = val;
|
|
320
|
-
}
|
|
@@ -1,320 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Debug helper: returns a human-readable summary of the loaded model's
|
|
3
|
-
* header (format version, embedding format, dimensions, vocab size).
|
|
4
|
-
* @returns {string}
|
|
5
|
-
*/
|
|
6
|
-
export function config_summary() {
|
|
7
|
-
let deferred1_0;
|
|
8
|
-
let deferred1_1;
|
|
9
|
-
try {
|
|
10
|
-
const ret = wasm.config_summary();
|
|
11
|
-
deferred1_0 = ret[0];
|
|
12
|
-
deferred1_1 = ret[1];
|
|
13
|
-
return getStringFromWasm0(ret[0], ret[1]);
|
|
14
|
-
} finally {
|
|
15
|
-
wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
|
|
16
|
-
}
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
/**
|
|
20
|
-
* Primary entry point: text → 384-dim L2-normalized embedding.
|
|
21
|
-
*
|
|
22
|
-
* Tokenizes via the embedded BERT vocab, runs the full forward pass against
|
|
23
|
-
* the embedded `.bin`, returns the output vector.
|
|
24
|
-
* @param {string} text
|
|
25
|
-
* @returns {Float32Array}
|
|
26
|
-
*/
|
|
27
|
-
export function embed(text) {
|
|
28
|
-
const ptr0 = passStringToWasm0(text, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
|
29
|
-
const len0 = WASM_VECTOR_LEN;
|
|
30
|
-
const ret = wasm.embed(ptr0, len0);
|
|
31
|
-
var v2 = getArrayF32FromWasm0(ret[0], ret[1]).slice();
|
|
32
|
-
wasm.__wbindgen_free(ret[0], ret[1] * 4, 4);
|
|
33
|
-
return v2;
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
/**
|
|
37
|
-
* Identity of the exact packed model embedded in this build.
|
|
38
|
-
* @returns {string}
|
|
39
|
-
*/
|
|
40
|
-
export function model_sha256() {
|
|
41
|
-
let deferred1_0;
|
|
42
|
-
let deferred1_1;
|
|
43
|
-
try {
|
|
44
|
-
const ret = wasm.model_sha256();
|
|
45
|
-
deferred1_0 = ret[0];
|
|
46
|
-
deferred1_1 = ret[1];
|
|
47
|
-
return getStringFromWasm0(ret[0], ret[1]);
|
|
48
|
-
} finally {
|
|
49
|
-
wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
/**
|
|
54
|
-
* Debug helper: returns token IDs the tokenizer produced. Used by per-stage
|
|
55
|
-
* parity tests to confirm tokenization matches the Python `tokenizers` library.
|
|
56
|
-
* @param {string} text
|
|
57
|
-
* @returns {Uint32Array}
|
|
58
|
-
*/
|
|
59
|
-
export function tokenize(text) {
|
|
60
|
-
const ptr0 = passStringToWasm0(text, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
|
61
|
-
const len0 = WASM_VECTOR_LEN;
|
|
62
|
-
const ret = wasm.tokenize(ptr0, len0);
|
|
63
|
-
var v2 = getArrayU32FromWasm0(ret[0], ret[1]).slice();
|
|
64
|
-
wasm.__wbindgen_free(ret[0], ret[1] * 4, 4);
|
|
65
|
-
return v2;
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
/**
|
|
69
|
-
* Identity of the exact tokenizer embedded in this build.
|
|
70
|
-
* @returns {string}
|
|
71
|
-
*/
|
|
72
|
-
export function tokenizer_sha256() {
|
|
73
|
-
let deferred1_0;
|
|
74
|
-
let deferred1_1;
|
|
75
|
-
try {
|
|
76
|
-
const ret = wasm.tokenizer_sha256();
|
|
77
|
-
deferred1_0 = ret[0];
|
|
78
|
-
deferred1_1 = ret[1];
|
|
79
|
-
return getStringFromWasm0(ret[0], ret[1]);
|
|
80
|
-
} finally {
|
|
81
|
-
wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
export function __wbg___wbindgen_is_function_754e9f305ff6029e(arg0) {
|
|
85
|
-
const ret = typeof(arg0) === 'function';
|
|
86
|
-
return ret;
|
|
87
|
-
}
|
|
88
|
-
export function __wbg___wbindgen_is_object_56732c2bc353f41d(arg0) {
|
|
89
|
-
const val = arg0;
|
|
90
|
-
const ret = typeof(val) === 'object' && val !== null;
|
|
91
|
-
return ret;
|
|
92
|
-
}
|
|
93
|
-
export function __wbg___wbindgen_is_string_c236cabd84a4d769(arg0) {
|
|
94
|
-
const ret = typeof(arg0) === 'string';
|
|
95
|
-
return ret;
|
|
96
|
-
}
|
|
97
|
-
export function __wbg___wbindgen_is_undefined_67b456be8673d3d7(arg0) {
|
|
98
|
-
const ret = arg0 === undefined;
|
|
99
|
-
return ret;
|
|
100
|
-
}
|
|
101
|
-
export function __wbg___wbindgen_throw_1506f2235d1bdba0(arg0, arg1) {
|
|
102
|
-
throw new Error(getStringFromWasm0(arg0, arg1));
|
|
103
|
-
}
|
|
104
|
-
export function __wbg_call_9c758de292015997() { return handleError(function (arg0, arg1, arg2) {
|
|
105
|
-
const ret = arg0.call(arg1, arg2);
|
|
106
|
-
return ret;
|
|
107
|
-
}, arguments); }
|
|
108
|
-
export function __wbg_crypto_38df2bab126b63dc(arg0) {
|
|
109
|
-
const ret = arg0.crypto;
|
|
110
|
-
return ret;
|
|
111
|
-
}
|
|
112
|
-
export function __wbg_getRandomValues_c44a50d8cfdaebeb() { return handleError(function (arg0, arg1) {
|
|
113
|
-
arg0.getRandomValues(arg1);
|
|
114
|
-
}, arguments); }
|
|
115
|
-
export function __wbg_length_4a591ecaa01354d9(arg0) {
|
|
116
|
-
const ret = arg0.length;
|
|
117
|
-
return ret;
|
|
118
|
-
}
|
|
119
|
-
export function __wbg_msCrypto_bd5a034af96bcba6(arg0) {
|
|
120
|
-
const ret = arg0.msCrypto;
|
|
121
|
-
return ret;
|
|
122
|
-
}
|
|
123
|
-
export function __wbg_new_with_length_36a4998e27b014c5(arg0) {
|
|
124
|
-
const ret = new Uint8Array(arg0 >>> 0);
|
|
125
|
-
return ret;
|
|
126
|
-
}
|
|
127
|
-
export function __wbg_node_84ea875411254db1(arg0) {
|
|
128
|
-
const ret = arg0.node;
|
|
129
|
-
return ret;
|
|
130
|
-
}
|
|
131
|
-
export function __wbg_process_44c7a14e11e9f69e(arg0) {
|
|
132
|
-
const ret = arg0.process;
|
|
133
|
-
return ret;
|
|
134
|
-
}
|
|
135
|
-
export function __wbg_prototypesetcall_3249fc62a0fafa30(arg0, arg1, arg2) {
|
|
136
|
-
Uint8Array.prototype.set.call(getArrayU8FromWasm0(arg0, arg1), arg2);
|
|
137
|
-
}
|
|
138
|
-
export function __wbg_randomFillSync_6c25eac9869eb53c() { return handleError(function (arg0, arg1) {
|
|
139
|
-
arg0.randomFillSync(arg1);
|
|
140
|
-
}, arguments); }
|
|
141
|
-
export function __wbg_require_b4edbdcf3e2a1ef0() { return handleError(function () {
|
|
142
|
-
const ret = module.require;
|
|
143
|
-
return ret;
|
|
144
|
-
}, arguments); }
|
|
145
|
-
export function __wbg_static_accessor_GLOBAL_9d53f2689e622ca1() {
|
|
146
|
-
const ret = typeof global === 'undefined' ? null : global;
|
|
147
|
-
return isLikeNone(ret) ? 0 : addToExternrefTable0(ret);
|
|
148
|
-
}
|
|
149
|
-
export function __wbg_static_accessor_GLOBAL_THIS_a1a35cec07001a8a() {
|
|
150
|
-
const ret = typeof globalThis === 'undefined' ? null : globalThis;
|
|
151
|
-
return isLikeNone(ret) ? 0 : addToExternrefTable0(ret);
|
|
152
|
-
}
|
|
153
|
-
export function __wbg_static_accessor_SELF_4c59f6c7ea29a144() {
|
|
154
|
-
const ret = typeof self === 'undefined' ? null : self;
|
|
155
|
-
return isLikeNone(ret) ? 0 : addToExternrefTable0(ret);
|
|
156
|
-
}
|
|
157
|
-
export function __wbg_static_accessor_WINDOW_e70ae9f2eb052253() {
|
|
158
|
-
const ret = typeof window === 'undefined' ? null : window;
|
|
159
|
-
return isLikeNone(ret) ? 0 : addToExternrefTable0(ret);
|
|
160
|
-
}
|
|
161
|
-
export function __wbg_subarray_4aa221f6a4f5ab22(arg0, arg1, arg2) {
|
|
162
|
-
const ret = arg0.subarray(arg1 >>> 0, arg2 >>> 0);
|
|
163
|
-
return ret;
|
|
164
|
-
}
|
|
165
|
-
export function __wbg_versions_276b2795b1c6a219(arg0) {
|
|
166
|
-
const ret = arg0.versions;
|
|
167
|
-
return ret;
|
|
168
|
-
}
|
|
169
|
-
export function __wbindgen_cast_0000000000000001(arg0, arg1) {
|
|
170
|
-
// Cast intrinsic for `Ref(Slice(U8)) -> NamedExternref("Uint8Array")`.
|
|
171
|
-
const ret = getArrayU8FromWasm0(arg0, arg1);
|
|
172
|
-
return ret;
|
|
173
|
-
}
|
|
174
|
-
export function __wbindgen_cast_0000000000000002(arg0, arg1) {
|
|
175
|
-
// Cast intrinsic for `Ref(String) -> Externref`.
|
|
176
|
-
const ret = getStringFromWasm0(arg0, arg1);
|
|
177
|
-
return ret;
|
|
178
|
-
}
|
|
179
|
-
export function __wbindgen_init_externref_table() {
|
|
180
|
-
const table = wasm.__wbindgen_externrefs;
|
|
181
|
-
const offset = table.grow(4);
|
|
182
|
-
table.set(0, undefined);
|
|
183
|
-
table.set(offset + 0, undefined);
|
|
184
|
-
table.set(offset + 1, null);
|
|
185
|
-
table.set(offset + 2, true);
|
|
186
|
-
table.set(offset + 3, false);
|
|
187
|
-
}
|
|
188
|
-
function addToExternrefTable0(obj) {
|
|
189
|
-
const idx = wasm.__externref_table_alloc();
|
|
190
|
-
wasm.__wbindgen_externrefs.set(idx, obj);
|
|
191
|
-
return idx;
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
function getArrayF32FromWasm0(ptr, len) {
|
|
195
|
-
ptr = ptr >>> 0;
|
|
196
|
-
return getFloat32ArrayMemory0().subarray(ptr / 4, ptr / 4 + len);
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
function getArrayU32FromWasm0(ptr, len) {
|
|
200
|
-
ptr = ptr >>> 0;
|
|
201
|
-
return getUint32ArrayMemory0().subarray(ptr / 4, ptr / 4 + len);
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
function getArrayU8FromWasm0(ptr, len) {
|
|
205
|
-
ptr = ptr >>> 0;
|
|
206
|
-
return getUint8ArrayMemory0().subarray(ptr / 1, ptr / 1 + len);
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
let cachedFloat32ArrayMemory0 = null;
|
|
210
|
-
function getFloat32ArrayMemory0() {
|
|
211
|
-
if (cachedFloat32ArrayMemory0 === null || cachedFloat32ArrayMemory0.byteLength === 0) {
|
|
212
|
-
cachedFloat32ArrayMemory0 = new Float32Array(wasm.memory.buffer);
|
|
213
|
-
}
|
|
214
|
-
return cachedFloat32ArrayMemory0;
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
function getStringFromWasm0(ptr, len) {
|
|
218
|
-
return decodeText(ptr >>> 0, len);
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
let cachedUint32ArrayMemory0 = null;
|
|
222
|
-
function getUint32ArrayMemory0() {
|
|
223
|
-
if (cachedUint32ArrayMemory0 === null || cachedUint32ArrayMemory0.byteLength === 0) {
|
|
224
|
-
cachedUint32ArrayMemory0 = new Uint32Array(wasm.memory.buffer);
|
|
225
|
-
}
|
|
226
|
-
return cachedUint32ArrayMemory0;
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
let cachedUint8ArrayMemory0 = null;
|
|
230
|
-
function getUint8ArrayMemory0() {
|
|
231
|
-
if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) {
|
|
232
|
-
cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer);
|
|
233
|
-
}
|
|
234
|
-
return cachedUint8ArrayMemory0;
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
function handleError(f, args) {
|
|
238
|
-
try {
|
|
239
|
-
return f.apply(this, args);
|
|
240
|
-
} catch (e) {
|
|
241
|
-
const idx = addToExternrefTable0(e);
|
|
242
|
-
wasm.__wbindgen_exn_store(idx);
|
|
243
|
-
}
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
function isLikeNone(x) {
|
|
247
|
-
return x === undefined || x === null;
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
function passStringToWasm0(arg, malloc, realloc) {
|
|
251
|
-
if (realloc === undefined) {
|
|
252
|
-
const buf = cachedTextEncoder.encode(arg);
|
|
253
|
-
const ptr = malloc(buf.length, 1) >>> 0;
|
|
254
|
-
getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf);
|
|
255
|
-
WASM_VECTOR_LEN = buf.length;
|
|
256
|
-
return ptr;
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
let len = arg.length;
|
|
260
|
-
let ptr = malloc(len, 1) >>> 0;
|
|
261
|
-
|
|
262
|
-
const mem = getUint8ArrayMemory0();
|
|
263
|
-
|
|
264
|
-
let offset = 0;
|
|
265
|
-
|
|
266
|
-
for (; offset < len; offset++) {
|
|
267
|
-
const code = arg.charCodeAt(offset);
|
|
268
|
-
if (code > 0x7F) break;
|
|
269
|
-
mem[ptr + offset] = code;
|
|
270
|
-
}
|
|
271
|
-
if (offset !== len) {
|
|
272
|
-
if (offset !== 0) {
|
|
273
|
-
arg = arg.slice(offset);
|
|
274
|
-
}
|
|
275
|
-
ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0;
|
|
276
|
-
const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len);
|
|
277
|
-
const ret = cachedTextEncoder.encodeInto(arg, view);
|
|
278
|
-
|
|
279
|
-
offset += ret.written;
|
|
280
|
-
ptr = realloc(ptr, len, offset, 1) >>> 0;
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
WASM_VECTOR_LEN = offset;
|
|
284
|
-
return ptr;
|
|
285
|
-
}
|
|
286
|
-
|
|
287
|
-
let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
|
|
288
|
-
cachedTextDecoder.decode();
|
|
289
|
-
const MAX_SAFARI_DECODE_BYTES = 2146435072;
|
|
290
|
-
let numBytesDecoded = 0;
|
|
291
|
-
function decodeText(ptr, len) {
|
|
292
|
-
numBytesDecoded += len;
|
|
293
|
-
if (numBytesDecoded >= MAX_SAFARI_DECODE_BYTES) {
|
|
294
|
-
cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
|
|
295
|
-
cachedTextDecoder.decode();
|
|
296
|
-
numBytesDecoded = len;
|
|
297
|
-
}
|
|
298
|
-
return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len));
|
|
299
|
-
}
|
|
300
|
-
|
|
301
|
-
const cachedTextEncoder = new TextEncoder();
|
|
302
|
-
|
|
303
|
-
if (!('encodeInto' in cachedTextEncoder)) {
|
|
304
|
-
cachedTextEncoder.encodeInto = function (arg, view) {
|
|
305
|
-
const buf = cachedTextEncoder.encode(arg);
|
|
306
|
-
view.set(buf);
|
|
307
|
-
return {
|
|
308
|
-
read: arg.length,
|
|
309
|
-
written: buf.length
|
|
310
|
-
};
|
|
311
|
-
};
|
|
312
|
-
}
|
|
313
|
-
|
|
314
|
-
let WASM_VECTOR_LEN = 0;
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
let wasm;
|
|
318
|
-
export function __wbg_set_wasm(val) {
|
|
319
|
-
wasm = val;
|
|
320
|
-
}
|