@usex/ternava 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -0
- package/LICENSE +21 -0
- package/README.md +74 -2
- package/package.json +81 -4
- package/pkg-bundler/package.json +1 -0
- package/pkg-bundler/tern_engine.d.ts +32 -0
- package/pkg-bundler/tern_engine.js +9 -0
- package/pkg-bundler/tern_engine_bg.js +320 -0
- package/pkg-bundler/tern_engine_bg.wasm +0 -0
- package/pkg-bundler/tern_engine_bg.wasm.d.ts +15 -0
- package/pkg-node/tern_engine.d.ts +32 -0
- package/pkg-node/tern_engine.js +330 -0
- package/pkg-node/tern_engine_bg.js +320 -0
- package/pkg-node/tern_engine_bg.wasm +0 -0
- package/pkg-node/tern_engine_bg.wasm.d.ts +15 -0
- package/pkg-web/index.d.mts +16 -0
- package/pkg-web/index.mjs +67 -0
- package/pkg-web/package.json +1 -0
- package/pkg-web/tern_engine.d.ts +72 -0
- package/pkg-web/tern_engine.js +421 -0
- package/pkg-web/tern_engine_bg.js +320 -0
- package/pkg-web/tern_engine_bg.wasm +0 -0
- package/pkg-web/tern_engine_bg.wasm.d.ts +15 -0
- package/scripts/check-package.mjs +39 -0
- package/src/index.browser.mjs +80 -0
- package/src/index.d.ts +86 -0
- package/src/index.js +84 -0
- package/src/index.mjs +80 -0
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Debug helper: returns a human-readable summary of the loaded model's
|
|
3
|
+
* header (format version, embedding format, dimensions, vocab size).
|
|
4
|
+
* @returns {string}
|
|
5
|
+
*/
|
|
6
|
+
export function config_summary() {
|
|
7
|
+
let deferred1_0;
|
|
8
|
+
let deferred1_1;
|
|
9
|
+
try {
|
|
10
|
+
const ret = wasm.config_summary();
|
|
11
|
+
deferred1_0 = ret[0];
|
|
12
|
+
deferred1_1 = ret[1];
|
|
13
|
+
return getStringFromWasm0(ret[0], ret[1]);
|
|
14
|
+
} finally {
|
|
15
|
+
wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Primary entry point: text → 384-dim L2-normalized embedding.
|
|
21
|
+
*
|
|
22
|
+
* Tokenizes via the embedded BERT vocab, runs the full forward pass against
|
|
23
|
+
* the embedded `.bin`, returns the output vector.
|
|
24
|
+
* @param {string} text
|
|
25
|
+
* @returns {Float32Array}
|
|
26
|
+
*/
|
|
27
|
+
export function embed(text) {
|
|
28
|
+
const ptr0 = passStringToWasm0(text, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
|
29
|
+
const len0 = WASM_VECTOR_LEN;
|
|
30
|
+
const ret = wasm.embed(ptr0, len0);
|
|
31
|
+
var v2 = getArrayF32FromWasm0(ret[0], ret[1]).slice();
|
|
32
|
+
wasm.__wbindgen_free(ret[0], ret[1] * 4, 4);
|
|
33
|
+
return v2;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Identity of the exact packed model embedded in this build.
|
|
38
|
+
* @returns {string}
|
|
39
|
+
*/
|
|
40
|
+
export function model_sha256() {
|
|
41
|
+
let deferred1_0;
|
|
42
|
+
let deferred1_1;
|
|
43
|
+
try {
|
|
44
|
+
const ret = wasm.model_sha256();
|
|
45
|
+
deferred1_0 = ret[0];
|
|
46
|
+
deferred1_1 = ret[1];
|
|
47
|
+
return getStringFromWasm0(ret[0], ret[1]);
|
|
48
|
+
} finally {
|
|
49
|
+
wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Debug helper: returns token IDs the tokenizer produced. Used by per-stage
|
|
55
|
+
* parity tests to confirm tokenization matches the Python `tokenizers` library.
|
|
56
|
+
* @param {string} text
|
|
57
|
+
* @returns {Uint32Array}
|
|
58
|
+
*/
|
|
59
|
+
export function tokenize(text) {
|
|
60
|
+
const ptr0 = passStringToWasm0(text, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
|
61
|
+
const len0 = WASM_VECTOR_LEN;
|
|
62
|
+
const ret = wasm.tokenize(ptr0, len0);
|
|
63
|
+
var v2 = getArrayU32FromWasm0(ret[0], ret[1]).slice();
|
|
64
|
+
wasm.__wbindgen_free(ret[0], ret[1] * 4, 4);
|
|
65
|
+
return v2;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Identity of the exact tokenizer embedded in this build.
|
|
70
|
+
* @returns {string}
|
|
71
|
+
*/
|
|
72
|
+
export function tokenizer_sha256() {
|
|
73
|
+
let deferred1_0;
|
|
74
|
+
let deferred1_1;
|
|
75
|
+
try {
|
|
76
|
+
const ret = wasm.tokenizer_sha256();
|
|
77
|
+
deferred1_0 = ret[0];
|
|
78
|
+
deferred1_1 = ret[1];
|
|
79
|
+
return getStringFromWasm0(ret[0], ret[1]);
|
|
80
|
+
} finally {
|
|
81
|
+
wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
export function __wbg___wbindgen_is_function_754e9f305ff6029e(arg0) {
|
|
85
|
+
const ret = typeof(arg0) === 'function';
|
|
86
|
+
return ret;
|
|
87
|
+
}
|
|
88
|
+
export function __wbg___wbindgen_is_object_56732c2bc353f41d(arg0) {
|
|
89
|
+
const val = arg0;
|
|
90
|
+
const ret = typeof(val) === 'object' && val !== null;
|
|
91
|
+
return ret;
|
|
92
|
+
}
|
|
93
|
+
export function __wbg___wbindgen_is_string_c236cabd84a4d769(arg0) {
|
|
94
|
+
const ret = typeof(arg0) === 'string';
|
|
95
|
+
return ret;
|
|
96
|
+
}
|
|
97
|
+
export function __wbg___wbindgen_is_undefined_67b456be8673d3d7(arg0) {
|
|
98
|
+
const ret = arg0 === undefined;
|
|
99
|
+
return ret;
|
|
100
|
+
}
|
|
101
|
+
export function __wbg___wbindgen_throw_1506f2235d1bdba0(arg0, arg1) {
|
|
102
|
+
throw new Error(getStringFromWasm0(arg0, arg1));
|
|
103
|
+
}
|
|
104
|
+
export function __wbg_call_9c758de292015997() { return handleError(function (arg0, arg1, arg2) {
|
|
105
|
+
const ret = arg0.call(arg1, arg2);
|
|
106
|
+
return ret;
|
|
107
|
+
}, arguments); }
|
|
108
|
+
export function __wbg_crypto_38df2bab126b63dc(arg0) {
|
|
109
|
+
const ret = arg0.crypto;
|
|
110
|
+
return ret;
|
|
111
|
+
}
|
|
112
|
+
export function __wbg_getRandomValues_c44a50d8cfdaebeb() { return handleError(function (arg0, arg1) {
|
|
113
|
+
arg0.getRandomValues(arg1);
|
|
114
|
+
}, arguments); }
|
|
115
|
+
export function __wbg_length_4a591ecaa01354d9(arg0) {
|
|
116
|
+
const ret = arg0.length;
|
|
117
|
+
return ret;
|
|
118
|
+
}
|
|
119
|
+
export function __wbg_msCrypto_bd5a034af96bcba6(arg0) {
|
|
120
|
+
const ret = arg0.msCrypto;
|
|
121
|
+
return ret;
|
|
122
|
+
}
|
|
123
|
+
export function __wbg_new_with_length_36a4998e27b014c5(arg0) {
|
|
124
|
+
const ret = new Uint8Array(arg0 >>> 0);
|
|
125
|
+
return ret;
|
|
126
|
+
}
|
|
127
|
+
export function __wbg_node_84ea875411254db1(arg0) {
|
|
128
|
+
const ret = arg0.node;
|
|
129
|
+
return ret;
|
|
130
|
+
}
|
|
131
|
+
export function __wbg_process_44c7a14e11e9f69e(arg0) {
|
|
132
|
+
const ret = arg0.process;
|
|
133
|
+
return ret;
|
|
134
|
+
}
|
|
135
|
+
export function __wbg_prototypesetcall_3249fc62a0fafa30(arg0, arg1, arg2) {
|
|
136
|
+
Uint8Array.prototype.set.call(getArrayU8FromWasm0(arg0, arg1), arg2);
|
|
137
|
+
}
|
|
138
|
+
export function __wbg_randomFillSync_6c25eac9869eb53c() { return handleError(function (arg0, arg1) {
|
|
139
|
+
arg0.randomFillSync(arg1);
|
|
140
|
+
}, arguments); }
|
|
141
|
+
export function __wbg_require_b4edbdcf3e2a1ef0() { return handleError(function () {
|
|
142
|
+
const ret = module.require;
|
|
143
|
+
return ret;
|
|
144
|
+
}, arguments); }
|
|
145
|
+
export function __wbg_static_accessor_GLOBAL_9d53f2689e622ca1() {
|
|
146
|
+
const ret = typeof global === 'undefined' ? null : global;
|
|
147
|
+
return isLikeNone(ret) ? 0 : addToExternrefTable0(ret);
|
|
148
|
+
}
|
|
149
|
+
export function __wbg_static_accessor_GLOBAL_THIS_a1a35cec07001a8a() {
|
|
150
|
+
const ret = typeof globalThis === 'undefined' ? null : globalThis;
|
|
151
|
+
return isLikeNone(ret) ? 0 : addToExternrefTable0(ret);
|
|
152
|
+
}
|
|
153
|
+
export function __wbg_static_accessor_SELF_4c59f6c7ea29a144() {
|
|
154
|
+
const ret = typeof self === 'undefined' ? null : self;
|
|
155
|
+
return isLikeNone(ret) ? 0 : addToExternrefTable0(ret);
|
|
156
|
+
}
|
|
157
|
+
export function __wbg_static_accessor_WINDOW_e70ae9f2eb052253() {
|
|
158
|
+
const ret = typeof window === 'undefined' ? null : window;
|
|
159
|
+
return isLikeNone(ret) ? 0 : addToExternrefTable0(ret);
|
|
160
|
+
}
|
|
161
|
+
export function __wbg_subarray_4aa221f6a4f5ab22(arg0, arg1, arg2) {
|
|
162
|
+
const ret = arg0.subarray(arg1 >>> 0, arg2 >>> 0);
|
|
163
|
+
return ret;
|
|
164
|
+
}
|
|
165
|
+
export function __wbg_versions_276b2795b1c6a219(arg0) {
|
|
166
|
+
const ret = arg0.versions;
|
|
167
|
+
return ret;
|
|
168
|
+
}
|
|
169
|
+
export function __wbindgen_cast_0000000000000001(arg0, arg1) {
|
|
170
|
+
// Cast intrinsic for `Ref(Slice(U8)) -> NamedExternref("Uint8Array")`.
|
|
171
|
+
const ret = getArrayU8FromWasm0(arg0, arg1);
|
|
172
|
+
return ret;
|
|
173
|
+
}
|
|
174
|
+
export function __wbindgen_cast_0000000000000002(arg0, arg1) {
|
|
175
|
+
// Cast intrinsic for `Ref(String) -> Externref`.
|
|
176
|
+
const ret = getStringFromWasm0(arg0, arg1);
|
|
177
|
+
return ret;
|
|
178
|
+
}
|
|
179
|
+
export function __wbindgen_init_externref_table() {
|
|
180
|
+
const table = wasm.__wbindgen_externrefs;
|
|
181
|
+
const offset = table.grow(4);
|
|
182
|
+
table.set(0, undefined);
|
|
183
|
+
table.set(offset + 0, undefined);
|
|
184
|
+
table.set(offset + 1, null);
|
|
185
|
+
table.set(offset + 2, true);
|
|
186
|
+
table.set(offset + 3, false);
|
|
187
|
+
}
|
|
188
|
+
function addToExternrefTable0(obj) {
|
|
189
|
+
const idx = wasm.__externref_table_alloc();
|
|
190
|
+
wasm.__wbindgen_externrefs.set(idx, obj);
|
|
191
|
+
return idx;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function getArrayF32FromWasm0(ptr, len) {
|
|
195
|
+
ptr = ptr >>> 0;
|
|
196
|
+
return getFloat32ArrayMemory0().subarray(ptr / 4, ptr / 4 + len);
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function getArrayU32FromWasm0(ptr, len) {
|
|
200
|
+
ptr = ptr >>> 0;
|
|
201
|
+
return getUint32ArrayMemory0().subarray(ptr / 4, ptr / 4 + len);
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
function getArrayU8FromWasm0(ptr, len) {
|
|
205
|
+
ptr = ptr >>> 0;
|
|
206
|
+
return getUint8ArrayMemory0().subarray(ptr / 1, ptr / 1 + len);
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
let cachedFloat32ArrayMemory0 = null;
|
|
210
|
+
function getFloat32ArrayMemory0() {
|
|
211
|
+
if (cachedFloat32ArrayMemory0 === null || cachedFloat32ArrayMemory0.byteLength === 0) {
|
|
212
|
+
cachedFloat32ArrayMemory0 = new Float32Array(wasm.memory.buffer);
|
|
213
|
+
}
|
|
214
|
+
return cachedFloat32ArrayMemory0;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function getStringFromWasm0(ptr, len) {
|
|
218
|
+
return decodeText(ptr >>> 0, len);
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
let cachedUint32ArrayMemory0 = null;
|
|
222
|
+
function getUint32ArrayMemory0() {
|
|
223
|
+
if (cachedUint32ArrayMemory0 === null || cachedUint32ArrayMemory0.byteLength === 0) {
|
|
224
|
+
cachedUint32ArrayMemory0 = new Uint32Array(wasm.memory.buffer);
|
|
225
|
+
}
|
|
226
|
+
return cachedUint32ArrayMemory0;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
let cachedUint8ArrayMemory0 = null;
|
|
230
|
+
function getUint8ArrayMemory0() {
|
|
231
|
+
if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) {
|
|
232
|
+
cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer);
|
|
233
|
+
}
|
|
234
|
+
return cachedUint8ArrayMemory0;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function handleError(f, args) {
|
|
238
|
+
try {
|
|
239
|
+
return f.apply(this, args);
|
|
240
|
+
} catch (e) {
|
|
241
|
+
const idx = addToExternrefTable0(e);
|
|
242
|
+
wasm.__wbindgen_exn_store(idx);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
function isLikeNone(x) {
|
|
247
|
+
return x === undefined || x === null;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
function passStringToWasm0(arg, malloc, realloc) {
|
|
251
|
+
if (realloc === undefined) {
|
|
252
|
+
const buf = cachedTextEncoder.encode(arg);
|
|
253
|
+
const ptr = malloc(buf.length, 1) >>> 0;
|
|
254
|
+
getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf);
|
|
255
|
+
WASM_VECTOR_LEN = buf.length;
|
|
256
|
+
return ptr;
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
let len = arg.length;
|
|
260
|
+
let ptr = malloc(len, 1) >>> 0;
|
|
261
|
+
|
|
262
|
+
const mem = getUint8ArrayMemory0();
|
|
263
|
+
|
|
264
|
+
let offset = 0;
|
|
265
|
+
|
|
266
|
+
for (; offset < len; offset++) {
|
|
267
|
+
const code = arg.charCodeAt(offset);
|
|
268
|
+
if (code > 0x7F) break;
|
|
269
|
+
mem[ptr + offset] = code;
|
|
270
|
+
}
|
|
271
|
+
if (offset !== len) {
|
|
272
|
+
if (offset !== 0) {
|
|
273
|
+
arg = arg.slice(offset);
|
|
274
|
+
}
|
|
275
|
+
ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0;
|
|
276
|
+
const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len);
|
|
277
|
+
const ret = cachedTextEncoder.encodeInto(arg, view);
|
|
278
|
+
|
|
279
|
+
offset += ret.written;
|
|
280
|
+
ptr = realloc(ptr, len, offset, 1) >>> 0;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
WASM_VECTOR_LEN = offset;
|
|
284
|
+
return ptr;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
|
|
288
|
+
cachedTextDecoder.decode();
|
|
289
|
+
const MAX_SAFARI_DECODE_BYTES = 2146435072;
|
|
290
|
+
let numBytesDecoded = 0;
|
|
291
|
+
function decodeText(ptr, len) {
|
|
292
|
+
numBytesDecoded += len;
|
|
293
|
+
if (numBytesDecoded >= MAX_SAFARI_DECODE_BYTES) {
|
|
294
|
+
cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
|
|
295
|
+
cachedTextDecoder.decode();
|
|
296
|
+
numBytesDecoded = len;
|
|
297
|
+
}
|
|
298
|
+
return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len));
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
const cachedTextEncoder = new TextEncoder();
|
|
302
|
+
|
|
303
|
+
if (!('encodeInto' in cachedTextEncoder)) {
|
|
304
|
+
cachedTextEncoder.encodeInto = function (arg, view) {
|
|
305
|
+
const buf = cachedTextEncoder.encode(arg);
|
|
306
|
+
view.set(buf);
|
|
307
|
+
return {
|
|
308
|
+
read: arg.length,
|
|
309
|
+
written: buf.length
|
|
310
|
+
};
|
|
311
|
+
};
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
let WASM_VECTOR_LEN = 0;
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
let wasm;
|
|
318
|
+
export function __wbg_set_wasm(val) {
|
|
319
|
+
wasm = val;
|
|
320
|
+
}
|
|
Binary file
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/* tslint:disable */
|
|
2
|
+
/* eslint-disable */
|
|
3
|
+
export const memory: WebAssembly.Memory;
|
|
4
|
+
export const config_summary: () => [number, number];
|
|
5
|
+
export const embed: (a: number, b: number) => [number, number];
|
|
6
|
+
export const model_sha256: () => [number, number];
|
|
7
|
+
export const tokenize: (a: number, b: number) => [number, number];
|
|
8
|
+
export const tokenizer_sha256: () => [number, number];
|
|
9
|
+
export const __wbindgen_exn_store: (a: number) => void;
|
|
10
|
+
export const __externref_table_alloc: () => number;
|
|
11
|
+
export const __wbindgen_externrefs: WebAssembly.Table;
|
|
12
|
+
export const __wbindgen_free: (a: number, b: number, c: number) => void;
|
|
13
|
+
export const __wbindgen_malloc: (a: number, b: number) => number;
|
|
14
|
+
export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
|
|
15
|
+
export const __wbindgen_start: () => void;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import { readFileSync, statSync } from 'node:fs';
|
|
3
|
+
import { createRequire } from 'node:module';
|
|
4
|
+
|
|
5
|
+
const root = new URL('../', import.meta.url);
|
|
6
|
+
const manifest = JSON.parse(readFileSync(new URL('package.json', root)));
|
|
7
|
+
assert.equal(manifest.name, '@usex/ternava');
|
|
8
|
+
assert.equal(manifest.repository.url, 'https://github.com/ali-master/ternava.git');
|
|
9
|
+
assert.equal(manifest.homepage, 'https://ternava.usestrict.dev/');
|
|
10
|
+
assert.equal(manifest.bugs.url, 'https://github.com/ali-master/ternava/issues');
|
|
11
|
+
assert.equal(manifest.publishConfig.access, 'public');
|
|
12
|
+
assert.notEqual(manifest.private, true);
|
|
13
|
+
function checkExports(value) {
|
|
14
|
+
if (typeof value === 'string') {
|
|
15
|
+
assert.ok(statSync(new URL(value, root)).isFile(), `Missing export: ${value}`);
|
|
16
|
+
} else {
|
|
17
|
+
Object.values(value).forEach(checkExports);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
checkExports(manifest.exports);
|
|
21
|
+
for (const target of ['node', 'bundler', 'web']) {
|
|
22
|
+
const bytes = readFileSync(new URL(`pkg-${target}/tern_engine_bg.wasm`, root));
|
|
23
|
+
assert.ok(WebAssembly.validate(bytes), `Invalid ${target} WASM`);
|
|
24
|
+
}
|
|
25
|
+
const node = createRequire(import.meta.url)('../src/index.js');
|
|
26
|
+
assert.match(node.engineInfo(), /embedding_format=int4/);
|
|
27
|
+
assert.match(node.engineInfo(), /vocab=32000 d_model=256 n_layers=2 n_heads=4 ffn_dim=1024 output_dim=384 max_seq_len=128/);
|
|
28
|
+
const web = await import(new URL('pkg-web/index.mjs', root));
|
|
29
|
+
await web.init();
|
|
30
|
+
const browser = await import(new URL('src/index.browser.mjs', root));
|
|
31
|
+
assert.equal(browser.engineInfo(), node.engineInfo());
|
|
32
|
+
assert.equal(web.engineInfo(), node.engineInfo());
|
|
33
|
+
for (const text of ['خیلی دوستت دارم', 'هیچ سفارشی بدون پرداخت ارسال نمیشود.', '۱۲ بسته و ۵ بسته', 'Reset my password']) {
|
|
34
|
+
const vector = node.embed(text);
|
|
35
|
+
assert.equal(vector.length, 384);
|
|
36
|
+
assert.ok(vector.every(Number.isFinite));
|
|
37
|
+
assert.deepEqual(browser.embed(text), vector, 'Bundler model/tokenizer differs');
|
|
38
|
+
assert.deepEqual(web.embed(text), vector, 'Node and web must embed the same model/tokenizer');
|
|
39
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
// ternava — public browser/bundler ESM entry.
|
|
2
|
+
//
|
|
3
|
+
// Routed to by the package.json "browser"/"default" export conditions. Wraps
|
|
4
|
+
// wasm-pack's bundler-target output in ../pkg-bundler/, whose glue imports the
|
|
5
|
+
// .wasm as an ESM asset — Vite, webpack ≥5 (asyncWebAssembly), Next.js, and
|
|
6
|
+
// Cloudflare Workers handle the instantiation as part of the module graph, so
|
|
7
|
+
// consumers just `import { embed } from '@usex/ternava'` with no init call.
|
|
8
|
+
//
|
|
9
|
+
// Same public API as the Node entries (src/index.js / src/index.mjs).
|
|
10
|
+
|
|
11
|
+
import { embed as _embed, config_summary } from '../pkg-bundler/tern_engine.js';
|
|
12
|
+
|
|
13
|
+
export class TernError extends Error {
|
|
14
|
+
constructor(message, code) {
|
|
15
|
+
super(message);
|
|
16
|
+
this.name = 'TernError';
|
|
17
|
+
this.code = code;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Embed text → 384-dim L2-normalized Float32Array.
|
|
23
|
+
* Pure CPU inference via WASM. Synchronous. No network calls.
|
|
24
|
+
*
|
|
25
|
+
* Input is tokenized via BERT WordPiece and truncated to 128 tokens
|
|
26
|
+
* (~95 English words). Longer text is silently truncated.
|
|
27
|
+
*/
|
|
28
|
+
export function embed(text) {
|
|
29
|
+
if (typeof text !== 'string') {
|
|
30
|
+
throw new TernError(
|
|
31
|
+
'embed(text): text must be a string',
|
|
32
|
+
'INVALID_INPUT',
|
|
33
|
+
);
|
|
34
|
+
}
|
|
35
|
+
return _embed(text);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Cosine similarity between two embeddings.
|
|
40
|
+
*
|
|
41
|
+
* Since ternava embeddings are L2-normalized, this reduces to a dot product
|
|
42
|
+
* — no per-call sqrt or division.
|
|
43
|
+
*/
|
|
44
|
+
export function cosineSim(a, b) {
|
|
45
|
+
if (a.length !== b.length) {
|
|
46
|
+
throw new TernError(
|
|
47
|
+
`vector length mismatch: ${a.length} vs ${b.length}`,
|
|
48
|
+
'DIM_MISMATCH',
|
|
49
|
+
);
|
|
50
|
+
}
|
|
51
|
+
let dot = 0;
|
|
52
|
+
const len = a.length;
|
|
53
|
+
for (let i = 0; i < len; i++) dot += a[i] * b[i];
|
|
54
|
+
return dot;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Convenience: embed query + each corpus item, return top-K matches sorted
|
|
59
|
+
* descending by similarity.
|
|
60
|
+
*
|
|
61
|
+
* For repeated searches over the same corpus, embed it once upfront and call
|
|
62
|
+
* cosineSim() yourself — see the README "Reuse embeddings" pattern.
|
|
63
|
+
*/
|
|
64
|
+
export function similar(query, corpus, opts = {}) {
|
|
65
|
+
const topK = opts.topK ?? 5;
|
|
66
|
+
const q = embed(query);
|
|
67
|
+
return corpus
|
|
68
|
+
.map((text) => ({ text, sim: cosineSim(q, embed(text)) }))
|
|
69
|
+
.sort((a, b) => b.sim - a.sim)
|
|
70
|
+
.slice(0, topK);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Debug helper: returns a string describing the loaded engine's configuration
|
|
75
|
+
* (format version, embedding format, dimensions, vocab size). Useful for
|
|
76
|
+
* confirming which build of the engine is actually loaded.
|
|
77
|
+
*/
|
|
78
|
+
export function engineInfo() {
|
|
79
|
+
return config_summary();
|
|
80
|
+
}
|
package/src/index.d.ts
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A 384-dim L2-normalized text embedding.
|
|
3
|
+
*/
|
|
4
|
+
export type Embedding = Float32Array;
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* One row from a nearest-neighbor search.
|
|
8
|
+
*/
|
|
9
|
+
export interface SimilarityResult {
|
|
10
|
+
/** The original input string. */
|
|
11
|
+
text: string;
|
|
12
|
+
/** Cosine similarity to the query, in `[-1, 1]`. */
|
|
13
|
+
sim: number;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Common error class for ternava. The `code` property gives a stable
|
|
18
|
+
* identifier for catching specific error categories programmatically:
|
|
19
|
+
*
|
|
20
|
+
* - `INVALID_INPUT` — argument was not the expected type
|
|
21
|
+
* - `DIM_MISMATCH` — vectors of different lengths were compared
|
|
22
|
+
*/
|
|
23
|
+
export class TernError extends Error {
|
|
24
|
+
code?: string;
|
|
25
|
+
constructor(message: string, code?: string);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Embed text → 384-dim L2-normalized embedding.
|
|
30
|
+
*
|
|
31
|
+
* Pure on-device CPU inference via WASM. Synchronous in Node. No network call.
|
|
32
|
+
*
|
|
33
|
+
* Input is tokenized via BERT WordPiece (vocab size 32000) and truncated to
|
|
34
|
+
* 128 tokens (~95 English words). Longer text is silently truncated.
|
|
35
|
+
*
|
|
36
|
+
* @param text — Input string.
|
|
37
|
+
* @returns 384-dimensional `Float32Array` on the unit hypersphere.
|
|
38
|
+
*
|
|
39
|
+
* @example
|
|
40
|
+
* const v = embed("how do I reset my password");
|
|
41
|
+
* v.length; // 384
|
|
42
|
+
*/
|
|
43
|
+
export function embed(text: string): Embedding;
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Cosine similarity between two embeddings.
|
|
47
|
+
*
|
|
48
|
+
* Since ternava embeddings are L2-normalized, this reduces to a dot product
|
|
49
|
+
* (no per-call sqrt or division).
|
|
50
|
+
*
|
|
51
|
+
* @returns Scalar in `[-1, 1]`. For typical text, output is in `[0, 1]`.
|
|
52
|
+
*
|
|
53
|
+
* @example
|
|
54
|
+
* const a = embed("forgot password");
|
|
55
|
+
* const b = embed("password reset");
|
|
56
|
+
* cosineSim(a, b); // ~0.85
|
|
57
|
+
*/
|
|
58
|
+
export function cosineSim(a: Embedding, b: Embedding): number;
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Convenience: embed query + each corpus item, return top-K matches sorted
|
|
62
|
+
* descending by similarity.
|
|
63
|
+
*
|
|
64
|
+
* For repeated searches over the same corpus, embed it once upfront and call
|
|
65
|
+
* `cosineSim()` yourself — see the README for that pattern.
|
|
66
|
+
*
|
|
67
|
+
* @param query - Query string.
|
|
68
|
+
* @param corpus - Array of candidate strings.
|
|
69
|
+
* @param opts.topK - Number of results to return (default 5).
|
|
70
|
+
*/
|
|
71
|
+
export function similar(
|
|
72
|
+
query: string,
|
|
73
|
+
corpus: string[],
|
|
74
|
+
opts?: { topK?: number },
|
|
75
|
+
): SimilarityResult[];
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Debug helper: returns a string describing the loaded engine's configuration
|
|
79
|
+
* (format version, embedding format, dimensions, vocab size). Useful for
|
|
80
|
+
* confirming which build of the engine is actually loaded.
|
|
81
|
+
*
|
|
82
|
+
* @example
|
|
83
|
+
* engineInfo();
|
|
84
|
+
* // → "tern-engine v1 | embedding_format=int4 | vocab=32000 d_model=256 ..."
|
|
85
|
+
*/
|
|
86
|
+
export function engineInfo(): string;
|
package/src/index.js
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
// ternava — public API.
|
|
2
|
+
//
|
|
3
|
+
// Thin JS wrapper around the WASM engine. The engine does the real work
|
|
4
|
+
// (tokenization, BitLinear forward pass, returning a normalized embedding);
|
|
5
|
+
// this file adds ergonomic helpers (cosine similarity, top-K search) and
|
|
6
|
+
// stable error types.
|
|
7
|
+
//
|
|
8
|
+
// The compiled engine lives in ../pkg-node/, populated by scripts/build-engine.sh
|
|
9
|
+
// from engine/pkg/ (the wasm-pack nodejs-target output). The model weights
|
|
10
|
+
// and BERT tokenizer are baked into the .wasm itself via Rust's
|
|
11
|
+
// `include_bytes!`, so this package has no runtime asset fetches.
|
|
12
|
+
|
|
13
|
+
const { embed: _embed, config_summary } = require('../pkg-node/tern_engine.js');
|
|
14
|
+
|
|
15
|
+
class TernError extends Error {
|
|
16
|
+
constructor(message, code) {
|
|
17
|
+
super(message);
|
|
18
|
+
this.name = 'TernError';
|
|
19
|
+
this.code = code;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Embed text → 384-dim L2-normalized Float32Array.
|
|
25
|
+
* Pure CPU inference via WASM. Synchronous. No network calls.
|
|
26
|
+
*
|
|
27
|
+
* Input is tokenized via BERT WordPiece and truncated to 128 tokens
|
|
28
|
+
* (~95 English words). Longer text is silently truncated.
|
|
29
|
+
*/
|
|
30
|
+
function embed(text) {
|
|
31
|
+
if (typeof text !== 'string') {
|
|
32
|
+
throw new TernError(
|
|
33
|
+
'embed(text): text must be a string',
|
|
34
|
+
'INVALID_INPUT',
|
|
35
|
+
);
|
|
36
|
+
}
|
|
37
|
+
return _embed(text);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Cosine similarity between two embeddings.
|
|
42
|
+
*
|
|
43
|
+
* Since ternava embeddings are L2-normalized, this reduces to a dot product
|
|
44
|
+
* — no per-call sqrt or division.
|
|
45
|
+
*/
|
|
46
|
+
function cosineSim(a, b) {
|
|
47
|
+
if (a.length !== b.length) {
|
|
48
|
+
throw new TernError(
|
|
49
|
+
`vector length mismatch: ${a.length} vs ${b.length}`,
|
|
50
|
+
'DIM_MISMATCH',
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
let dot = 0;
|
|
54
|
+
const len = a.length;
|
|
55
|
+
for (let i = 0; i < len; i++) dot += a[i] * b[i];
|
|
56
|
+
return dot;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Convenience: embed query + each corpus item, return top-K matches sorted
|
|
61
|
+
* descending by similarity.
|
|
62
|
+
*
|
|
63
|
+
* For repeated searches over the same corpus, embed it once upfront and call
|
|
64
|
+
* cosineSim() yourself — see the README "Reuse embeddings" pattern.
|
|
65
|
+
*/
|
|
66
|
+
function similar(query, corpus, opts = {}) {
|
|
67
|
+
const topK = opts.topK ?? 5;
|
|
68
|
+
const q = embed(query);
|
|
69
|
+
return corpus
|
|
70
|
+
.map((text) => ({ text, sim: cosineSim(q, embed(text)) }))
|
|
71
|
+
.sort((a, b) => b.sim - a.sim)
|
|
72
|
+
.slice(0, topK);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Debug helper: returns a string describing the loaded engine's configuration
|
|
77
|
+
* (format version, embedding format, dimensions, vocab size). Useful for
|
|
78
|
+
* confirming which build of the engine is actually loaded.
|
|
79
|
+
*/
|
|
80
|
+
function engineInfo() {
|
|
81
|
+
return config_summary();
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
module.exports = { embed, cosineSim, similar, engineInfo, TernError };
|