@usex/ternava 0.0.0-stage → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,67 @@
1
+ import initWasm, { embed as embedWasm, config_summary } from './tern_engine.js';
2
+
3
+ let initialization;
4
+ let ready = false;
5
+
6
+ export class TernError extends Error {
7
+ constructor(message, code) {
8
+ super(message);
9
+ this.name = 'TernError';
10
+ this.code = code;
11
+ }
12
+ }
13
+
14
+ /** Load once, then run synchronous inference in Node.js, Bun, or a browser. */
15
+ export function init(source = new URL('./tern_engine_bg.wasm', import.meta.url)) {
16
+ if (!initialization) {
17
+ initialization = (async () => {
18
+ if (source instanceof URL && source.protocol === 'file:') {
19
+ const { readFile } = await import('node:fs/promises');
20
+ source = await readFile(source);
21
+ }
22
+ await initWasm({ module_or_path: source });
23
+ ready = true;
24
+ })().catch(error => {
25
+ initialization = undefined;
26
+ throw error;
27
+ });
28
+ }
29
+ return initialization;
30
+ }
31
+
32
+ export default init;
33
+
34
+ function requireReady() {
35
+ if (!ready) throw new TernError('Call await init() before inference', 'NOT_INITIALIZED');
36
+ }
37
+
38
+ export function embed(text) {
39
+ if (typeof text !== 'string') {
40
+ throw new TernError('embed(text): text must be a string', 'INVALID_INPUT');
41
+ }
42
+ requireReady();
43
+ return embedWasm(text);
44
+ }
45
+
46
+ /** Dot product of L2-normalized embeddings. */
47
+ export function cosineSim(a, b) {
48
+ if (a.length !== b.length) {
49
+ throw new TernError(`vector length mismatch: ${a.length} vs ${b.length}`, 'DIM_MISMATCH');
50
+ }
51
+ let dot = 0;
52
+ for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
53
+ return dot;
54
+ }
55
+
56
+ export function similar(query, corpus, opts = {}) {
57
+ const q = embed(query);
58
+ return corpus
59
+ .map(text => ({ text, sim: cosineSim(q, embed(text)) }))
60
+ .sort((a, b) => b.sim - a.sim)
61
+ .slice(0, opts.topK ?? 5);
62
+ }
63
+
64
+ export function engineInfo() {
65
+ requireReady();
66
+ return config_summary();
67
+ }
@@ -0,0 +1 @@
1
+ { "type": "module" }
@@ -0,0 +1,72 @@
1
+ /* tslint:disable */
2
+ /* eslint-disable */
3
+
4
+ /**
5
+ * Debug helper: returns a human-readable summary of the loaded model's
6
+ * header (format version, embedding format, dimensions, vocab size).
7
+ */
8
+ export function config_summary(): string;
9
+
10
+ /**
11
+ * Primary entry point: text → 384-dim L2-normalized embedding.
12
+ *
13
+ * Tokenizes via the embedded BERT vocab, runs the full forward pass against
14
+ * the embedded `.bin`, returns the output vector.
15
+ */
16
+ export function embed(text: string): Float32Array;
17
+
18
+ /**
19
+ * Identity of the exact packed model embedded in this build.
20
+ */
21
+ export function model_sha256(): string;
22
+
23
+ /**
24
+ * Debug helper: returns token IDs the tokenizer produced. Used by per-stage
25
+ * parity tests to confirm tokenization matches the Python `tokenizers` library.
26
+ */
27
+ export function tokenize(text: string): Uint32Array;
28
+
29
+ /**
30
+ * Identity of the exact tokenizer embedded in this build.
31
+ */
32
+ export function tokenizer_sha256(): string;
33
+
34
+ export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module;
35
+
36
+ export interface InitOutput {
37
+ readonly memory: WebAssembly.Memory;
38
+ readonly config_summary: () => [number, number];
39
+ readonly embed: (a: number, b: number) => [number, number];
40
+ readonly model_sha256: () => [number, number];
41
+ readonly tokenize: (a: number, b: number) => [number, number];
42
+ readonly tokenizer_sha256: () => [number, number];
43
+ readonly __wbindgen_exn_store: (a: number) => void;
44
+ readonly __externref_table_alloc: () => number;
45
+ readonly __wbindgen_externrefs: WebAssembly.Table;
46
+ readonly __wbindgen_free: (a: number, b: number, c: number) => void;
47
+ readonly __wbindgen_malloc: (a: number, b: number) => number;
48
+ readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
49
+ readonly __wbindgen_start: () => void;
50
+ }
51
+
52
+ export type SyncInitInput = BufferSource | WebAssembly.Module;
53
+
54
+ /**
55
+ * Instantiates the given `module`, which can either be bytes or
56
+ * a precompiled `WebAssembly.Module`.
57
+ *
58
+ * @param {{ module: SyncInitInput }} module - Passing `SyncInitInput` directly is deprecated.
59
+ *
60
+ * @returns {InitOutput}
61
+ */
62
+ export function initSync(module: { module: SyncInitInput } | SyncInitInput): InitOutput;
63
+
64
+ /**
65
+ * If `module_or_path` is {RequestInfo} or {URL}, makes a request and
66
+ * for everything else, calls `WebAssembly.instantiate` directly.
67
+ *
68
+ * @param {{ module_or_path: InitInput | Promise<InitInput> }} module_or_path - Passing `InitInput` directly is deprecated.
69
+ *
70
+ * @returns {Promise<InitOutput>}
71
+ */
72
+ export default function __wbg_init (module_or_path?: { module_or_path: InitInput | Promise<InitInput> } | InitInput | Promise<InitInput>): Promise<InitOutput>;
@@ -0,0 +1,421 @@
1
+ /* @ts-self-types="./tern_engine.d.ts" */
2
+
3
+ /**
4
+ * Debug helper: returns a human-readable summary of the loaded model's
5
+ * header (format version, embedding format, dimensions, vocab size).
6
+ * @returns {string}
7
+ */
8
+ export function config_summary() {
9
+ let deferred1_0;
10
+ let deferred1_1;
11
+ try {
12
+ const ret = wasm.config_summary();
13
+ deferred1_0 = ret[0];
14
+ deferred1_1 = ret[1];
15
+ return getStringFromWasm0(ret[0], ret[1]);
16
+ } finally {
17
+ wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
18
+ }
19
+ }
20
+
21
+ /**
22
+ * Primary entry point: text → 384-dim L2-normalized embedding.
23
+ *
24
+ * Tokenizes via the embedded BERT vocab, runs the full forward pass against
25
+ * the embedded `.bin`, returns the output vector.
26
+ * @param {string} text
27
+ * @returns {Float32Array}
28
+ */
29
+ export function embed(text) {
30
+ const ptr0 = passStringToWasm0(text, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
31
+ const len0 = WASM_VECTOR_LEN;
32
+ const ret = wasm.embed(ptr0, len0);
33
+ var v2 = getArrayF32FromWasm0(ret[0], ret[1]).slice();
34
+ wasm.__wbindgen_free(ret[0], ret[1] * 4, 4);
35
+ return v2;
36
+ }
37
+
38
+ /**
39
+ * Identity of the exact packed model embedded in this build.
40
+ * @returns {string}
41
+ */
42
+ export function model_sha256() {
43
+ let deferred1_0;
44
+ let deferred1_1;
45
+ try {
46
+ const ret = wasm.model_sha256();
47
+ deferred1_0 = ret[0];
48
+ deferred1_1 = ret[1];
49
+ return getStringFromWasm0(ret[0], ret[1]);
50
+ } finally {
51
+ wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
52
+ }
53
+ }
54
+
55
+ /**
56
+ * Debug helper: returns token IDs the tokenizer produced. Used by per-stage
57
+ * parity tests to confirm tokenization matches the Python `tokenizers` library.
58
+ * @param {string} text
59
+ * @returns {Uint32Array}
60
+ */
61
+ export function tokenize(text) {
62
+ const ptr0 = passStringToWasm0(text, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
63
+ const len0 = WASM_VECTOR_LEN;
64
+ const ret = wasm.tokenize(ptr0, len0);
65
+ var v2 = getArrayU32FromWasm0(ret[0], ret[1]).slice();
66
+ wasm.__wbindgen_free(ret[0], ret[1] * 4, 4);
67
+ return v2;
68
+ }
69
+
70
+ /**
71
+ * Identity of the exact tokenizer embedded in this build.
72
+ * @returns {string}
73
+ */
74
+ export function tokenizer_sha256() {
75
+ let deferred1_0;
76
+ let deferred1_1;
77
+ try {
78
+ const ret = wasm.tokenizer_sha256();
79
+ deferred1_0 = ret[0];
80
+ deferred1_1 = ret[1];
81
+ return getStringFromWasm0(ret[0], ret[1]);
82
+ } finally {
83
+ wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
84
+ }
85
+ }
86
+ function __wbg_get_imports() {
87
+ const import0 = {
88
+ __proto__: null,
89
+ __wbg___wbindgen_is_function_754e9f305ff6029e: function(arg0) {
90
+ const ret = typeof(arg0) === 'function';
91
+ return ret;
92
+ },
93
+ __wbg___wbindgen_is_object_56732c2bc353f41d: function(arg0) {
94
+ const val = arg0;
95
+ const ret = typeof(val) === 'object' && val !== null;
96
+ return ret;
97
+ },
98
+ __wbg___wbindgen_is_string_c236cabd84a4d769: function(arg0) {
99
+ const ret = typeof(arg0) === 'string';
100
+ return ret;
101
+ },
102
+ __wbg___wbindgen_is_undefined_67b456be8673d3d7: function(arg0) {
103
+ const ret = arg0 === undefined;
104
+ return ret;
105
+ },
106
+ __wbg___wbindgen_throw_1506f2235d1bdba0: function(arg0, arg1) {
107
+ throw new Error(getStringFromWasm0(arg0, arg1));
108
+ },
109
+ __wbg_call_9c758de292015997: function() { return handleError(function (arg0, arg1, arg2) {
110
+ const ret = arg0.call(arg1, arg2);
111
+ return ret;
112
+ }, arguments); },
113
+ __wbg_crypto_38df2bab126b63dc: function(arg0) {
114
+ const ret = arg0.crypto;
115
+ return ret;
116
+ },
117
+ __wbg_getRandomValues_c44a50d8cfdaebeb: function() { return handleError(function (arg0, arg1) {
118
+ arg0.getRandomValues(arg1);
119
+ }, arguments); },
120
+ __wbg_length_4a591ecaa01354d9: function(arg0) {
121
+ const ret = arg0.length;
122
+ return ret;
123
+ },
124
+ __wbg_msCrypto_bd5a034af96bcba6: function(arg0) {
125
+ const ret = arg0.msCrypto;
126
+ return ret;
127
+ },
128
+ __wbg_new_with_length_36a4998e27b014c5: function(arg0) {
129
+ const ret = new Uint8Array(arg0 >>> 0);
130
+ return ret;
131
+ },
132
+ __wbg_node_84ea875411254db1: function(arg0) {
133
+ const ret = arg0.node;
134
+ return ret;
135
+ },
136
+ __wbg_process_44c7a14e11e9f69e: function(arg0) {
137
+ const ret = arg0.process;
138
+ return ret;
139
+ },
140
+ __wbg_prototypesetcall_3249fc62a0fafa30: function(arg0, arg1, arg2) {
141
+ Uint8Array.prototype.set.call(getArrayU8FromWasm0(arg0, arg1), arg2);
142
+ },
143
+ __wbg_randomFillSync_6c25eac9869eb53c: function() { return handleError(function (arg0, arg1) {
144
+ arg0.randomFillSync(arg1);
145
+ }, arguments); },
146
+ __wbg_require_b4edbdcf3e2a1ef0: function() { return handleError(function () {
147
+ const ret = module.require;
148
+ return ret;
149
+ }, arguments); },
150
+ __wbg_static_accessor_GLOBAL_9d53f2689e622ca1: function() {
151
+ const ret = typeof global === 'undefined' ? null : global;
152
+ return isLikeNone(ret) ? 0 : addToExternrefTable0(ret);
153
+ },
154
+ __wbg_static_accessor_GLOBAL_THIS_a1a35cec07001a8a: function() {
155
+ const ret = typeof globalThis === 'undefined' ? null : globalThis;
156
+ return isLikeNone(ret) ? 0 : addToExternrefTable0(ret);
157
+ },
158
+ __wbg_static_accessor_SELF_4c59f6c7ea29a144: function() {
159
+ const ret = typeof self === 'undefined' ? null : self;
160
+ return isLikeNone(ret) ? 0 : addToExternrefTable0(ret);
161
+ },
162
+ __wbg_static_accessor_WINDOW_e70ae9f2eb052253: function() {
163
+ const ret = typeof window === 'undefined' ? null : window;
164
+ return isLikeNone(ret) ? 0 : addToExternrefTable0(ret);
165
+ },
166
+ __wbg_subarray_4aa221f6a4f5ab22: function(arg0, arg1, arg2) {
167
+ const ret = arg0.subarray(arg1 >>> 0, arg2 >>> 0);
168
+ return ret;
169
+ },
170
+ __wbg_versions_276b2795b1c6a219: function(arg0) {
171
+ const ret = arg0.versions;
172
+ return ret;
173
+ },
174
+ __wbindgen_cast_0000000000000001: function(arg0, arg1) {
175
+ // Cast intrinsic for `Ref(Slice(U8)) -> NamedExternref("Uint8Array")`.
176
+ const ret = getArrayU8FromWasm0(arg0, arg1);
177
+ return ret;
178
+ },
179
+ __wbindgen_cast_0000000000000002: function(arg0, arg1) {
180
+ // Cast intrinsic for `Ref(String) -> Externref`.
181
+ const ret = getStringFromWasm0(arg0, arg1);
182
+ return ret;
183
+ },
184
+ __wbindgen_init_externref_table: function() {
185
+ const table = wasm.__wbindgen_externrefs;
186
+ const offset = table.grow(4);
187
+ table.set(0, undefined);
188
+ table.set(offset + 0, undefined);
189
+ table.set(offset + 1, null);
190
+ table.set(offset + 2, true);
191
+ table.set(offset + 3, false);
192
+ },
193
+ };
194
+ return {
195
+ __proto__: null,
196
+ "./tern_engine_bg.js": import0,
197
+ };
198
+ }
199
+
200
+ function addToExternrefTable0(obj) {
201
+ const idx = wasm.__externref_table_alloc();
202
+ wasm.__wbindgen_externrefs.set(idx, obj);
203
+ return idx;
204
+ }
205
+
206
+ function getArrayF32FromWasm0(ptr, len) {
207
+ ptr = ptr >>> 0;
208
+ return getFloat32ArrayMemory0().subarray(ptr / 4, ptr / 4 + len);
209
+ }
210
+
211
+ function getArrayU32FromWasm0(ptr, len) {
212
+ ptr = ptr >>> 0;
213
+ return getUint32ArrayMemory0().subarray(ptr / 4, ptr / 4 + len);
214
+ }
215
+
216
+ function getArrayU8FromWasm0(ptr, len) {
217
+ ptr = ptr >>> 0;
218
+ return getUint8ArrayMemory0().subarray(ptr / 1, ptr / 1 + len);
219
+ }
220
+
221
+ let cachedFloat32ArrayMemory0 = null;
222
+ function getFloat32ArrayMemory0() {
223
+ if (cachedFloat32ArrayMemory0 === null || cachedFloat32ArrayMemory0.byteLength === 0) {
224
+ cachedFloat32ArrayMemory0 = new Float32Array(wasm.memory.buffer);
225
+ }
226
+ return cachedFloat32ArrayMemory0;
227
+ }
228
+
229
+ function getStringFromWasm0(ptr, len) {
230
+ return decodeText(ptr >>> 0, len);
231
+ }
232
+
233
+ let cachedUint32ArrayMemory0 = null;
234
+ function getUint32ArrayMemory0() {
235
+ if (cachedUint32ArrayMemory0 === null || cachedUint32ArrayMemory0.byteLength === 0) {
236
+ cachedUint32ArrayMemory0 = new Uint32Array(wasm.memory.buffer);
237
+ }
238
+ return cachedUint32ArrayMemory0;
239
+ }
240
+
241
+ let cachedUint8ArrayMemory0 = null;
242
+ function getUint8ArrayMemory0() {
243
+ if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) {
244
+ cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer);
245
+ }
246
+ return cachedUint8ArrayMemory0;
247
+ }
248
+
249
+ function handleError(f, args) {
250
+ try {
251
+ return f.apply(this, args);
252
+ } catch (e) {
253
+ const idx = addToExternrefTable0(e);
254
+ wasm.__wbindgen_exn_store(idx);
255
+ }
256
+ }
257
+
258
+ function isLikeNone(x) {
259
+ return x === undefined || x === null;
260
+ }
261
+
262
+ function passStringToWasm0(arg, malloc, realloc) {
263
+ if (realloc === undefined) {
264
+ const buf = cachedTextEncoder.encode(arg);
265
+ const ptr = malloc(buf.length, 1) >>> 0;
266
+ getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf);
267
+ WASM_VECTOR_LEN = buf.length;
268
+ return ptr;
269
+ }
270
+
271
+ let len = arg.length;
272
+ let ptr = malloc(len, 1) >>> 0;
273
+
274
+ const mem = getUint8ArrayMemory0();
275
+
276
+ let offset = 0;
277
+
278
+ for (; offset < len; offset++) {
279
+ const code = arg.charCodeAt(offset);
280
+ if (code > 0x7F) break;
281
+ mem[ptr + offset] = code;
282
+ }
283
+ if (offset !== len) {
284
+ if (offset !== 0) {
285
+ arg = arg.slice(offset);
286
+ }
287
+ ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0;
288
+ const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len);
289
+ const ret = cachedTextEncoder.encodeInto(arg, view);
290
+
291
+ offset += ret.written;
292
+ ptr = realloc(ptr, len, offset, 1) >>> 0;
293
+ }
294
+
295
+ WASM_VECTOR_LEN = offset;
296
+ return ptr;
297
+ }
298
+
299
+ let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
300
+ cachedTextDecoder.decode();
301
+ const MAX_SAFARI_DECODE_BYTES = 2146435072;
302
+ let numBytesDecoded = 0;
303
+ function decodeText(ptr, len) {
304
+ numBytesDecoded += len;
305
+ if (numBytesDecoded >= MAX_SAFARI_DECODE_BYTES) {
306
+ cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
307
+ cachedTextDecoder.decode();
308
+ numBytesDecoded = len;
309
+ }
310
+ return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len));
311
+ }
312
+
313
+ const cachedTextEncoder = new TextEncoder();
314
+
315
+ if (!('encodeInto' in cachedTextEncoder)) {
316
+ cachedTextEncoder.encodeInto = function (arg, view) {
317
+ const buf = cachedTextEncoder.encode(arg);
318
+ view.set(buf);
319
+ return {
320
+ read: arg.length,
321
+ written: buf.length
322
+ };
323
+ };
324
+ }
325
+
326
+ let WASM_VECTOR_LEN = 0;
327
+
328
+ let wasmModule, wasmInstance, wasm;
329
+ function __wbg_finalize_init(instance, module) {
330
+ wasmInstance = instance;
331
+ wasm = instance.exports;
332
+ wasmModule = module;
333
+ cachedFloat32ArrayMemory0 = null;
334
+ cachedUint32ArrayMemory0 = null;
335
+ cachedUint8ArrayMemory0 = null;
336
+ wasm.__wbindgen_start();
337
+ return wasm;
338
+ }
339
+
340
+ async function __wbg_load(module, imports) {
341
+ if (typeof Response === 'function' && module instanceof Response) {
342
+ if (typeof WebAssembly.instantiateStreaming === 'function') {
343
+ try {
344
+ return await WebAssembly.instantiateStreaming(module, imports);
345
+ } catch (e) {
346
+ const validResponse = module.ok && expectedResponseType(module.type);
347
+
348
+ if (validResponse && module.headers.get('Content-Type') !== 'application/wasm') {
349
+ console.warn("`WebAssembly.instantiateStreaming` failed because your server does not serve Wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n", e);
350
+
351
+ } else { throw e; }
352
+ }
353
+ }
354
+
355
+ const bytes = await module.arrayBuffer();
356
+ return await WebAssembly.instantiate(bytes, imports);
357
+ } else {
358
+ const instance = await WebAssembly.instantiate(module, imports);
359
+
360
+ if (instance instanceof WebAssembly.Instance) {
361
+ return { instance, module };
362
+ } else {
363
+ return instance;
364
+ }
365
+ }
366
+
367
+ function expectedResponseType(type) {
368
+ switch (type) {
369
+ case 'basic': case 'cors': case 'default': return true;
370
+ }
371
+ return false;
372
+ }
373
+ }
374
+
375
+ function initSync(module) {
376
+ if (wasm !== undefined) return wasm;
377
+
378
+
379
+ if (module !== undefined) {
380
+ if (Object.getPrototypeOf(module) === Object.prototype) {
381
+ ({module} = module)
382
+ } else {
383
+ console.warn('using deprecated parameters for `initSync()`; pass a single object instead')
384
+ }
385
+ }
386
+
387
+ const imports = __wbg_get_imports();
388
+ if (!(module instanceof WebAssembly.Module)) {
389
+ module = new WebAssembly.Module(module);
390
+ }
391
+ const instance = new WebAssembly.Instance(module, imports);
392
+ return __wbg_finalize_init(instance, module);
393
+ }
394
+
395
+ async function __wbg_init(module_or_path) {
396
+ if (wasm !== undefined) return wasm;
397
+
398
+
399
+ if (module_or_path !== undefined) {
400
+ if (Object.getPrototypeOf(module_or_path) === Object.prototype) {
401
+ ({module_or_path} = module_or_path)
402
+ } else {
403
+ console.warn('using deprecated parameters for the initialization function; pass a single object instead')
404
+ }
405
+ }
406
+
407
+ if (module_or_path === undefined) {
408
+ module_or_path = new URL('tern_engine_bg.wasm', import.meta.url);
409
+ }
410
+ const imports = __wbg_get_imports();
411
+
412
+ if (typeof module_or_path === 'string' || (typeof Request === 'function' && module_or_path instanceof Request) || (typeof URL === 'function' && module_or_path instanceof URL)) {
413
+ module_or_path = fetch(module_or_path);
414
+ }
415
+
416
+ const { instance, module } = await __wbg_load(await module_or_path, imports);
417
+
418
+ return __wbg_finalize_init(instance, module);
419
+ }
420
+
421
+ export { initSync, __wbg_init as default };