@flaxia/node 0.5.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/ai-inference.js +129 -83
- package/dist/assets/transformers.web.js +2 -2
- package/dist/assets/vector-embed.js +1 -1
- package/dist/assets/webgpuProbe.js +91 -83
- package/dist/client/SignalingClient.d.ts.map +1 -1
- package/dist/client/__tests__/light-preloader.test.d.ts +2 -0
- package/dist/client/__tests__/light-preloader.test.d.ts.map +1 -0
- package/dist/client/light-preloader.d.ts +47 -0
- package/dist/client/light-preloader.d.ts.map +1 -0
- package/dist/executor/WorkerPool.d.ts.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +443 -356
- package/dist/worker.js +177 -168
- package/dist/workloads/ai-inference.d.ts +21 -0
- package/dist/workloads/ai-inference.d.ts.map +1 -1
- package/dist/workloads/ai-model-registry.d.ts +6 -0
- package/dist/workloads/ai-model-registry.d.ts.map +1 -1
- package/package.json +2 -2
|
@@ -1,28 +1,34 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { t as e } from "./chunk.js";
|
|
2
|
+
import { E as t, x as n } from "./webgpuProbe.js";
|
|
3
|
+
import { n as r, r as i, t as a } from "./transformers.web.js";
|
|
3
4
|
//#region src/workloads/ai-model-registry.ts
|
|
4
|
-
var
|
|
5
|
+
var o = 512 * 1024 * 1024, s = 4 * 1024 ** 3, c = new Map(Object.entries({
|
|
5
6
|
"Xenova/bert-base-multilingual-uncased-sentiment": {
|
|
6
7
|
repo: "Xenova/bert-base-multilingual-uncased-sentiment",
|
|
7
8
|
revision: "main",
|
|
8
|
-
maxBytes:
|
|
9
|
+
maxBytes: o
|
|
9
10
|
},
|
|
10
11
|
"onnx-community/Qwen3-Embedding-0.6B-ONNX": {
|
|
11
12
|
repo: "onnx-community/Qwen3-Embedding-0.6B-ONNX",
|
|
12
13
|
revision: "main",
|
|
13
|
-
maxBytes:
|
|
14
|
+
maxBytes: o
|
|
15
|
+
},
|
|
16
|
+
"Xenova/m2m100_418M": {
|
|
17
|
+
repo: "Xenova/m2m100_418M",
|
|
18
|
+
revision: "main",
|
|
19
|
+
maxBytes: o
|
|
14
20
|
}
|
|
15
|
-
}).map(([e, t]) => [e, { ...t }])),
|
|
21
|
+
}).map(([e, t]) => [e, { ...t }])), l = [
|
|
16
22
|
"wasm",
|
|
17
23
|
"cpu",
|
|
18
24
|
"webgpu"
|
|
19
|
-
],
|
|
25
|
+
], u = [
|
|
20
26
|
"q4",
|
|
21
27
|
"q4f16",
|
|
22
28
|
"q8",
|
|
23
29
|
"int8",
|
|
24
30
|
"uint8"
|
|
25
|
-
],
|
|
31
|
+
], d = {
|
|
26
32
|
max_new_tokens: {
|
|
27
33
|
min: 1,
|
|
28
34
|
max: 1024
|
|
@@ -43,45 +49,50 @@ var i = 512 * 1024 * 1024, a = 4 * 1024 ** 3, o = new Map(Object.entries({
|
|
|
43
49
|
min: 0,
|
|
44
50
|
max: 4
|
|
45
51
|
}
|
|
46
|
-
},
|
|
47
|
-
function
|
|
48
|
-
return [...
|
|
52
|
+
}, f = 1e5;
|
|
53
|
+
function p() {
|
|
54
|
+
return [...c.keys()];
|
|
49
55
|
}
|
|
50
|
-
function
|
|
56
|
+
function m(e) {
|
|
51
57
|
if (typeof e != "string" || !e) throw Error(`ai-inference: model must be a non-empty string, got ${JSON.stringify(e)}`);
|
|
52
|
-
let t =
|
|
53
|
-
if (!t) throw Error(`ai-inference: model not allowed: ${e} (allowed: ${
|
|
58
|
+
let t = c.get(e);
|
|
59
|
+
if (!t) throw Error(`ai-inference: model not allowed: ${e} (allowed: ${p().join(", ") || "none"})`);
|
|
54
60
|
return t;
|
|
55
61
|
}
|
|
56
|
-
function
|
|
62
|
+
function h(e) {
|
|
57
63
|
if (e != null) {
|
|
58
|
-
if (typeof e != "string" || !
|
|
64
|
+
if (typeof e != "string" || !l.includes(e)) throw Error(`ai-inference: device not allowed: ${JSON.stringify(e)} (allowed: ${l.join(", ")})`);
|
|
59
65
|
return e;
|
|
60
66
|
}
|
|
61
67
|
}
|
|
62
|
-
function
|
|
68
|
+
function g(e) {
|
|
63
69
|
if (e != null) {
|
|
64
|
-
if (typeof e != "string" || !
|
|
70
|
+
if (typeof e != "string" || !u.includes(e)) throw Error(`ai-inference: dtype not allowed: ${JSON.stringify(e)} (allowed: ${u.join(", ")})`);
|
|
65
71
|
return e;
|
|
66
72
|
}
|
|
67
73
|
}
|
|
68
|
-
function
|
|
74
|
+
function _(e) {
|
|
75
|
+
if (e == null) return !1;
|
|
76
|
+
if (typeof e != "boolean") throw Error(`ai-inference: option ondeviceOnly must be a boolean, got ${JSON.stringify(e)}`);
|
|
77
|
+
return e;
|
|
78
|
+
}
|
|
79
|
+
function v(e) {
|
|
69
80
|
if (typeof e == "string") {
|
|
70
|
-
if (e.length > 1e5) throw Error(`ai-inference: input exceeds ${
|
|
81
|
+
if (e.length > 1e5) throw Error(`ai-inference: input exceeds ${f} characters`);
|
|
71
82
|
return e;
|
|
72
83
|
}
|
|
73
84
|
if (Array.isArray(e)) {
|
|
74
85
|
if (e.length === 0 || e.length > 64) throw Error("ai-inference: input array must hold 1..64 items");
|
|
75
86
|
for (let t of e) {
|
|
76
87
|
if (typeof t != "string") throw Error("ai-inference: input array items must be strings");
|
|
77
|
-
if (t.length > 1e5) throw Error(`ai-inference: input exceeds ${
|
|
88
|
+
if (t.length > 1e5) throw Error(`ai-inference: input exceeds ${f} characters`);
|
|
78
89
|
}
|
|
79
90
|
return e;
|
|
80
91
|
}
|
|
81
92
|
throw Error("ai-inference: input must be a string or an array of strings");
|
|
82
93
|
}
|
|
83
|
-
function
|
|
84
|
-
for (let [n, r] of Object.entries(
|
|
94
|
+
function y(e, t) {
|
|
95
|
+
for (let [n, r] of Object.entries(d)) {
|
|
85
96
|
let i = t[n];
|
|
86
97
|
if (i != null) {
|
|
87
98
|
if (typeof i != "number" || !Number.isFinite(i) || i < r.min || i > r.max) throw Error(`ai-inference: option ${n} must be a number within ${r.min}..${r.max}, got ${JSON.stringify(i)}`);
|
|
@@ -100,18 +111,18 @@ function g(e, t) {
|
|
|
100
111
|
}
|
|
101
112
|
}
|
|
102
113
|
}
|
|
103
|
-
var
|
|
114
|
+
var b = class {
|
|
104
115
|
constructor(e = {}) {
|
|
105
116
|
this.options = e, this.entries = /* @__PURE__ */ new Map();
|
|
106
117
|
}
|
|
107
|
-
async acquire(
|
|
108
|
-
let a = this.entries.get(
|
|
109
|
-
if (a?.state === "ready" && a.references === 0 && a.expiresAt <= this.now() && (await this.disposeEntry(a), a = this.entries.get(
|
|
118
|
+
async acquire(e, t, r, i) {
|
|
119
|
+
let a = this.entries.get(e);
|
|
120
|
+
if (a?.state === "ready" && a.references === 0 && a.expiresAt <= this.now() && (await this.disposeEntry(a), a = this.entries.get(e)), a?.state === "failed" && (this.entries.delete(e), a = void 0), a || (await this.evictExpired(), a = this.entries.get(e), a)) return this.acquireEntry(a);
|
|
110
121
|
let o = Math.max(1, Math.floor(this.options.capacity ?? 2));
|
|
111
|
-
if (this.entries.size >= o && await this.makeRoom(), a = this.entries.get(
|
|
122
|
+
if (this.entries.size >= o && await this.makeRoom(), a = this.entries.get(e), a) return this.acquireEntry(a);
|
|
112
123
|
let s = this.now(), c = {
|
|
113
|
-
key:
|
|
114
|
-
descriptor: { ...
|
|
124
|
+
key: e,
|
|
125
|
+
descriptor: { ...t },
|
|
115
126
|
state: "loading",
|
|
116
127
|
references: 1,
|
|
117
128
|
updatedAt: s,
|
|
@@ -120,15 +131,15 @@ var _ = class {
|
|
|
120
131
|
promise: Promise.resolve(void 0),
|
|
121
132
|
dispose: i
|
|
122
133
|
};
|
|
123
|
-
this.entries.set(
|
|
134
|
+
this.entries.set(e, c), c.promise = (async () => {
|
|
124
135
|
try {
|
|
125
136
|
let e = await r();
|
|
126
137
|
return c.value = e.value, c.descriptor = {
|
|
127
138
|
...c.descriptor,
|
|
128
139
|
...e.descriptor
|
|
129
140
|
}, c.state === "loading" && (c.state = "ready"), c.updatedAt = this.now(), c.expiresAt = this.now() + this.ttlMs(), e.value;
|
|
130
|
-
} catch (
|
|
131
|
-
throw c.state = "failed", c.updatedAt = this.now(), c.expiresAt = this.now() + Math.min(this.ttlMs(),
|
|
141
|
+
} catch (e) {
|
|
142
|
+
throw c.state = "failed", c.updatedAt = this.now(), c.expiresAt = this.now() + Math.min(this.ttlMs(), n), e;
|
|
132
143
|
}
|
|
133
144
|
})();
|
|
134
145
|
try {
|
|
@@ -149,11 +160,11 @@ var _ = class {
|
|
|
149
160
|
throw e.references = Math.max(0, e.references - 1), t;
|
|
150
161
|
}
|
|
151
162
|
}
|
|
152
|
-
getSnapshot(
|
|
163
|
+
getSnapshot(e = this.now()) {
|
|
153
164
|
return {
|
|
154
165
|
version: 1,
|
|
155
|
-
sampledAt:
|
|
156
|
-
expiresAt:
|
|
166
|
+
sampledAt: e,
|
|
167
|
+
expiresAt: e + n,
|
|
157
168
|
entries: [...this.entries.values()].slice(0, 8).map((e) => ({
|
|
158
169
|
...e.descriptor,
|
|
159
170
|
state: e.state,
|
|
@@ -220,7 +231,7 @@ var _ = class {
|
|
|
220
231
|
return Math.max(1, this.options.ttlMs ?? 18e5);
|
|
221
232
|
}
|
|
222
233
|
};
|
|
223
|
-
function
|
|
234
|
+
function x(e) {
|
|
224
235
|
return JSON.stringify([
|
|
225
236
|
e.task,
|
|
226
237
|
e.modelId,
|
|
@@ -232,7 +243,21 @@ function v(e) {
|
|
|
232
243
|
}
|
|
233
244
|
//#endregion
|
|
234
245
|
//#region src/workloads/ai-inference.ts
|
|
235
|
-
var
|
|
246
|
+
var S = /* @__PURE__ */ e({
|
|
247
|
+
DEFAULT_AI_MODEL_MAX_BYTES: () => o,
|
|
248
|
+
LIGHT_INFERENCE_TRANSLATE_DTYPE: () => k,
|
|
249
|
+
LIGHT_INFERENCE_TRANSLATE_MODEL: () => D,
|
|
250
|
+
LIGHT_INFERENCE_TRANSLATE_TASK: () => O,
|
|
251
|
+
MAX_AI_INPUT_CHARS: () => f,
|
|
252
|
+
MAX_AI_INPUT_ITEMS: () => 64,
|
|
253
|
+
MAX_PENDING_TOKEN_CALLBACKS: () => 64,
|
|
254
|
+
MAX_PENDING_TOKEN_CHARS: () => T,
|
|
255
|
+
getModelCacheSnapshot: () => j,
|
|
256
|
+
handleAiInference: () => P,
|
|
257
|
+
isAiModelPreloaded: () => E,
|
|
258
|
+
preloadAiModel: () => F,
|
|
259
|
+
releaseCache: () => A
|
|
260
|
+
}), C = [
|
|
236
261
|
"text-classification",
|
|
237
262
|
"token-classification",
|
|
238
263
|
"question-answering",
|
|
@@ -258,9 +283,9 @@ var y = [
|
|
|
258
283
|
"depth-estimation",
|
|
259
284
|
"feature-extraction",
|
|
260
285
|
"image-feature-extraction"
|
|
261
|
-
],
|
|
262
|
-
|
|
263
|
-
},
|
|
286
|
+
], w = new b(), T = 256 * 1024, E = (e) => j().entries.some((t) => t.modelId === e && t.state === "ready"), D = "Xenova/m2m100_418M", O = "translation", k = "q4f16", A = () => {
|
|
287
|
+
w.releaseAll();
|
|
288
|
+
}, j = () => w.getSnapshot(), M = (e, t) => {
|
|
264
289
|
let n = "", r = null, i = Promise.resolve(), a = 0, o = 0, s, c = !1, l = !1, u = (t) => {
|
|
265
290
|
if (a >= 64 || o + t.length > 262144) throw c = !0, s = /* @__PURE__ */ Error("ai-inference token consumer is too slow; generation stopped at the bounded stream buffer"), s;
|
|
266
291
|
a++, o += t.length, i = i.then(() => {
|
|
@@ -299,74 +324,95 @@ var y = [
|
|
|
299
324
|
cancel: () => p(!1)
|
|
300
325
|
};
|
|
301
326
|
};
|
|
302
|
-
function
|
|
327
|
+
function N() {
|
|
303
328
|
let e = globalThis.fetch.bind(globalThis);
|
|
304
|
-
|
|
329
|
+
r.fetch = async (t, n) => {
|
|
305
330
|
let r = new URL(String(t));
|
|
306
331
|
if (!/(^|\.)huggingface\.co$/.test(r.hostname) && !/(^|\.)hf\.co$/.test(r.hostname)) throw Error(`ai-inference: model download blocked from ${r.hostname}`);
|
|
307
|
-
let i = await e(r, n),
|
|
308
|
-
if (Number.isFinite(
|
|
332
|
+
let i = await e(r, n), a = Number(i.headers.get("content-length"));
|
|
333
|
+
if (Number.isFinite(a) && a > s) throw Error(`ai-inference: model file too large (${a} bytes > ${s})`);
|
|
309
334
|
return i;
|
|
310
335
|
};
|
|
311
336
|
}
|
|
312
|
-
|
|
313
|
-
var
|
|
314
|
-
let { task:
|
|
315
|
-
if (!
|
|
316
|
-
let
|
|
317
|
-
|
|
318
|
-
let
|
|
319
|
-
task:
|
|
320
|
-
modelId:
|
|
321
|
-
revision:
|
|
322
|
-
dtype:
|
|
323
|
-
backend:
|
|
324
|
-
}),
|
|
325
|
-
modelId:
|
|
326
|
-
revision:
|
|
327
|
-
dtype:
|
|
328
|
-
backend:
|
|
337
|
+
N(), r.backends?.onnx?.wasm && (r.backends.onnx.wasm.numThreads = 1, r.backends.onnx.wasm.proxy = !1);
|
|
338
|
+
var P = async (e, n) => {
|
|
339
|
+
let { task: r, model: o, input: s, options: c = {} } = e;
|
|
340
|
+
if (!C.includes(r)) throw Error(`Invalid or unsupported task: ${r}. Supported tasks are: ${C.join(", ")}`);
|
|
341
|
+
let l = m(o), u = h(c.device), d = g(c.dtype), f = _(c.ondeviceOnly), p = v(s), b = {};
|
|
342
|
+
if (y(b, c), f && !E(o)) throw new t(`ondeviceOnly: model ${o} is not preloaded on this device`, "preloading");
|
|
343
|
+
let S = d ?? "q4f16", T = u ?? "wasm", D = x({
|
|
344
|
+
task: r,
|
|
345
|
+
modelId: o,
|
|
346
|
+
revision: l.revision,
|
|
347
|
+
dtype: S,
|
|
348
|
+
backend: T
|
|
349
|
+
}), O = performance.now(), k = await w.acquire(D, {
|
|
350
|
+
modelId: o,
|
|
351
|
+
revision: l.revision,
|
|
352
|
+
dtype: S,
|
|
353
|
+
backend: T
|
|
329
354
|
}, async () => {
|
|
330
355
|
let e = {
|
|
331
|
-
dtype:
|
|
332
|
-
device:
|
|
333
|
-
revision:
|
|
356
|
+
dtype: S,
|
|
357
|
+
device: u,
|
|
358
|
+
revision: l.revision
|
|
334
359
|
};
|
|
335
|
-
console.log(`[flaxia-node] ai-inference: loading pipeline task=${
|
|
360
|
+
console.log(`[flaxia-node] ai-inference: loading pipeline task=${r} model=${l.repo} revision=${l.revision} dtype=${e.dtype} device=${u ?? "wasm"}`);
|
|
336
361
|
let t;
|
|
337
362
|
try {
|
|
338
|
-
t = await r
|
|
363
|
+
t = await i(r, l.repo, e);
|
|
339
364
|
} catch (n) {
|
|
340
|
-
if (
|
|
341
|
-
else throw console.error(`[flaxia-node] ai-inference: pipeline load FAILED task=${
|
|
365
|
+
if (u && u !== "wasm") console.warn(`[AiInference] ${u} failed, falling back to wasm:`, n), e.device = "wasm", t = await i(r, l.repo, e);
|
|
366
|
+
else throw console.error(`[flaxia-node] ai-inference: pipeline load FAILED task=${r} model=${l.repo} error=${n instanceof Error ? n.message : String(n)}`), n;
|
|
342
367
|
}
|
|
343
|
-
return console.log(`[flaxia-node] ai-inference: pipeline ready task=${
|
|
368
|
+
return console.log(`[flaxia-node] ai-inference: pipeline ready task=${r} model=${l.repo} loadMs=${Math.round(performance.now() - O)}`), {
|
|
344
369
|
value: t,
|
|
345
370
|
descriptor: { backend: e.device ?? "wasm" }
|
|
346
371
|
};
|
|
347
|
-
}, (e) => e.dispose?.()),
|
|
372
|
+
}, (e) => e.dispose?.()), A = k.value;
|
|
348
373
|
try {
|
|
349
374
|
let e = {
|
|
350
|
-
max_new_tokens:
|
|
351
|
-
do_sample:
|
|
375
|
+
max_new_tokens: b.max_new_tokens ?? 128,
|
|
376
|
+
do_sample: b.do_sample ?? !1
|
|
352
377
|
};
|
|
353
|
-
for (let [t, n] of Object.entries(
|
|
354
|
-
let
|
|
355
|
-
|
|
378
|
+
for (let [t, n] of Object.entries(b)) t === "max_new_tokens" || t === "do_sample" || (e[t] = n);
|
|
379
|
+
let t = n && A.tokenizer ? M(n, c) : null;
|
|
380
|
+
t && (e.streamer = new a(A.tokenizer, {
|
|
356
381
|
skip_prompt: !0,
|
|
357
382
|
skip_special_tokens: !0,
|
|
358
|
-
callback_function:
|
|
383
|
+
callback_function: t.callback
|
|
359
384
|
}));
|
|
360
|
-
let
|
|
385
|
+
let i = performance.now(), o;
|
|
361
386
|
try {
|
|
362
|
-
o = await
|
|
387
|
+
o = await A(p, e);
|
|
363
388
|
} catch (e) {
|
|
364
|
-
throw /out of memory|oom|allocation failed|could not allocate/i.test(e instanceof Error ? e.message : String(e)) &&
|
|
389
|
+
throw /out of memory|oom|allocation failed|could not allocate/i.test(e instanceof Error ? e.message : String(e)) && k.invalidate(), t && await t.cancel(), e;
|
|
365
390
|
}
|
|
366
|
-
return await
|
|
391
|
+
return await t?.finish(), console.log(`[flaxia-node] ai-inference: done task=${r} model=${l.repo} execMs=${Math.round(performance.now() - i)} inputLen=${String(p).length}`), { output: o };
|
|
367
392
|
} finally {
|
|
368
|
-
await
|
|
393
|
+
await k.release();
|
|
369
394
|
}
|
|
395
|
+
}, F = async (e) => {
|
|
396
|
+
let { task: t, model: n } = e, r = m(n), a = h(e.device), o = g(e.dtype) ?? "q4f16", s = a ?? "wasm", c = x({
|
|
397
|
+
task: t,
|
|
398
|
+
modelId: n,
|
|
399
|
+
revision: r.revision,
|
|
400
|
+
dtype: o,
|
|
401
|
+
backend: s
|
|
402
|
+
});
|
|
403
|
+
await (await w.acquire(c, {
|
|
404
|
+
modelId: n,
|
|
405
|
+
revision: r.revision,
|
|
406
|
+
dtype: o,
|
|
407
|
+
backend: s
|
|
408
|
+
}, async () => (console.log(`[flaxia-node] ai-inference: preloading task=${t} model=${r.repo}`), {
|
|
409
|
+
value: await i(t, r.repo, {
|
|
410
|
+
dtype: o,
|
|
411
|
+
device: a,
|
|
412
|
+
revision: r.revision
|
|
413
|
+
}),
|
|
414
|
+
descriptor: { backend: s }
|
|
415
|
+
}), (e) => e.dispose?.())).release();
|
|
370
416
|
};
|
|
371
417
|
//#endregion
|
|
372
|
-
export { S as
|
|
418
|
+
export { E as a, S as i, D as n, F as o, O as r, k as t };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
[diffend] Oversized file quarantined before diffing.
|
|
2
2
|
name: package/dist/assets/transformers.web.js
|
|
3
|
-
size:
|
|
4
|
-
sha256:
|
|
3
|
+
size: 63528578 bytes
|
|
4
|
+
sha256: c4ea4c2cc11efabbcbd71ff8631ac3d08593a8cde1a4a2a64518ac81f837f4b0
|
|
@@ -7,7 +7,7 @@ async function n(t) {
|
|
|
7
7
|
if (!e) {
|
|
8
8
|
let t = performance.now();
|
|
9
9
|
console.log("[flaxia-node] vector-embed: loading pipeline model=onnx-community/Qwen3-Embedding-0.6B-ONNX device=wasm");
|
|
10
|
-
let { pipeline: n, env: r } = await import("./transformers.web.js");
|
|
10
|
+
let { pipeline: n, env: r } = await import("./transformers.web.js").then((e) => e.i);
|
|
11
11
|
r?.backends?.onnx?.wasm && (r.backends.onnx.wasm.numThreads = 1, r.backends.onnx.wasm.proxy = !1);
|
|
12
12
|
try {
|
|
13
13
|
e = await n("feature-extraction", "onnx-community/Qwen3-Embedding-0.6B-ONNX", { device: "wasm" });
|