@flaxia/node 0.5.0 → 0.5.1

package/dist/workloads/__tests__/model-pipeline-cache.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=model-pipeline-cache.test.d.ts.map

package/dist/workloads/__tests__/model-pipeline-cache.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"model-pipeline-cache.test.d.ts","sourceRoot":"","sources":["../../../src/workloads/__tests__/model-pipeline-cache.test.ts"],"names":[],"mappings":""}

package/dist/workloads/ai-inference.d.ts

@@ -1,10 +1,16 @@
 import type { AiInferencePayload, AiInferenceResult } from '@flaxia/sdk';
+import type { NodeModelCacheSnapshot } from '@flaxia/sdk';
 export { DEFAULT_AI_MODEL_MAX_BYTES, MAX_AI_MODEL_MAX_BYTES, MAX_AI_INPUT_CHARS, MAX_AI_INPUT_ITEMS, ALLOWED_AI_DEVICES, ALLOWED_AI_DTYPES, listAiModels, registerAiModels, unregisterAiModel, type AiModelDefinition, } from './ai-model-registry';
+/** Bound work queued behind TextStreamer's synchronous callback API. */
+export declare const MAX_PENDING_TOKEN_CALLBACKS = 64;
+export declare const MAX_PENDING_TOKEN_CHARS: number;
 /**
  * Drops all cached model pipelines so the (potentially hundreds of MB of)
  * transformers/onnxruntime memory can be garbage-collected. Called by the
  * worker after an idle period so a node does not pin every model it ever saw.
  */
 export declare const releaseCache: () => void;
+/** Current executable pipelines; weight files in the Transformers cache are a separate disk-cache layer. */
+export declare const getModelCacheSnapshot: () => NodeModelCacheSnapshot;
 export declare const handleAiInference: (payload: AiInferencePayload, onToken?: (token: string) => void | Promise<void>) => Promise<AiInferenceResult>;
 //# sourceMappingURL=ai-inference.d.ts.map

package/dist/workloads/ai-inference.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ai-inference.d.ts","sourceRoot":"","sources":["../../src/workloads/ai-inference.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,kBAAkB,EAAE,iBAAiB,EAAsB,MAAM,aAAa,CAAC;AAa7F,OAAO,EACL,0BAA0B,EAC1B,sBAAsB,EACtB,kBAAkB,EAClB,kBAAkB,EAClB,kBAAkB,EAClB,iBAAiB,EACjB,YAAY,EACZ,gBAAgB,EAChB,iBAAiB,EACjB,KAAK,iBAAiB,GACvB,MAAM,qBAAqB,CAAC;AAe7B;;;;GAIG;AACH,eAAO,MAAM,YAAY,QAAO,IAE/B,CAAC;AA0FF,eAAO,MAAM,iBAAiB,GAC5B,SAAS,kBAAkB,EAC3B,UAAU,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAChD,OAAO,CAAC,iBAAiB,CAqF3B,CAAC"}
+{"version":3,"file":"ai-inference.d.ts","sourceRoot":"","sources":["../../src/workloads/ai-inference.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,kBAAkB,EAAE,iBAAiB,EAAsB,MAAM,aAAa,CAAC;AAC7F,OAAO,KAAK,EAAqB,sBAAsB,EAAE,MAAM,aAAa,CAAC;AAc7E,OAAO,EACL,0BAA0B,EAC1B,sBAAsB,EACtB,kBAAkB,EAClB,kBAAkB,EAClB,kBAAkB,EAClB,iBAAiB,EACjB,YAAY,EACZ,gBAAgB,EAChB,iBAAiB,EACjB,KAAK,iBAAiB,GACvB,MAAM,qBAAqB,CAAC;AAa7B,wEAAwE;AACxE,eAAO,MAAM,2BAA2B,KAAK,CAAC;AAC9C,eAAO,MAAM,uBAAuB,QAAa,CAAC;AAElD;;;;GAIG;AACH,eAAO,MAAM,YAAY,QAAO,IAE/B,CAAC;AAEF,4GAA4G;AAC5G,eAAO,MAAM,qBAAqB,QAAO,sBAAqD,CAAC;AAkH/F,eAAO,MAAM,iBAAiB,GAC5B,SAAS,kBAAkB,EAC3B,UAAU,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAChD,OAAO,CAAC,iBAAiB,CA8G3B,CAAC"}

package/dist/workloads/model-pipeline-cache.d.ts

@@ -0,0 +1,48 @@
+import { type ModelCacheBackend, type ModelCacheEntryStatus, type NodeModelCacheSnapshot } from '@flaxia/sdk';
+export declare const DEFAULT_MODEL_PIPELINE_TTL_MS: number;
+export declare const DEFAULT_MODEL_PIPELINE_CACHE_CAPACITY = 2;
+export type ModelPipelineDescriptor = Omit<ModelCacheEntryStatus, 'state' | 'referenceCount'>;
+interface LoadedModelPipeline<T> {
+    value: T;
+    descriptor?: Partial<ModelPipelineDescriptor>;
+}
+export interface ModelPipelineLease<T> {
+    readonly value: T;
+    updateDescriptor(descriptor: Partial<ModelPipelineDescriptor>): void;
+    invalidate(): void;
+    release(): Promise<void>;
+}
+/**
+ * Bounded executable-model cache with load deduplication and lease-aware LRU.
+ * The caller decides what persistent weight cache to use; this class owns only
+ * executable pipelines/devices and never evicts one while a lease is active.
+ */
+export declare class ModelPipelineCache<T> {
+    private readonly options;
+    private entries;
+    constructor(options?: {
+        capacity?: number;
+        ttlMs?: number;
+        now?: () => number;
+    });
+    acquire(key: string, descriptor: ModelPipelineDescriptor, load: () => Promise<LoadedModelPipeline<T>>, dispose: (value: T) => void | Promise<void>): Promise<ModelPipelineLease<T>>;
+    private acquireEntry;
+    getSnapshot(now?: number): NodeModelCacheSnapshot;
+    releaseAll(): Promise<void>;
+    private createLease;
+    private evictExpired;
+    private makeRoom;
+    private disposeEntry;
+    private now;
+    private ttlMs;
+}
+export declare function modelPipelineKey(identity: {
+    task: string;
+    modelId: string;
+    revision: string;
+    dtype: string;
+    backend: ModelCacheBackend;
+    layerRange?: [number, number];
+}): string;
+export {};
+//# sourceMappingURL=model-pipeline-cache.d.ts.map

package/dist/workloads/model-pipeline-cache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"model-pipeline-cache.d.ts","sourceRoot":"","sources":["../../src/workloads/model-pipeline-cache.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,iBAAiB,EACtB,KAAK,qBAAqB,EAC1B,KAAK,sBAAsB,EAC5B,MAAM,aAAa,CAAC;AAErB,eAAO,MAAM,6BAA6B,QAAiB,CAAC;AAC5D,eAAO,MAAM,qCAAqC,IAAI,CAAC;AAEvD,MAAM,MAAM,uBAAuB,GAAG,IAAI,CAAC,qBAAqB,EAAE,OAAO,GAAG,gBAAgB,CAAC,CAAC;AAE9F,UAAU,mBAAmB,CAAC,CAAC;IAC7B,KAAK,EAAE,CAAC,CAAC;IACT,UAAU,CAAC,EAAE,OAAO,CAAC,uBAAuB,CAAC,CAAC;CAC/C;AAeD,MAAM,WAAW,kBAAkB,CAAC,CAAC;IACnC,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;IAClB,gBAAgB,CAAC,UAAU,EAAE,OAAO,CAAC,uBAAuB,CAAC,GAAG,IAAI,CAAC;IACrE,UAAU,IAAI,IAAI,CAAC;IACnB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED;;;;GAIG;AACH,qBAAa,kBAAkB,CAAC,CAAC;IAI7B,OAAO,CAAC,QAAQ,CAAC,OAAO;IAH1B,OAAO,CAAC,OAAO,CAAoC;gBAGhC,OAAO,GAAE;QACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;KACf;IAGF,OAAO,CACX,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,uBAAuB,EACnC,IAAI,EAAE,MAAM,OAAO,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,EAC3C,OAAO,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,GAC1C,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC;YA0DnB,YAAY;IAkB1B,WAAW,CAAC,GAAG,SAAa,GAAG,sBAAsB;IAa/C,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAcjC,OAAO,CAAC,WAAW;YA6BL,YAAY;YAUZ,QAAQ;YAaR,YAAY;IAa1B,OAAO,CAAC,GAAG;IAIX,OAAO,CAAC,KAAK;CAGd;AAED,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE;IACzC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,iBAAiB,CAAC;IAC3B,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC/B,GAAG,MAAM,CAST"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flaxia/node",
-  "version": "0.5.0",
+  "version": "0.5.1",
   "private": false,
   "main": "dist/index.js",
   "types": "./dist/index.d.ts",

package/dist/assets/dist.js

@@ -1,209 +0,0 @@
-//#region src/executor/memoryProbe.ts
-var e = 2 * 1024 ** 3, t = 65536, n = 16 * 1024 * 1024, r = 4096, i = null, a = 0;
-function o(e, n) {
-	let r = Math.ceil(n / t);
-	try {
-		e.grow(r);
-	} catch {
-		return !1;
-	}
-	let i = new Uint8Array(e.buffer);
-	for (let e = 0; e < r; e++) i[e * t] = 0;
-	return !0;
-}
-function s(s = e, c = n) {
-	if (i) return i.bytes;
-	a++;
-	let l;
-	try {
-		l = new WebAssembly.Memory({ initial: 0 });
-	} catch {
-		return i = { bytes: 0 }, 0;
-	}
-	let u = Math.max(0, Math.floor(s)), d = Math.max(t, Math.floor(c)), f = 0, p = 0;
-	for (; f < u && p < r;) {
-		let e = Math.min(d, u - f);
-		if (!o(l, e)) break;
-		f += e, p++;
-	}
-	return l = void 0, i = { bytes: f }, f;
-}
-function c() {
-	return i ? i.bytes : s();
-}
-function l() {
-	return c() >= e;
-}
-//#endregion
-//#region src/executor/webgpuProbe.ts
-async function u() {
-	let e = navigator.gpu;
-	if (!e || typeof e.requestAdapter != "function") return { webgpu: !1 };
-	try {
-		let t = await e.requestAdapter({ powerPreference: "high-performance" });
-		if (!t) return { webgpu: !1 };
-		let n = { webgpu: !0 }, r = t.info;
-		if (r) {
-			let e = [r.vendor, r.architecture].filter(Boolean).join(" ").trim();
-			e && (n.gpuArchitecture = e);
-		}
-		let i = t.limits?.maxStorageBufferBindingSize;
-		return typeof i == "number" && Number.isFinite(i) && (n.maxStorageBufferBindingSize = i), n;
-	} catch {
-		return { webgpu: !1 };
-	}
-}
-//#endregion
-//#region ../sdk/dist/index.js
-var d = "flaxia-node-v1", f = "bearer.";
-function p(e) {
-	return [d, `${f}${e}`];
-}
-function m(e, t, n = {}) {
-	let r = e.replace(/\/+$/, "").replace(/^http/, "ws"), i = new URL(`${r}${t.startsWith("/") ? t : `/${t}`}`);
-	for (let [e, t] of Object.entries(n)) t !== void 0 && i.searchParams.set(e, String(t));
-	return i.toString();
-}
-var h = new Set([
-	"ai-inference",
-	"image-process",
-	"container",
-	"vector-embed",
-	"vector-query",
-	"nudenet",
-	"swarm-inference"
-]);
-function g(e, t) {
-	if (t.length === 0) return [];
-	if (!Number.isInteger(e) || e < 1) throw Error(`layers must be a positive integer, got ${e}`);
-	let n = Math.min(t.length, e), r = t.slice(0, n).map((e) => Number.isFinite(e) && e > 0 ? e : 0), i = r.reduce((e, t) => e + t, 0) || 1, a = r.map((t) => Math.max(1, Math.floor(e * t / i))), o = a.reduce((e, t) => e + t, 0);
-	for (; o > e;) {
-		let e = 0;
-		for (let t = 1; t < n; t++) a[t] > a[e] && (e = t);
-		if (a[e] <= 1) break;
-		a[e]--, o--;
-	}
-	let s = r.map((t, n) => ({
-		i: n,
-		f: e * t / i - Math.floor(e * t / i)
-	})).sort((e, t) => t.f - e.f), c = 0;
-	for (; o < e;) a[s[c % n].i]++, o++, c++;
-	return a;
-}
-function _(e) {
-	let t = g(e.layers, e.nodes.map((e) => e.capacity)), n = [], r = 0;
-	for (let i = 0; i < t.length; i++) {
-		let a = r + t[i], o = i === 0 ? "host" : "worker", s = {
-			start: r,
-			end: a,
-			hasEmbed: i === 0,
-			hasHead: i === 0
-		};
-		n.push({
-			nodeId: e.nodes[i].nodeId,
-			role: o,
-			slice: s
-		}), r = a;
-	}
-	return {
-		sessionId: e.sessionId,
-		taskId: e.taskId,
-		model: e.model,
-		layers: e.layers,
-		chain: n
-	};
-}
-var v = 21335, y = 0, b = 1;
-function x(e, t, n) {
-	let r = new TextEncoder().encode(e), i = n ?? new Uint8Array(), a = 4 + r.length + 1 + i.byteLength, o = new ArrayBuffer(a + t.byteLength), s = new DataView(o);
-	return s.setUint32(0, r.length, !0), new Uint8Array(o, 4, r.length).set(r), s.setUint8(4 + r.length, i.byteLength), i.byteLength && new Uint8Array(o, 4 + r.length + 1, i.byteLength).set(i), new Uint8Array(o, a).set(new Uint8Array(t)), o;
-}
-function S(e) {
-	if (e.byteLength < 5) return null;
-	let t = new DataView(e), n = t.getUint32(0, !0);
-	if (!n || n > 256) return null;
-	let r = 4 + n;
-	if (e.byteLength < r + 1) return null;
-	let i = t.getUint8(r);
-	if (i !== 0 && i !== 16) return null;
-	let a = r + 1 + i;
-	return e.byteLength < a + 16 ? null : {
-		sessionId: new TextDecoder().decode(new Uint8Array(e, 4, n)),
-		mac: i ? new Uint8Array(e, r + 1, i) : null,
-		frame: e.slice(a)
-	};
-}
-async function C(e, t, n) {
-	let r = await crypto.subtle.importKey("raw", new TextEncoder().encode(e), {
-		name: "HMAC",
-		hash: "SHA-256"
-	}, !1, ["sign"]), i = await crypto.subtle.sign("HMAC", r, x(t, n));
-	return new Uint8Array(i).slice(0, 16);
-}
-async function w(e, t, n) {
-	return x(t, n, await C(e, t, n));
-}
-async function T(e, t, n, r) {
-	if (r.byteLength !== 16) return !1;
-	let i = await C(e, t, n), a = 0;
-	for (let e = 0; e < i.byteLength; e++) a |= i[e] ^ r[e];
-	return a === 0;
-}
-function E(e, t) {
-	let n = new ArrayBuffer(16 + t.byteLength), r = new DataView(n);
-	return r.setUint16(0, v, !0), r.setUint8(2, y), r.setUint8(3, 0), r.setUint32(4, e.requestId >>> 0, !0), r.setUint32(8, e.pos >>> 0, !0), r.setUint16(12, e.tokens, !0), r.setUint16(14, 0, !0), new Uint8Array(n, 16).set(t), n;
-}
-function D(e = 0) {
-	let t = /* @__PURE__ */ new ArrayBuffer(16), n = new DataView(t);
-	return n.setUint16(0, v, !0), n.setUint8(2, b), n.setUint8(3, 0), n.setUint32(4, e >>> 0, !0), t;
-}
-function O(e) {
-	if (e.byteLength !== 16) return !1;
-	let t = new DataView(e);
-	return t.getUint16(0, !0) === 21335 && t.getUint8(2) === b && t.getUint8(3) === 0 && t.getUint32(8, !0) === 0 && t.getUint16(12, !0) === 0 && t.getUint16(14, !0) === 0;
-}
-function k(e) {
-	if (e.byteLength < 16) return null;
-	let t = new DataView(e);
-	return t.getUint16(0, !0) !== 21335 || t.getUint8(2) !== y ? null : {
-		header: {
-			requestId: t.getUint32(4, !0),
-			pos: t.getUint32(8, !0),
-			tokens: t.getUint16(12, !0)
-		},
-		payload: new Uint8Array(e, 16)
-	};
-}
-function A(e) {
-	return typeof e == "object" && !!e;
-}
-function j(e) {
-	return typeof e == "number" && Number.isInteger(e) && e >= 0;
-}
-function M(e, t = {}) {
-	let n = t.maxLayers ?? 1024, r = t.maxSliceLayers ?? 256;
-	return !(!N(e) || !j(e.start) || !j(e.end) || e.end <= e.start || e.end - e.start > r || e.end > n);
-}
-function N(e) {
-	return A(e) ? typeof e.start == "number" && typeof e.end == "number" && typeof e.hasEmbed == "boolean" && typeof e.hasHead == "boolean" : !1;
-}
-function P(e) {
-	return A(e) ? typeof e.nodeId == "string" && e.nodeId.length > 0 && typeof e.capacity == "number" : !1;
-}
-function F(e) {
-	return e === void 0 || typeof e == "string" && e.length > 0 && e.length <= 128;
-}
-function I(e) {
-	return !A(e) || e.type !== "swarm-init" || typeof e.sessionId != "string" || !e.sessionId || typeof e.taskId != "string" || !e.taskId || typeof e.model != "string" || typeof e.timeoutMs != "number" || !Array.isArray(e.members) || e.members.length === 0 || e.members.length > 16 || typeof e.prompt != "string" && !Array.isArray(e.prompt) ? !1 : e.members.every(P);
-}
-function L(e) {
-	return !(!A(e) || e.type !== "swarm-slice" || typeof e.sessionId != "string" || !e.sessionId || typeof e.taskId != "string" || !e.taskId || typeof e.model != "string" || !e.model || typeof e.timeoutMs != "number" || !j(e.index) || !j(e.chainLength) || e.chainLength < 1 || e.chainLength > 16 || e.index >= e.chainLength || e.role !== (e.index === 0 ? "host" : "worker") || !M(e.slice) || e.slice.hasEmbed !== (e.index === 0) || e.slice.hasHead !== (e.index === 0) || !F(e.inboundKey) || !F(e.outboundKey));
-}
-function R(e) {
-	return !A(e) || e.type !== "swarm-start" ? !1 : typeof e.sessionId == "string" && !!e.sessionId && typeof e.taskId == "string" && !!e.taskId;
-}
-function z(e) {
-	return !A(e) || e.type !== "swarm-error" ? !1 : typeof e.sessionId == "string" && !!e.sessionId && typeof e.taskId == "string" && !!e.taskId && typeof e.error == "string";
-}
-//#endregion
-export { O as _, z as a, l as b, E as c, m as d, T as f, R as g, D as h, I as i, k as l, S as m, L as n, M as o, w as p, x as r, h as s, _ as t, p as u, u as v, s as x, e as y };