@framefields/vision 0.0.0-stage → 2.0.2

package/dist/web.mjs

@@ -0,0 +1,131 @@
+import { n as __reExport, t as __exportAll } from "./chunk-CFhWmker.mjs";
+import { a as modelKeyFor, i as VISION_VARIANTS, n as VISION_MODELS, r as VISION_TASKS, t as COCO_CLASSES } from "./registry-CSFimt7C.mjs";
+import { A as decodeRtmdetIns, C as PoseSkeletonRenderer, D as decodeSelfie, E as COCO17_KEYPOINT_NAMES, F as preprocessSpec, I as toSourcePoint, L as analyzeSequence, M as PREPROCESS_BY_FAMILY, N as computeInputTransform, O as COCO17_KEYPOINT_COUNT, P as imageToTensor, R as TemporalObjectTracker, S as SegmentationTexturePool, T as COCO17_KEYPOINTS, _ as createNeutralLandmarks, a as VisionBundle, b as VisionModelStore, c as matchPoseToTracks, d as mergeSubjectMask, f as selectSubjectMask, g as setDefaultSessionProvider, h as createDefaultSessionProvider, i as pinNodeToObject, j as probabilityToAlpha, k as decodeRtmo, l as SpatialLandmarkTransformer, m as NodeSessionProvider, n as VisionNode, o as createVisionBundle, p as VisionRunner, r as pinNodeToLandmark, s as getPersonBoundingBox, t as src_exports, u as maskBounds, v as createNeutralObjectResult, w as COCO17_BONES, x as getDefaultModelsDir, y as createNeutralPoseResult, z as computeIoU } from "./src-DbPyIIat.mjs";
+
+export * from "@framefields/core"
+
+//#region src/runtime/webgpu-provider.ts
+const ORT_WEB_SPECIFIER = "onnxruntime-web";
+const defaultLoader = () => import(
+	/* @vite-ignore */
+	ORT_WEB_SPECIFIER
+);
+var WebGPUProvider = class {
+	kind = "webgpu";
+	_loader;
+	_providers;
+	_wasmFallback;
+	_ortPromise;
+	constructor(options = {}) {
+		this._loader = options.loader ?? defaultLoader;
+		this._providers = options.executionProviders ?? ["webgpu", "wasm"];
+		this._wasmFallback = options.wasmFallback !== false;
+	}
+	ort() {
+		if (!this._ortPromise) this._ortPromise = this._loader().catch((err) => {
+			this._ortPromise = void 0;
+			throw new Error(`onnxruntime-web failed to load (${String(err)}). Install it in the browser bundle or pass a custom loader.`);
+		});
+		return this._ortPromise;
+	}
+	async createSession(modelBytes) {
+		const ort = await this.ort();
+		let session;
+		try {
+			session = await ort.InferenceSession.create(modelBytes, {
+				executionProviders: [...this._providers],
+				graphOptimizationLevel: "all",
+				logSeverityLevel: 3
+			});
+		} catch (err) {
+			if (!this._wasmFallback || !this._providers.includes("webgpu")) throw err;
+			session = await ort.InferenceSession.create(modelBytes, {
+				executionProviders: ["wasm"],
+				graphOptimizationLevel: "all",
+				logSeverityLevel: 3
+			});
+		}
+		return {
+			run: async (input) => {
+				const inputName = session.inputNames.includes(input.name) ? input.name : session.inputNames[0] ?? input.name;
+				const results = await session.run({ [inputName]: new ort.Tensor("float32", input.data, [...input.dims]) });
+				const outputs = {};
+				for (const [name, tensor] of Object.entries(results)) outputs[name] = {
+					data: tensor.data,
+					dims: [...tensor.dims]
+				};
+				return outputs;
+			},
+			release: () => {
+				session.release();
+			}
+		};
+	}
+};
+/**
+* Convenience factory: a `SessionProvider` type guard for the browser entry. Kept tiny so
+* `@framefields/vision/web` stays tree-shakeable.
+*/
+function createWebGPUProvider(options) {
+	return new WebGPUProvider(options);
+}
+/** True when the environment exposes a WebGPU device (synchronous capability probe). */
+function hasWebGPU() {
+	return typeof navigator !== "undefined" && "gpu" in navigator;
+}
+
+//#endregion
+//#region src/web.ts
+var web_exports = /* @__PURE__ */ __exportAll({
+	COCO17_BONES: () => COCO17_BONES,
+	COCO17_KEYPOINTS: () => COCO17_KEYPOINTS,
+	COCO17_KEYPOINT_COUNT: () => COCO17_KEYPOINT_COUNT,
+	COCO17_KEYPOINT_NAMES: () => COCO17_KEYPOINT_NAMES,
+	COCO_CLASSES: () => COCO_CLASSES,
+	NodeSessionProvider: () => NodeSessionProvider,
+	PREPROCESS_BY_FAMILY: () => PREPROCESS_BY_FAMILY,
+	PoseSkeletonRenderer: () => PoseSkeletonRenderer,
+	SegmentationTexturePool: () => SegmentationTexturePool,
+	SpatialLandmarkTransformer: () => SpatialLandmarkTransformer,
+	TemporalObjectTracker: () => TemporalObjectTracker,
+	VISION_MODELS: () => VISION_MODELS,
+	VISION_TASKS: () => VISION_TASKS,
+	VISION_VARIANTS: () => VISION_VARIANTS,
+	VisionBundle: () => VisionBundle,
+	VisionModelStore: () => VisionModelStore,
+	VisionNode: () => VisionNode,
+	VisionRunner: () => VisionRunner,
+	WebGPUProvider: () => WebGPUProvider,
+	analyzeSequence: () => analyzeSequence,
+	computeInputTransform: () => computeInputTransform,
+	computeIoU: () => computeIoU,
+	createDefaultSessionProvider: () => createDefaultSessionProvider,
+	createNeutralLandmarks: () => createNeutralLandmarks,
+	createNeutralObjectResult: () => createNeutralObjectResult,
+	createNeutralPoseResult: () => createNeutralPoseResult,
+	createVisionBundle: () => createVisionBundle,
+	createWebGPUProvider: () => createWebGPUProvider,
+	decodeRtmdetIns: () => decodeRtmdetIns,
+	decodeRtmo: () => decodeRtmo,
+	decodeSelfie: () => decodeSelfie,
+	getDefaultModelsDir: () => getDefaultModelsDir,
+	getPersonBoundingBox: () => getPersonBoundingBox,
+	hasWebGPU: () => hasWebGPU,
+	imageToTensor: () => imageToTensor,
+	maskBounds: () => maskBounds,
+	matchPoseToTracks: () => matchPoseToTracks,
+	mergeSubjectMask: () => mergeSubjectMask,
+	modelKeyFor: () => modelKeyFor,
+	pinNodeToLandmark: () => pinNodeToLandmark,
+	pinNodeToObject: () => pinNodeToObject,
+	preprocessSpec: () => preprocessSpec,
+	probabilityToAlpha: () => probabilityToAlpha,
+	selectSubjectMask: () => selectSubjectMask,
+	setDefaultSessionProvider: () => setDefaultSessionProvider,
+	toSourcePoint: () => toSourcePoint
+});
+__reExport(web_exports, src_exports);
+
+//#endregion
+export { COCO17_BONES, COCO17_KEYPOINTS, COCO17_KEYPOINT_COUNT, COCO17_KEYPOINT_NAMES, COCO_CLASSES, NodeSessionProvider, PREPROCESS_BY_FAMILY, PoseSkeletonRenderer, SegmentationTexturePool, SpatialLandmarkTransformer, TemporalObjectTracker, VISION_MODELS, VISION_TASKS, VISION_VARIANTS, VisionBundle, VisionModelStore, VisionNode, VisionRunner, WebGPUProvider, analyzeSequence, computeInputTransform, computeIoU, createDefaultSessionProvider, createNeutralLandmarks, createNeutralObjectResult, createNeutralPoseResult, createVisionBundle, createWebGPUProvider, decodeRtmdetIns, decodeRtmo, decodeSelfie, getDefaultModelsDir, getPersonBoundingBox, hasWebGPU, imageToTensor, maskBounds, matchPoseToTracks, mergeSubjectMask, modelKeyFor, pinNodeToLandmark, pinNodeToObject, preprocessSpec, probabilityToAlpha, selectSubjectMask, setDefaultSessionProvider, toSourcePoint };
+//# sourceMappingURL=web.mjs.map

package/dist/web.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"web.mjs","names":[],"sources":["../src/runtime/webgpu-provider.ts","../src/web.ts"],"sourcesContent":["/**\n * Browser / WebGPU session provider .\n *\n * Mirrors `NodeSessionProvider` but uses `onnxruntime-web` with the WebGPU execution provider\n * and a WASM fallback. The ORT module is imported lazily behind an injectable loader, so:\n *  - the Node core entry never pulls browser code, and\n *  - unit tests can run fully offline with a fake module.\n *\n * `onnxruntime-web` is an OPTIONAL peer dependency — consumers of `@framefields/vision/web`\n * provide it; the main `@framefields/vision` entry never touches it.\n */\n\nimport type {\n\tSessionProvider,\n\tVisionSession,\n\tVisionTensorInput,\n\tVisionTensorOutputs,\n} from \"./session-provider.js\";\n\n/** Minimal structural view of the `onnxruntime-web` module — isolates the ORT API surface. */\nexport interface OrtWebTensor {\n\treadonly data: Float32Array | BigInt64Array | Int32Array;\n\treadonly dims: readonly number[];\n}\n\nexport interface OrtWebSession {\n\treadonly inputNames: readonly string[];\n\trun(feeds: Record<string, unknown>): Promise<Record<string, OrtWebTensor>>;\n\trelease(): Promise<void>;\n}\n\nexport interface OrtWebModule {\n\treadonly InferenceSession: {\n\t\tcreate(\n\t\t\tmodelBytes: Uint8Array,\n\t\t\toptions: {\n\t\t\t\texecutionProviders: string[];\n\t\t\t\tgraphOptimizationLevel?: \"disabled\" | \"basic\" | \"extended\" | \"all\";\n\t\t\t\t/** 0 verbose … 4 fatal. */\n\t\t\t\tlogSeverityLevel?: 0 | 1 | 2 | 3 | 4;\n\t\t\t},\n\t\t): Promise<OrtWebSession>;\n\t};\n\t/** ORT tensor constructor (called with `new`). */\n\treadonly Tensor: new (\n\t\ttype: \"float32\",\n\t\tdata: Float32Array,\n\t\tdims: readonly number[],\n\t) => unknown;\n}\n\nexport interface WebGPUProviderOptions {\n\t/** Injected ORT loader — defaults to a dynamic `import(\"onnxruntime-web\")`. */\n\treadonly loader?: () => Promise<OrtWebModule>;\n\t/** Execution providers in priority order. Defaults to `[\"webgpu\", \"wasm\"]`. */\n\treadonly executionProviders?: readonly string[];\n\t/** Fall back to WASM automatically if the WebGPU session fails to create. Default true. */\n\treadonly wasmFallback?: boolean;\n}\n\nconst ORT_WEB_SPECIFIER = \"onnxruntime-web\";\n\nconst defaultLoader = (): Promise<OrtWebModule> =>\n\timport(\n\t\t/* @vite-ignore */ ORT_WEB_SPECIFIER\n\t) as unknown as Promise<OrtWebModule>;\n\nexport class WebGPUProvider implements SessionProvider {\n\tpublic readonly kind = \"webgpu\" as const;\n\n\tprivate readonly _loader: () => Promise<OrtWebModule>;\n\tprivate readonly _providers: readonly string[];\n\tprivate readonly _wasmFallback: boolean;\n\tprivate _ortPromise?: Promise<OrtWebModule>;\n\n\tconstructor(options: WebGPUProviderOptions = {}) {\n\t\tthis._loader = options.loader ?? defaultLoader;\n\t\tthis._providers = options.executionProviders ?? [\"webgpu\", \"wasm\"];\n\t\tthis._wasmFallback = options.wasmFallback !== false;\n\t}\n\n\tprivate ort(): Promise<OrtWebModule> {\n\t\tif (!this._ortPromise) {\n\t\t\tthis._ortPromise = this._loader().catch((err: unknown) => {\n\t\t\t\tthis._ortPromise = undefined;\n\t\t\t\tthrow new Error(\n\t\t\t\t\t`onnxruntime-web failed to load (${String(err)}). ` +\n\t\t\t\t\t\t\"Install it in the browser bundle or pass a custom loader.\",\n\t\t\t\t);\n\t\t\t});\n\t\t}\n\t\treturn this._ortPromise;\n\t}\n\n\tpublic async createSession(modelBytes: Uint8Array): Promise<VisionSession> {\n\t\tconst ort = await this.ort();\n\n\t\tlet session: OrtWebSession;\n\t\ttry {\n\t\t\tsession = await ort.InferenceSession.create(modelBytes, {\n\t\t\t\texecutionProviders: [...this._providers],\n\t\t\t\tgraphOptimizationLevel: \"all\",\n\t\t\t\t// Errors only: node-placement notes print as console errors otherwise.\n\t\t\t\tlogSeverityLevel: 3,\n\t\t\t});\n\t\t} catch (err) {\n\t\t\tif (!this._wasmFallback || !this._providers.includes(\"webgpu\")) throw err;\n\t\t\tsession = await ort.InferenceSession.create(modelBytes, {\n\t\t\t\texecutionProviders: [\"wasm\"],\n\t\t\t\tgraphOptimizationLevel: \"all\",\n\t\t\t\t// Errors only: node-placement notes print as console errors otherwise.\n\t\t\t\tlogSeverityLevel: 3,\n\t\t\t});\n\t\t}\n\n\t\treturn {\n\t\t\trun: async (input: VisionTensorInput): Promise<VisionTensorOutputs> => {\n\t\t\t\tconst inputName = session.inputNames.includes(input.name)\n\t\t\t\t\t? input.name\n\t\t\t\t\t: (session.inputNames[0] ?? input.name);\n\t\t\t\tconst results = await session.run({\n\t\t\t\t\t[inputName]: new ort.Tensor(\"float32\", input.data, [...input.dims]),\n\t\t\t\t});\n\t\t\t\tconst outputs: Record<string, VisionTensorOutputs[string]> = {};\n\t\t\t\tfor (const [name, tensor] of Object.entries(results)) {\n\t\t\t\t\toutputs[name] = {\n\t\t\t\t\t\tdata: tensor.data,\n\t\t\t\t\t\tdims: [...tensor.dims],\n\t\t\t\t\t};\n\t\t\t\t}\n\t\t\t\treturn outputs;\n\t\t\t},\n\t\t\trelease: () => {\n\t\t\t\tvoid session.release();\n\t\t\t},\n\t\t};\n\t}\n}\n\n/**\n * Convenience factory: a `SessionProvider` type guard for the browser entry. Kept tiny so\n * `@framefields/vision/web` stays tree-shakeable.\n */\nexport function createWebGPUProvider(\n\toptions?: WebGPUProviderOptions,\n): WebGPUProvider {\n\treturn new WebGPUProvider(options);\n}\n\n/** True when the environment exposes a WebGPU device (synchronous capability probe). */\nexport function hasWebGPU(): boolean {\n\treturn typeof navigator !== \"undefined\" && \"gpu\" in navigator;\n}\n","/**\n * Browser entry (`@framefields/vision/web`).\n *\n * Re-exports the full engine plus the WebGPU session provider. Consumers pass a provider to\n * the runner explicitly (keeps the runner environment-agnostic and tree-shakeable):\n *\n *   const runner = VisionRunner.create({ provider: createWebGPUProvider() });\n *\n * `onnxruntime-web` is an optional peer dependency, loaded lazily on the first session.\n */\n\nexport * from \"./index.js\";\nexport * from \"./runtime/webgpu-provider.js\";\n"],"mappings":";;;;;;;AA4DA,MAAM,oBAAoB;AAE1B,MAAM,sBACL;;CACoB;;AAGrB,IAAa,iBAAb,MAAuD;CACtD,AAAgB,OAAO;CAEvB,AAAiB;CACjB,AAAiB;CACjB,AAAiB;CACjB,AAAQ;CAER,YAAY,UAAiC,EAAE,EAAE;AAChD,OAAK,UAAU,QAAQ,UAAU;AACjC,OAAK,aAAa,QAAQ,sBAAsB,CAAC,UAAU,OAAO;AAClE,OAAK,gBAAgB,QAAQ,iBAAiB;;CAG/C,AAAQ,MAA6B;AACpC,MAAI,CAAC,KAAK,YACT,MAAK,cAAc,KAAK,SAAS,CAAC,OAAO,QAAiB;AACzD,QAAK,cAAc;AACnB,SAAM,IAAI,MACT,mCAAmC,OAAO,IAAI,CAAC,8DAE/C;IACA;AAEH,SAAO,KAAK;;CAGb,MAAa,cAAc,YAAgD;EAC1E,MAAM,MAAM,MAAM,KAAK,KAAK;EAE5B,IAAI;AACJ,MAAI;AACH,aAAU,MAAM,IAAI,iBAAiB,OAAO,YAAY;IACvD,oBAAoB,CAAC,GAAG,KAAK,WAAW;IACxC,wBAAwB;IAExB,kBAAkB;IAClB,CAAC;WACM,KAAK;AACb,OAAI,CAAC,KAAK,iBAAiB,CAAC,KAAK,WAAW,SAAS,SAAS,CAAE,OAAM;AACtE,aAAU,MAAM,IAAI,iBAAiB,OAAO,YAAY;IACvD,oBAAoB,CAAC,OAAO;IAC5B,wBAAwB;IAExB,kBAAkB;IAClB,CAAC;;AAGH,SAAO;GACN,KAAK,OAAO,UAA2D;IACtE,MAAM,YAAY,QAAQ,WAAW,SAAS,MAAM,KAAK,GACtD,MAAM,OACL,QAAQ,WAAW,MAAM,MAAM;IACnC,MAAM,UAAU,MAAM,QAAQ,IAAI,GAChC,YAAY,IAAI,IAAI,OAAO,WAAW,MAAM,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,EACnE,CAAC;IACF,MAAM,UAAuD,EAAE;AAC/D,SAAK,MAAM,CAAC,MAAM,WAAW,OAAO,QAAQ,QAAQ,CACnD,SAAQ,QAAQ;KACf,MAAM,OAAO;KACb,MAAM,CAAC,GAAG,OAAO,KAAK;KACtB;AAEF,WAAO;;GAER,eAAe;AACd,IAAK,QAAQ,SAAS;;GAEvB;;;;;;;AAQH,SAAgB,qBACf,SACiB;AACjB,QAAO,IAAI,eAAe,QAAQ;;;AAInC,SAAgB,YAAqB;AACpC,QAAO,OAAO,cAAc,eAAe,SAAS"}

package/package.json

@@ -1,6 +1,67 @@
 {
   "name": "@framefields/vision",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "description": "On-device vision for framefields: RTMDet-Ins detection + instance masks, RTMO pose, and Selfie Segmenter mattes over ONNX Runtime.",
+  "version": "2.0.2",
+  "private": false,
+  "publishConfig": {
+    "access": "public"
+  },
+  "type": "module",
+  "sideEffects": false,
+  "license": "Apache-2.0",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "development": "./src/index.ts",
+      "import": "./dist/index.mjs",
+      "default": "./dist/index.mjs"
+    },
+    "./schemas": {
+      "types": "./dist/schemas.d.mts",
+      "development": "./src/schemas.ts",
+      "import": "./dist/schemas.mjs",
+      "default": "./dist/schemas.mjs"
+    },
+    "./web": {
+      "types": "./dist/web.d.mts",
+      "development": "./src/web.ts",
+      "import": "./dist/web.mjs",
+      "default": "./dist/web.mjs"
+    }
+  },
+  "files": [
+    "dist/**"
+  ],
+  "peerDependencies": {
+    "onnxruntime-web": "^1.24.0"
+  },
+  "peerDependenciesMeta": {
+    "onnxruntime-web": {
+      "optional": true
+    }
+  },
+  "dependencies": {
+    "onnxruntime-node": "^1.24.0",
+    "webgpu": "^0.4.0",
+    "zod": "^4.3.6",
+    "@framefields/core": "2.0.2",
+    "@framefields/tensor-webgpu": "2.0.2"
+  },
+  "devDependencies": {
+    "@types/node": "^24.1.0",
+    "skia-canvas": "^3.0.8",
+    "tsdown": "^0.19.0",
+    "typescript": "^5.8.3",
+    "vitest": "^4.1.5",
+    "@framefields/tsconfig": "2.0.2"
+  },
+  "scripts": {
+    "build": "tsdown",
+    "dev": "tsdown --watch",
+    "clean": "rm -rf .turbo && rm -rf node_modules && rm -rf dist",
+    "test": "vitest run",
+    "test:models": "FRAMEFIELDS_VISION_E2E=1 vitest run src/models.e2e.test.ts"
+  }
 }

package/README.md

@@ -1,3 +0,0 @@
-# Temporary Holding Version
-
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.