framefields 2.13.1 → 2.15.0

package/dist/{src-BU6vm9sP-DdTHmCr4.mjs → src-B23p_eh7-Cj6-lvV6.mjs}

@@ -2,11 +2,11 @@ import { Pt as frameSignal, pn as programmaticSignal } from "./dist-C8XSqfTx.mjs
 import { a as PoseSkeletonComputePipeline } from "./dist-DgZpN6IK.mjs";
 import { existsSync, mkdirSync, statSync } from "node:fs";
 import { dirname, join, resolve } from "node:path";
-import { readFile, rename, unlink, writeFile } from "node:fs/promises";
 import { createHash } from "node:crypto";
+import { readFile, rename, unlink, writeFile } from "node:fs/promises";
 import { homedir } from "node:os";
 
-//#region ../vision/dist/registry-DOHiTjt1.mjs
+//#region ../vision/dist/registry-DJ-XmFZF.mjs
 const hf = (repo, revision, path$1) => `https://huggingface.co/${repo}/resolve/${revision}/${path$1}`;
 const RTMDET_INS_REPO = "tori29umai/rtmdet-ins-onnx-person-masks";
 const RTMDET_INS_REV = "e5218e8d47d7c50077f27f9893a28e0ef2ad5656";
@@ -14,6 +14,7 @@ const RTMDET_INS_SOURCE = "RTMDet-Ins (OpenMMLab mmdetection, Apache-2.0), mmdep
 const RTMO_SOURCE = "RTMO (OpenMMLab mmpose, Apache-2.0), ONNX export";
 const SELFIE_SOURCE = "MediaPipe Selfie Segmenter (Google, Apache-2.0), ONNX export";
 const DEPTH_SOURCE = "Depth Anything V2 Small (TikTok / HKU, Apache-2.0), ONNX export";
+const VITMATTE_SOURCE = "ViTMatte Small, Distinctions-646 (HUST Vision Lab, code MIT, weights Apache-2.0), ONNX export";
 const models = {
 	"rtmdet-ins-t": {
 		key: "rtmdet-ins-t",
@@ -104,6 +105,16 @@ const models = {
 		sha256: "afb6a5c28f3b6bf1618c6e43f02073ef9dfdc70e937502d51603e57b0a1df10c",
 		input: [518, 518],
 		source: DEPTH_SOURCE
+	},
+	"vitmatte-s": {
+		key: "vitmatte-s",
+		family: "vitmatte",
+		filename: "vitmatte-s.onnx",
+		url: hf("Xenova/vitmatte-small-distinctions-646", "358d428c452e5e0cd52955011a8b51944731d28e", "onnx/model.onnx"),
+		bytes: 103885865,
+		sha256: "a1cf48234c369faa3ea1711981d961fe1ec71f51e593f9d6553aa5a0e7d557e3",
+		input: [1024, 1024],
+		source: VITMATTE_SOURCE
 	}
 };
 const VISION_MODELS = models;
@@ -205,7 +216,7 @@ const COCO_CLASSES = [
 ];
 
 //#endregion
-//#region ../vision/dist/src-BU6vm9sP.mjs
+//#region ../vision/dist/src-B23p_eh7.mjs
 function computeIoU(boxA, boxB) {
 	const xA = Math.max(boxA.originX, boxB.originX);
 	const yA = Math.max(boxA.originY, boxB.originY);
@@ -1650,6 +1661,238 @@ var SegmentationTexturePool = class {
 	}
 };
 /**
+* Trimaps from hard masks, for alpha matting: pixels well inside the mask are
+* foreground, pixels well outside are background, and a band of `band` pixels
+* either side of the edge is unknown, for the matting model to resolve.
+*/
+/** Trimap values, as ViTMatte reads them (0, 0.5, 1 on its 0..1 scale). */
+const TRIMAP_BACKGROUND = 0;
+const TRIMAP_UNKNOWN = 128;
+const TRIMAP_FOREGROUND = 255;
+const FAR = 0x56bc75e2d63100000;
+/**
+* Squared Euclidean distance from every pixel to the nearest pixel where
+* `inside(i)` holds, with that pixel's index (−1 when there is none).
+* Felzenszwalb and Huttenlocher's two-pass lower envelope, O(width × height).
+*/
+function distanceTransform(width, height, inside) {
+	const n = width * height;
+	const columns = new Float32Array(n);
+	const columnFrom = new Int32Array(n);
+	const scratch = envelopeScratch(Math.max(width, height));
+	for (let x = 0; x < width; x++) {
+		for (let y = 0; y < height; y++) scratch.f[y] = inside(y * width + x) ? 0 : FAR;
+		envelope(scratch, height);
+		for (let y = 0; y < height; y++) {
+			columns[y * width + x] = scratch.d[y];
+			columnFrom[y * width + x] = scratch.from[y];
+		}
+	}
+	const distance = new Float32Array(n);
+	const nearest = new Int32Array(n);
+	for (let y = 0; y < height; y++) {
+		const row = y * width;
+		for (let x = 0; x < width; x++) scratch.f[x] = columns[row + x];
+		envelope(scratch, width);
+		for (let x = 0; x < width; x++) {
+			const d = scratch.d[x];
+			const fromX = scratch.from[x];
+			distance[row + x] = d;
+			nearest[row + x] = d >= FAR ? -1 : columnFrom[row + fromX] * width + fromX;
+		}
+	}
+	return {
+		distance,
+		nearest
+	};
+}
+function envelopeScratch(length) {
+	return {
+		f: new Float64Array(length),
+		d: new Float64Array(length),
+		from: new Int32Array(length),
+		v: new Int32Array(length),
+		z: new Float64Array(length + 1)
+	};
+}
+/** One dimension: d[q] = min over p of (q − p)² + f[p]. */
+function envelope(s, length) {
+	const { f, d, from, v, z } = s;
+	let k = -1;
+	for (let q = 0; q < length; q++) {
+		const fq = f[q];
+		if (fq >= FAR) continue;
+		let cut = Number.NEGATIVE_INFINITY;
+		while (k >= 0) {
+			const p = v[k];
+			cut = (fq + q * q - (f[p] + p * p)) / (2 * (q - p));
+			if (cut > z[k]) break;
+			k--;
+		}
+		k++;
+		v[k] = q;
+		z[k] = k === 0 ? Number.NEGATIVE_INFINITY : cut;
+		z[k + 1] = Number.POSITIVE_INFINITY;
+	}
+	if (k < 0) {
+		d.fill(FAR, 0, length);
+		from.fill(0, 0, length);
+		return;
+	}
+	let j = 0;
+	for (let q = 0; q < length; q++) {
+		while (z[j + 1] < q) j++;
+		const p = v[j];
+		d[q] = (q - p) * (q - p) + f[p];
+		from[q] = p;
+	}
+}
+/**
+* The trimap of a mask (`mask[i] >= 128` is inside): an unknown band of
+* `band` px either side of the edge, foreground inside it, background outside.
+*/
+function trimapFromMask(mask, width, height, band) {
+	const outside = distanceTransform(width, height, (i) => mask[i] >= 128);
+	const inside = distanceTransform(width, height, (i) => mask[i] < 128);
+	const r2 = band * band;
+	const data = new Uint8Array(width * height);
+	let x0 = width;
+	let y0 = height;
+	let x1 = 0;
+	let y1 = 0;
+	for (let y = 0; y < height; y++) for (let x = 0; x < width; x++) {
+		const i = y * width + x;
+		const isIn = mask[i] >= 128;
+		if ((isIn ? inside.distance[i] : outside.distance[i]) > r2) {
+			data[i] = isIn ? TRIMAP_FOREGROUND : TRIMAP_BACKGROUND;
+			continue;
+		}
+		data[i] = TRIMAP_UNKNOWN;
+		if (x < x0) x0 = x;
+		if (y < y0) y0 = y;
+		if (x >= x1) x1 = x + 1;
+		if (y >= y1) y1 = y + 1;
+	}
+	return {
+		data,
+		width,
+		height,
+		unknown: x1 > x0 ? [
+			x0,
+			y0,
+			x1,
+			y1
+		] : null,
+		nearest: outside.nearest
+	};
+}
+/** Default band for a frame: about 1.2% of its short side, 3…40 px (13 px at 1080p). */
+function defaultMatteBand(width, height) {
+	return Math.min(40, Math.max(3, Math.round(Math.min(width, height) * .012)));
+}
+/** ViTMatte's patch grid: both input sides pad to a multiple of 32 (as its image processor does). */
+const DIVISOR = 32;
+/**
+* The window around a trimap's unknown band, grown by `context` px so the
+* model sees what is on either side, and scaled down when its long side
+* exceeds `resolution`. Null when there is nothing to resolve.
+*/
+function matteWindow(trimap, context, resolution) {
+	if (!trimap.unknown) return null;
+	const [ux0, uy0, ux1, uy1] = trimap.unknown;
+	const x0 = Math.max(0, ux0 - context);
+	const y0 = Math.max(0, uy0 - context);
+	const x1 = Math.min(trimap.width, ux1 + context);
+	const y1 = Math.min(trimap.height, uy1 + context);
+	const scale = Math.min(1, resolution / Math.max(x1 - x0, y1 - y0));
+	const contentWidth = Math.max(1, Math.round((x1 - x0) * scale));
+	const contentHeight = Math.max(1, Math.round((y1 - y0) * scale));
+	return {
+		x0,
+		y0,
+		x1,
+		y1,
+		scale,
+		contentWidth,
+		contentHeight,
+		inputWidth: Math.ceil(contentWidth / DIVISOR) * DIVISOR,
+		inputHeight: Math.ceil(contentHeight / DIVISOR) * DIVISOR
+	};
+}
+/**
+* The model input `[1, 4, H, W]`: RGB as (v / 255 − 0.5) / 0.5, then the
+* trimap on 0…1. The image is sampled bilinearly, the trimap nearest so its
+* three levels stay exact; padding is zero in every channel.
+*/
+function vitmatteInput(image, trimap, window, out) {
+	const { inputWidth: w, inputHeight: h } = window;
+	const plane = w * h;
+	const tensor = out && out.length >= 4 * plane ? out.subarray(0, 4 * plane) : new Float32Array(4 * plane);
+	tensor.fill(0);
+	const { data, width: sw, height: sh } = image;
+	const inv = 1 / window.scale;
+	for (let y = 0; y < window.contentHeight; y++) {
+		const sy = Math.min(sh - 1, Math.max(0, window.y0 + (y + .5) * inv - .5));
+		const y0 = Math.floor(sy);
+		const y1 = Math.min(sh - 1, y0 + 1);
+		const fy = sy - y0;
+		const ty = Math.min(window.y1 - 1, Math.floor(window.y0 + (y + .5) * inv));
+		for (let x = 0; x < window.contentWidth; x++) {
+			const sx = Math.min(sw - 1, Math.max(0, window.x0 + (x + .5) * inv - .5));
+			const x0 = Math.floor(sx);
+			const x1 = Math.min(sw - 1, x0 + 1);
+			const fx = sx - x0;
+			const o = y * w + x;
+			for (let c = 0; c < 3; c++) {
+				const a = data[(y0 * sw + x0) * 4 + c];
+				const b = data[(y0 * sw + x1) * 4 + c];
+				const d = data[(y1 * sw + x0) * 4 + c];
+				const e = data[(y1 * sw + x1) * 4 + c];
+				const top = a + (b - a) * fx;
+				const v = top + (d + (e - d) * fx - top) * fy;
+				tensor[c * plane + o] = v / 127.5 - 1;
+			}
+			const tx = Math.min(window.x1 - 1, Math.floor(window.x0 + (x + .5) * inv));
+			tensor[3 * plane + o] = trimap.data[ty * trimap.width + tx] / 255;
+		}
+	}
+	return tensor;
+}
+/**
+* The frame's alpha: the trimap's foreground and background as they are,
+* and the model's alpha (sampled back to source px) across the unknown band.
+*/
+function composeMatte(trimap, window, alphas) {
+	const { width } = trimap;
+	const out = Uint8Array.from(trimap.data, (v) => v === TRIMAP_UNKNOWN ? 0 : v);
+	if (!window) return out;
+	const w = window.inputWidth;
+	const maxX = window.contentWidth - 1;
+	const maxY = window.contentHeight - 1;
+	for (let y = window.y0; y < window.y1; y++) {
+		const my = Math.min(maxY, Math.max(0, (y - window.y0 + .5) * window.scale - .5));
+		const my0 = Math.floor(my);
+		const my1 = Math.min(maxY, my0 + 1);
+		const fy = my - my0;
+		for (let x = window.x0; x < window.x1; x++) {
+			const i = y * width + x;
+			if (trimap.data[i] !== TRIMAP_UNKNOWN) continue;
+			const mx = Math.min(maxX, Math.max(0, (x - window.x0 + .5) * window.scale - .5));
+			const mx0 = Math.floor(mx);
+			const mx1 = Math.min(maxX, mx0 + 1);
+			const fx = mx - mx0;
+			const a = alphas[my0 * w + mx0];
+			const b = alphas[my0 * w + mx1];
+			const c = alphas[my1 * w + mx0];
+			const d = alphas[my1 * w + mx1];
+			const top = a + (b - a) * fx;
+			const v = top + (c + (d - c) * fx - top) * fy;
+			out[i] = Math.round(Math.min(1, Math.max(0, v)) * 255);
+		}
+	}
+	return out;
+}
+/**
 * Resolves the default model cache directory:
 * 1. `FRAMEFIELDS_MODELS_DIR`
 * 2. `~/.cache/framefields/models`
@@ -2915,6 +3158,46 @@ var VisionRunner = class VisionRunner$1 {
 		const dims = predicted.dims;
 		return decodeDepth(predicted.data, dims[dims.length - 1] ?? input[0], dims[dims.length - 2] ?? input[1]);
 	}
+	/**
+	* Resolves a hard mask's edge to soft alpha (ViTMatte Small): hair, motion
+	* blur and see-through parts. `mask` is frame-sized, inside where ≥ 128.
+	* Only a band around the edge changes; one model pass covers every part of
+	* the mask, cropped to the band and scaled to `resolution`.
+	*/
+	async refineMatte(image, mask, options = {}) {
+		const { width, height } = image;
+		if (mask.length !== width * height) throw new Error(`refineMatte: the mask has ${mask.length} pixels; the frame has ${width}×${height}.`);
+		const band = Math.max(1, Math.round(options.band ?? defaultMatteBand(width, height)));
+		const key = "vitmatte-s";
+		const trimap = trimapFromMask(mask, width, height, band);
+		const window = matteWindow(trimap, Math.max(32, band * 2), options.resolution ?? VISION_MODELS[key].input[0]);
+		let alphas = [];
+		if (window) {
+			const pixels = isGpuImage(image) ? await readTexturePixels(image) : image;
+			alphas = output(await this.queued(key, async () => {
+				const session = await this.session(key);
+				const scratch = this._core.tensorScratch;
+				const data = vitmatteInput(pixels, trimap, window, scratch.get(key));
+				if (data.buffer !== scratch.get(key)?.buffer) scratch.set(key, data);
+				return session.run({
+					name: "pixel_values",
+					data,
+					dims: [
+						1,
+						4,
+						window.inputHeight,
+						window.inputWidth
+					]
+				});
+			}), "alphas", key).data;
+		}
+		return {
+			mask: composeMatte(trimap, window, alphas),
+			width,
+			height,
+			nearest: trimap.nearest
+		};
+	}
 	close() {
 		if (!this._ownsCore) return;
 		const core = this._core;
@@ -2965,8 +3248,12 @@ var VisionRunner = class VisionRunner$1 {
 	* the input scratch buffer, and one run already uses the session's threads.
 	*/
 	infer(key, image, input) {
+		return this.queued(key, () => this.inferNow(key, image, input));
+	}
+	/** Runs `fn` after the model's earlier runs: one inference per session at a time. */
+	queued(key, fn) {
 		const queue = this._core.runQueue;
-		const run = (queue.get(key) ?? Promise.resolve()).then(() => this.inferNow(key, image, input), () => this.inferNow(key, image, input));
+		const run = (queue.get(key) ?? Promise.resolve()).then(fn, fn);
 		const tail = run.catch(() => void 0);
 		queue.set(key, tail);
 		tail.then(() => {
@@ -4380,5 +4667,5 @@ var VisionNode = class VisionNode$1 {
 };
 
 //#endregion
-export { setVisionDevice as A, matchPoseToTracks as C, readTexturePixels as D, pinNodeToObject as E, registerVisionModel as F, varint as M, COCO_CLASSES as N, readVarint as O, VISION_MODELS as P, maskObject as S, pinNodeToLandmark as T, fields as _, PoseSkeletonRenderer as a, lengthDelimited as b, TemporalObjectTracker as c, VisionNode as d, VisionRunner as f, encoder as g, createDefaultSessionProvider as h, COCO17_KEYPOINT_NAMES as i, strings as j, sessionCompute as k, VisionBundle as l, concat as m, COCO17_BONES as n, SegmentationTexturePool as o, analyzeSequence as p, COCO17_KEYPOINTS as r, SpatialLandmarkTransformer as s, CLIP_SUBJECT as t, VisionModelStore as u, firstString as v, mergeSubjectMask as w, maskBounds as x, imageToTensor as y };
-//# sourceMappingURL=src-BU6vm9sP-DdTHmCr4.mjs.map
+export { sessionCompute as A, maskObject as C, pinNodeToObject as D, pinNodeToLandmark as E, VISION_MODELS as F, registerVisionModel as I, strings as M, varint as N, readTexturePixels as O, COCO_CLASSES as P, maskBounds as S, mergeSubjectMask as T, encoder as _, PoseSkeletonRenderer as a, imageToTensor as b, TemporalObjectTracker as c, VisionNode as d, VisionRunner as f, defaultMatteBand as g, createDefaultSessionProvider as h, COCO17_KEYPOINT_NAMES as i, setVisionDevice as j, readVarint as k, VisionBundle as l, concat as m, COCO17_BONES as n, SegmentationTexturePool as o, analyzeSequence as p, COCO17_KEYPOINTS as r, SpatialLandmarkTransformer as s, CLIP_SUBJECT as t, VisionModelStore as u, fields as v, matchPoseToTracks as w, lengthDelimited as x, firstString as y };
+//# sourceMappingURL=src-B23p_eh7-Cj6-lvV6.mjs.map