@camstack/addon-pipeline 1.2.107 → 1.2.118

package/dist/detection-pipeline/index.mjs

@@ -1,9 +1,9 @@
 import { n as __require } from "../chunk-DnnnRqeS.mjs";
-import { $ as pickClusterStepSettings, At as nodePin, D as YAMNET_TO_MACRO, Et as hydrateSchema, G as evaluateZoneRules, Gt as union, Ht as object, I as defaultDeviceFor$1, It as array, Kt as EventCategory, Pt as sleep, Q as pickClusterStepModels, Tt as createEvent, V as detectionPipelineCapability, W as enumerateInferenceDevices, Wt as string, X as overlayClusterStepSettings, c as DEFAULT_CLUSTER_STEP_MODELS, ct as resolvePoolMemoryPolicy, ht as errMsg, jt as parseJsonUnknown, l as DEFAULT_CLUSTER_STEP_SETTINGS, nt as pipelineExecutorCapability, ot as resolveClusterStepModelId, p as DEVICE_BACKEND_TO_FORMAT, pt as supportedRuntimes$1, t as APPLE_SA_TO_MACRO, ut as runtimeDevices$1, x as PoolMemoryWatchdog, xt as BaseAddon } from "../dist-CkXAzTZ2.mjs";
-import { a as getDefaultModelForFormat, c as getStepDefinition, i as ALL_STEPS, l as resolveModelForFormat, n as startEventLoopStallMonitor, o as getDefaultModelForFormatFromDef, r as ALL_PIPELINE_STEPS, s as getStep, u as landmarkPrecisionVerdict } from "../event-loop-stall-monitor-fHkfgmlw.mjs";
+import { $ as pickClusterStepSettings, At as nodePin, D as YAMNET_TO_MACRO, Et as hydrateSchema, G as evaluateZoneRules, Gt as union, Ht as object, I as defaultDeviceFor$1, It as array, Kt as EventCategory, Pt as sleep, Q as pickClusterStepModels, Tt as createEvent, V as detectionPipelineCapability, W as enumerateInferenceDevices, Wt as string, X as overlayClusterStepSettings, c as DEFAULT_CLUSTER_STEP_MODELS, ct as resolvePoolMemoryPolicy, ht as errMsg, jt as parseJsonUnknown, l as DEFAULT_CLUSTER_STEP_SETTINGS, nt as pipelineExecutorCapability, ot as resolveClusterStepModelId, p as DEVICE_BACKEND_TO_FORMAT, pt as supportedRuntimes$1, t as APPLE_SA_TO_MACRO, ut as runtimeDevices$1, x as PoolMemoryWatchdog, xt as BaseAddon } from "../dist-B763E61Y.mjs";
+import { a as ALL_PIPELINE_STEPS, c as getDefaultModelForFormatFromDef, d as resolveModelForFormat, f as landmarkPrecisionVerdict, l as getStep, n as startEventLoopStallMonitor, o as ALL_STEPS, r as localFrameRegistry, s as getDefaultModelForFormat, u as getStepDefinition } from "../event-loop-stall-monitor-CYEvvp2Z.mjs";
 import { t as getSharp } from "../lazy-sharp-6oymT_yf.mjs";
-import { n as pickNodePlatformArch, t as readProcessMemory } from "../process-memory-DlSOzuHP.mjs";
-import { a as ensureModel, o as isModelDownloaded, r as deleteModelFromDisk } from "../addon-utils-A2S9D7pu.mjs";
+import { n as pickNodePlatformArch, t as readProcessMemory } from "../process-memory-CK3FnKyZ.mjs";
+import { a as ensureModel, o as isModelDownloaded, r as deleteModelFromDisk } from "../addon-utils-CgBCF-mE.mjs";
 import sharp from "sharp";
 import { spawn } from "node:child_process";
 import * as path$1 from "node:path";
@@ -257,7 +257,13 @@ var OBJECT_DETECTION_STEP_ID = "object-detection";
 * `null` → the caller substitutes the step's own `defaultModelId` (`yolo26n`).
 * INTEL NPU + iGPU default to OUR YOLOv9 (int8 @320): yolo26 does NOT compile on
 * the Intel NPU (arch incompat → 0 frames), while yolov9 runs on it (~298 fps
-* measured). Apple ANE stays on yolo26 until the CoreML yolov9 build lands.
+* measured). Apple ANE uses the CoreML yolov9 @320 build.
+*
+* Do **not** promote these ids to `yolov9m-640` / `*-640`. The controlled
+* evaluation recovered 0/3 misses at the current 0.50 threshold and failed the
+* false-positive and capacity gates — see
+* `docs/benchmarks/pipeline-frame-model-eval.md`. 640 remains an explicit
+* operator pick, not a hardware-aware default.
 */
 var MODEL_BY_CLASS = {
 	"apple-ane": "yolov9m-320",
@@ -1367,7 +1373,7 @@ function macroOfMinConfidenceKey(key) {
 */
 function reachableMacros(def, entry) {
 	if (entry.labels.length === 0) return void 0;
-	const map = def.classMap;
+	const map = entry.classMap ?? def.classMap;
 	if (!map) return new Set(entry.labels.map((l) => l.id));
 	if (map.preserveOriginal) return void 0;
 	const macros = /* @__PURE__ */ new Set();
@@ -1387,7 +1393,7 @@ function reachableMacros(def, entry) {
 */
 function collectClassMapDropWarning(def, entry) {
 	if (entry.labels.length === 0) return null;
-	const map = def.classMap;
+	const map = entry.classMap ?? def.classMap;
 	if (!map || map.preserveOriginal) return null;
 	const unmapped = entry.labels.filter((l) => map.mapping[l.id] === void 0).map((l) => l.id);
 	if (unmapped.length === 0) return null;
@@ -4163,7 +4169,7 @@ function applyChildOutput(parent, childStep, output, stepLatencyMs, ctx) {
 	switch (output.kind) {
 		case "detections":
 			if (childStep.definition.slot === "refiner") {
-				const classMap = childStep.definition.classMap?.mapping;
+				const classMap = childStep.effectiveClassMap?.mapping;
 				const parentMacro = parent.macroClass;
 				const best = output.detections.filter((d) => {
 					if (d.mask === void 0) return false;
@@ -4503,6 +4509,29 @@ function reprojectCropZoneDetections(output, cropOrigin) {
 		}))
 	};
 }
+function reprojectViewDetections(output, geometry) {
+	const mapPoint = (x, y) => ({
+		x: geometry.sourceCrop.left + x / geometry.scale.x,
+		y: geometry.sourceCrop.top + y / geometry.scale.y
+	});
+	return {
+		kind: "detections",
+		detections: output.detections.map((det) => {
+			const origin = mapPoint(det.bbox[0], det.bbox[1]);
+			const end = mapPoint(det.bbox[2], det.bbox[3]);
+			return {
+				...det,
+				bbox: [
+					origin.x,
+					origin.y,
+					end.x,
+					end.y
+				],
+				...det.landmarks ? { landmarks: det.landmarks.map((landmark) => mapPoint(landmark.x, landmark.y)) } : {}
+			};
+		})
+	};
+}
 var PipelineExecutor = class {
 	opts;
 	/** Bounds the full-frame-guard drop log: first example per (device,
@@ -4564,7 +4593,7 @@ var PipelineExecutor = class {
 	*
 	* @returns FrameResult + optional trace
 	*/
-	async run(tree, rootInput, fullFrameJpegProvider, imageWidth, imageHeight, deviceId, runOpts, nativeCropProvider, cropZoneBbox) {
+	async run(tree, rootInput, fullFrameJpegProvider, imageWidth, imageHeight, deviceId, runOpts, nativeCropProvider, cropZoneBbox, rootInputViewProvider) {
 		const startMs = Date.now();
 		const verbosity = runOpts?.traceVerbosity ?? "off";
 		const traceBuilder = new ExecutionTraceBuilder(verbosity, deviceId, imageWidth, imageHeight, this.opts.engineRuntime);
@@ -4612,18 +4641,20 @@ var PipelineExecutor = class {
 		};
 		for (const rootStep of tree.roots) {
 			if (debug) console.log(`[executor] rootStep=${rootStep.stepId} settings=${JSON.stringify(rootStep.settings ?? {})}`);
-			const cropPrep = rootStep.definition.extractMode === "crop-zone" && cropZoneBbox !== void 0 ? await this.prepareCropZoneInput(cropZoneBbox, fullFrameJpegProvider, imageWidth, imageHeight, deviceId) : null;
+			const rootCrop = rootStep.definition.extractMode === "crop-zone" ? cropZoneBbox : void 0;
+			const rootView = rootInputViewProvider ? await rootInputViewProvider(rootStep, rootCrop) : null;
+			const cropPrep = !rootView && rootCrop !== void 0 ? await this.prepareCropZoneInput(rootCrop, fullFrameJpegProvider, imageWidth, imageHeight, deviceId) : null;
 			const rootStart = Date.now();
-			const rawRootOutput = await this.executeStep(rootStep, cropPrep?.input ?? withInferenceDeviceId(rootInput, deviceId), cropPrep?.width ?? imageWidth, cropPrep?.height ?? imageHeight, "full-frame", void 0, void 0, traceBuilder, stepTimings, poolAgg, rootShed);
+			const rawRootOutput = await this.executeStep(rootStep, rootView?.input ?? cropPrep?.input ?? withInferenceDeviceId(rootInput, deviceId), rootView?.width ?? cropPrep?.width ?? imageWidth, rootView?.height ?? cropPrep?.height ?? imageHeight, "full-frame", void 0, void 0, traceBuilder, stepTimings, poolAgg, rootShed);
 			const rootMs = Date.now() - rootStart;
 			if (rawRootOutput === null) continue;
-			const rootOutput = cropPrep && rawRootOutput.kind === "detections" ? reprojectCropZoneDetections(rawRootOutput, cropPrep.origin) : rawRootOutput;
+			const rootOutput = rootView && rawRootOutput.kind === "detections" ? reprojectViewDetections(rawRootOutput, rootView.geometry) : cropPrep && rawRootOutput.kind === "detections" ? reprojectCropZoneDetections(rawRootOutput, cropPrep.origin) : rawRootOutput;
 			if (rootOutput.kind !== "detections") {
 				if (isEnrichmentOutput(rootOutput)) {
 					const mutable = this.synthesizeRootDetection(rootOutput, rootStep, idGen, rootMs, imageWidth, imageHeight);
 					applyChildOutput(mutable, rootStep, rootOutput, rootMs, ctx);
 					try {
-						await this.executeChildren(rootStep.children, mutable, fullFrameJpegProvider, imageWidth, imageHeight, traceBuilder, stepTimings, ctx, poolAgg, runOpts?.plane, deviceId, nativeCropProvider);
+						await this.executeChildren(rootStep.children, mutable, rootView?.jpegProvider ?? fullFrameJpegProvider, imageWidth, imageHeight, traceBuilder, stepTimings, ctx, poolAgg, runOpts?.plane, deviceId, nativeCropProvider);
 					} catch (err) {
 						this.opts.logger?.warn("Pipeline child execution failed — keeping parent detection", {
 							tags: { deviceId },
@@ -4641,12 +4672,12 @@ var PipelineExecutor = class {
 			}
 			for (const det of rootOutput.detections) {
 				const mutable = toMutableRootDetection(det, rootStep, idGen, rootMs);
-				if (rootStep.definition.classMap) {
-					const mapped = rootStep.definition.classMap.mapping[det.class];
+				if (rootStep.effectiveClassMap) {
+					const mapped = rootStep.effectiveClassMap.mapping[det.class];
 					if (mapped) {
 						mutable.originalClass = det.class;
 						mutable.macroClass = mapped;
-					} else if (!rootStep.definition.classMap.preserveOriginal) {
+					} else if (!rootStep.effectiveClassMap.preserveOriginal) {
 						(discarded ??= []).push({
 							bbox: [
 								mutable.bbox[0],
@@ -4728,7 +4759,7 @@ var PipelineExecutor = class {
 					}
 				});
 				try {
-					await this.executeChildren(rootStep.children, mutable, fullFrameJpegProvider, imageWidth, imageHeight, traceBuilder, stepTimings, ctx, poolAgg, runOpts?.plane, deviceId, nativeCropProvider);
+					await this.executeChildren(rootStep.children, mutable, rootView?.jpegProvider ?? fullFrameJpegProvider, imageWidth, imageHeight, traceBuilder, stepTimings, ctx, poolAgg, runOpts?.plane, deviceId, nativeCropProvider);
 				} catch (err) {
 					this.opts.logger?.warn("Pipeline child execution failed — keeping parent detection", {
 						tags: { deviceId },
@@ -5148,6 +5179,59 @@ function capitalize(s) {
 	return s.charAt(0).toUpperCase() + s.slice(1);
 }
 //#endregion
+//#region src/session-decode/frame-view-payload.ts
+var FRAME_VIEW_PAYLOAD_POLICY = "jpeg";
+//#endregion
+//#region src/detection-pipeline/root-frame-view.ts
+/** Resolve the pre-padding content requested from the decode worker for one root. */
+function resolveRootFrameViewSpec(model, source, crop, format = FRAME_VIEW_PAYLOAD_POLICY) {
+	const sourceWidth = crop?.width ?? source.width;
+	const sourceHeight = crop?.height ?? source.height;
+	if (model.preprocessMode !== "letterbox") return {
+		...crop ? { crop } : {},
+		content: {
+			width: model.inputSize.width,
+			height: model.inputSize.height
+		},
+		fit: "stretch",
+		format
+	};
+	const scale = Math.min(model.inputSize.width / sourceWidth, model.inputSize.height / sourceHeight);
+	return {
+		...crop ? { crop } : {},
+		content: {
+			width: Math.max(1, Math.round(sourceWidth * scale)),
+			height: Math.max(1, Math.round(sourceHeight * scale))
+		},
+		fit: "contain",
+		format
+	};
+}
+//#endregion
+//#region src/detection-pipeline/frame-ref-view-provider.ts
+var LocalFrameRefMissError = class extends Error {
+	reason;
+	constructor(reason) {
+		super(`local frameRef miss: ${reason}`);
+		this.reason = reason;
+		this.name = "LocalFrameRefMissError";
+	}
+};
+/** Bind one registry lease for all heterogeneous roots of a pipeline invocation. */
+function createRootFrameViewResolver(registry, ref) {
+	const lease = registry.acquire(ref);
+	if (!lease) throw new LocalFrameRefMissError(ref.registryId === registry.registryId ? "stale-ref" : "foreign-registry");
+	const resolve = async (root, crop) => {
+		const model = root.definition.models.find((candidate) => candidate.id === root.modelId);
+		if (!model) throw new Error(`frame view model not found: ${root.modelId}`);
+		const spec = resolveRootFrameViewSpec(model, ref, crop);
+		const result = await lease.resolve(spec);
+		if (result.kind === "miss") throw new LocalFrameRefMissError(result.reason);
+		return result;
+	};
+	return Object.assign(resolve, { release: (reason) => lease.release(reason) });
+}
+//#endregion
 //#region src/detection-pipeline/pipeline/native-crop-compose.ts
 /** Clamp `v` into `[lo, hi]`. */
 function clamp(v, lo, hi) {
@@ -5187,23 +5271,32 @@ function composeCropRoiToFrameNorm(roi, cropFrameSpace, frameWidth, frameHeight)
 //#endregion
 //#region src/detection-pipeline/pipeline/tree-builder.ts
 /**
+* Effective class map for `(definition, modelId)` without writing back to the
+* shared StepDefinition. Custom models missing from `definition.models` are
+* resolved through the optional registry callback.
+*/
+function resolveEffectiveClassMap(definition, modelId, resolveModel) {
+	return (definition.models.find((model) => model.id === modelId) ?? resolveModel?.(definition.id, modelId))?.classMap ?? definition.classMap;
+}
+/**
 * Build an executable tree from user config.
 *
 * @param steps - User-configured pipeline steps (from PipelineDefaultStep[])
 * @param getEngine - Function that returns an IInferenceEngine for a step ID.
 *                    Throws if step not loaded.
 */
-function buildExecutableTree(steps, getEngine) {
-	return { roots: steps.filter((s) => s.enabled).filter((s) => s.slot !== "audio-classifier").map((s) => buildNode(s, getEngine)) };
+function buildExecutableTree(steps, getEngine, resolveModel) {
+	return { roots: steps.filter((s) => s.enabled).filter((s) => s.slot !== "audio-classifier").map((s) => buildNode(s, getEngine, resolveModel)) };
 }
-function buildNode(step, getEngine) {
+function buildNode(step, getEngine, resolveModel) {
 	const definition = getStepDefinition(step.addonId);
 	const engine = getEngine(step.addonId);
-	const children = (step.children ?? []).filter((c) => c.enabled).map((c) => buildNode(c, getEngine));
+	const children = (step.children ?? []).filter((c) => c.enabled).map((c) => buildNode(c, getEngine, resolveModel));
 	const mergedSettings = {
 		...collectSchemaDefaults(step.addonId),
 		...step.settings
 	};
+	const effectiveClassMap = resolveEffectiveClassMap(definition, step.modelId, resolveModel);
 	return {
 		stepId: step.addonId,
 		definition,
@@ -5212,7 +5305,8 @@ function buildNode(step, getEngine) {
 		inputClasses: definition.inputClasses ?? [],
 		enabled: step.enabled,
 		children,
-		...Object.keys(mergedSettings).length > 0 ? { settings: mergedSettings } : {}
+		...Object.keys(mergedSettings).length > 0 ? { settings: mergedSettings } : {},
+		...effectiveClassMap !== void 0 ? { effectiveClassMap } : {}
 	};
 }
 /**
@@ -7091,17 +7185,30 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 		let jpegProvider;
 		const sources = [
 			input.frame ? "frame" : null,
+			input.frameRef ? "frameRef" : null,
 			input.frameHandle ? "frameHandle" : null,
 			input.image ? "image" : null,
 			input.imageBase64 ? "imageBase64" : null,
 			input.referenceImage ? "referenceImage" : null
 		].filter((s) => s !== null);
-		if (sources.length === 0) throw new Error("runPipeline requires exactly one of: frame, frameHandle, image, imageBase64, referenceImage");
+		if (sources.length === 0) throw new Error("runPipeline requires exactly one of: frame, frameRef, frameHandle, image, imageBase64, referenceImage");
 		if (sources.length > 1) throw new Error(`runPipeline received conflicting image sources: ${sources.join(", ")}`);
 		if (input.frameHandle) return this.emptyFrameHandleResult(input);
 		const runtimeFrame = input.frame ?? void 0;
+		const runtimeFrameRef = input.frameRef;
 		const decodeT0 = performance.now();
-		if (runtimeFrame) {
+		if (runtimeFrameRef) {
+			imageWidth = runtimeFrameRef.width;
+			imageHeight = runtimeFrameRef.height;
+			rootInput = {
+				kind: "jpeg",
+				data: Buffer.alloc(0)
+			};
+			jpegProvider = async () => {
+				throw new Error("frameRef root view was requested before model resolution");
+			};
+			emit(`FrameRef: ${imageWidth}×${imageHeight} (${runtimeFrameRef.format})`);
+		} else if (runtimeFrame) {
 			const frame = runtimeFrame;
 			imageWidth = frame.width;
 			imageHeight = frame.height;
@@ -7301,9 +7408,9 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 				emit(`All models loaded`);
 			}
 			emit("Running inference...");
-			const tree = buildExecutableTree(benchmarkSteps, (stepId) => dispatchFactory.getEngine(stepId));
+			const tree = buildExecutableTree(benchmarkSteps, (stepId) => dispatchFactory.getEngine(stepId), this.customModelResolver);
 			setupMs = performance.now() - wallT0 - decodeMs;
-			const isRuntime = Boolean(runtimeFrame);
+			const isRuntime = Boolean(runtimeFrame || runtimeFrameRef);
 			const effectiveDeviceId = input.deviceId ?? 0;
 			const deviceOverrides = effectiveDeviceId > 0 ? await this.readDeviceStore(effectiveDeviceId) : {};
 			const effectiveTree = Object.keys(deviceOverrides).length > 0 ? applyDeviceOverridesToTree(tree, "object-detection", deviceOverrides) : tree;
@@ -7314,10 +7421,42 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 				const proxy = this.deviceProxies.get(effectiveDeviceId);
 				if (proxy) cropZoneBbox = resolvePackageCropBbox(proxy.state.zones.value?.zones ?? [], proxy.state.zoneRules.value?.package ?? [], imageWidth, imageHeight) ?? void 0;
 			}
-			const { result, trace } = await executor.run(effectiveTree, rootInput, jpegProvider, imageWidth, imageHeight, effectiveDeviceId, {
-				traceVerbosity: isRuntime ? this.eventBus ? "summary" : "off" : "full",
-				plane: input.plane
-			}, nativeCropProvider, cropZoneBbox);
+			let frameViewResolver;
+			let rootInputViewProvider;
+			if (runtimeFrameRef) {
+				frameViewResolver = createRootFrameViewResolver(localFrameRegistry, runtimeFrameRef);
+				rootInputViewProvider = async (root, crop) => {
+					const view = await frameViewResolver(root, crop ? {
+						left: crop[0],
+						top: crop[1],
+						width: crop[2] - crop[0],
+						height: crop[3] - crop[1]
+					} : void 0);
+					const data = Buffer.from(view.data.buffer, view.data.byteOffset, view.data.byteLength);
+					return {
+						input: {
+							kind: "jpeg",
+							data
+						},
+						jpegProvider: async () => data,
+						width: view.width,
+						height: view.height,
+						geometry: view.geometry
+					};
+				};
+			}
+			let executionSucceeded = false;
+			let execution;
+			try {
+				execution = await executor.run(effectiveTree, rootInput, jpegProvider, imageWidth, imageHeight, effectiveDeviceId, {
+					traceVerbosity: isRuntime ? this.eventBus ? "summary" : "off" : "full",
+					plane: input.plane
+				}, nativeCropProvider, cropZoneBbox, rootInputViewProvider);
+				executionSucceeded = true;
+			} finally {
+				frameViewResolver?.release(executionSucceeded ? "success" : "error");
+			}
+			const { result, trace } = execution;
 			if (isRuntime) {
 				if (trace && this.eventBus) this.eventBus.emit(createEvent(EventCategory.PipelineTrace, {
 					type: "device",