@kubohiroya/turbowarp-teachable-machine 3.4.0 → 3.5.0

package/dist/tm.js

@@ -553,7 +553,7 @@
   }
   var package_default = {
   	name: "@kubohiroya/turbowarp-teachable-machine",
-  	version: "3.4.0",
+  	version: "3.5.0",
   	description: "A TurboWarp extension for recognition using Teachable Machine pose, image, and audio models.",
   	author: "Hiroya Kubo <hiroya@cuc.ac.jp>",
   	license: "MPL-2.0",
@@ -670,6 +670,33 @@
   var DOCS_URI = "https://kubohiroya.github.io/turbowarp-teachable-machine/";
   var ACCUMULATED_POSE_CHANGED_EVENT = "TM_ACCUMULATED_POSE_CHANGED";
   var BLOCK_ICON_URI = `data:image/svg+xml,${encodeURIComponent("<svg xmlns=\"http://www.w3.org/2000/svg\" viewBox=\"0 0 64 64\"><g fill=\"none\" stroke=\"#fff\" stroke-width=\"5\" stroke-linecap=\"round\" stroke-linejoin=\"round\"><path d=\"M8 21V8h13M43 8h13v13M8 43v13h13M43 56h13V43M32 25v15M20 31l12 5 12-5M32 40 23 52M32 40l9 12\"/><circle cx=\"32\" cy=\"18\" r=\"5\"/></g></svg>")}`;
+  /**
+  * Put keypoints where the preview shows them: in the camera frame's own space.
+  *
+  * The model sees a square centre crop with the pixels mirrored, so a keypoint comes back measured
+  * from the mirrored square's left edge. Undoing the mirror and adding the crop's offset puts it
+  * back on the frame, which is what the preview shows and what the overlay's viewBox spans. The
+  * preview's own mirroring is a CSS transform over both the video and the overlay, so these
+  * coordinates stay in frame space either way.
+  */
+  function mapKeypointsToFrame(keypoints, crop) {
+  	if (!Array.isArray(keypoints) || !(crop.size > 0)) return keypoints;
+  	return keypoints.map((keypoint) => {
+  		if (typeof keypoint !== "object" || keypoint === null) return keypoint;
+  		const position = keypoint.position;
+  		const x = Number(position?.x);
+  		const y = Number(position?.y);
+  		if (!Number.isFinite(x) || !Number.isFinite(y)) return keypoint;
+  		return {
+  			...keypoint,
+  			position: {
+  				...position,
+  				x: crop.left + (crop.size - x),
+  				y: crop.top + y
+  			}
+  		};
+  	});
+  }
   var BROWSER_RUNTIME_URL = `https://cdn.jsdelivr.net/npm/${package_default.name}@${package_default.version}/dist/runtime.js`;
   var POSITION_ITEMS = [
   	{
@@ -1214,6 +1241,7 @@
   			const constraints = lease ? void 0 : cameraConstraints(this.resolvedCameraSelection());
   			if (constraints) await this.webcam.setup(constraints);
   			else await this.webcam.setup();
+  			this.ensureCameraCanvas();
   			initializeCameraReadbackContext(this.webcam.canvas);
   			if (!lease) await this.webcam.play();
   			this.attachPreviewToStage();
@@ -1231,6 +1259,23 @@
   		}
   	}
   	/**
+  	* Give the camera the readback canvas `setup()` did not build.
+  	*
+  	* Upstream creates that canvas inside the same branch that opens the camera, so an injected
+  	* element skips the canvas with the `getUserMedia` call. 3.4.0 shipped assuming the two were
+  	* separate, and every leased start then failed at the first context request with "Webcam canvas
+  	* does not provide a 2D context" -- which is to say pose recognition did not start at all
+  	* wherever Camera Source was present. The canvas built here is the one upstream would have
+  	* built, so the mirrored centre crop every downstream reader expects is unchanged.
+  	*/
+  	ensureCameraCanvas() {
+  		if (!this.webcam || this.webcam.canvas) return;
+  		const canvas = document.createElement("canvas");
+  		canvas.width = this.webcam.width ?? 320;
+  		canvas.height = this.webcam.height ?? 240;
+  		this.webcam.canvas = canvas;
+  	}
+  	/**
   	* A lease on the shared camera, or null when Camera Source is not loaded.
   	*
   	* Camera Source owns the device and decides when a stream stops, which is what lets one camera
@@ -1629,7 +1674,7 @@
   		if (!this.featureFlags.poseOverlay) return null;
   		if (this.poseOverlaySvg) return this.poseOverlaySvg;
   		const svg = this.createSvgElement("svg");
-  		svg.setAttribute("viewBox", "0 0 320 240");
+  		svg.setAttribute("viewBox", `0 0 320 240`);
   		svg.setAttribute("width", "320");
   		svg.setAttribute("height", "240");
   		svg.setAttribute("preserveAspectRatio", "xMidYMid meet");
@@ -1735,11 +1780,23 @@
   			line.style.display = "block";
   		}
   	}
+  	/**
+  	* The element the stage shows.
+  	*
+  	* The camera's own video, when there is one: it carries the camera's resolution and its aspect,
+  	* so the preview stops being a 320x240 readback canvas stretched to fit. The canvas remains the
+  	* preview only where there is no video to show, which is the same place inference falls back to
+  	* it.
+  	*/
+  	previewSourceElement() {
+  		const video = this.webcam?.webcam;
+  		return video && video.videoWidth > 0 ? video : this.webcam?.canvas;
+  	}
   	attachPreviewToStage() {
   		if (!this.webcam) throw new Error("TM: Start the camera before attaching the preview.");
   		if (!this.webcam.canvas) throw new Error("TM: webcam.canvas is unavailable.");
   		const stage = this.findStageElement();
-  		const canvas = this.webcam.canvas;
+  		const canvas = this.previewSourceElement();
   		let stageCanvas = null;
   		try {
   			const candidate = this.findLikelyStageCanvas();
@@ -1783,6 +1840,22 @@
   		this.updatePoseOverlayVisibility();
   		this.validatePreviewAttachment(stage, canvas);
   	}
+  	/**
+  	* Span the overlay over what the preview is showing.
+  	*
+  	* Keypoints are in the camera frame's space, so the viewBox has to be the frame -- the video's
+  	* own width and height, not the readback canvas's fixed 320x240. On the canvas fallback the two
+  	* are the same thing.
+  	*/
+  	updatePoseOverlayViewBox() {
+  		const svg = this.poseOverlaySvg;
+  		const preview = this.previewCanvas;
+  		if (!svg || !preview) return;
+  		const width = preview.videoWidth ?? preview.width;
+  		const height = preview.videoHeight ?? preview.height;
+  		if (!(width > 0) || !(height > 0)) return;
+  		svg.setAttribute("viewBox", `0 0 ${width} ${height}`);
+  	}
   	updatePreviewStyle() {
   		const canvas = this.previewCanvas;
   		if (!canvas) throw new Error("TM: Start the camera before positioning the preview.");
@@ -1840,9 +1913,10 @@
   				target.style.bottom = "8px";
   			}
   		}
-  		const mirroringTransform = this.previewMirrored ? "" : "scaleX(-1)";
+  		const mirroringTransform = this.previewCanvas === this.webcam?.webcam ? this.previewMirrored ? "scaleX(-1)" : "" : this.previewMirrored ? "" : "scaleX(-1)";
   		for (const target of targets) target.style.transform = [positionTransform, mirroringTransform].filter(Boolean).join(" ");
   		this.poseOverlaySvg?.setAttribute("preserveAspectRatio", this.previewPosition === "full-stage" ? "xMidYMid slice" : "xMidYMid meet");
+  		this.updatePoseOverlayViewBox();
   	}
   	startLoopIfNeeded() {
   		if (this.loopStarted || !this.cameraRunning || !this.webcam) return;
@@ -1875,7 +1949,7 @@
   			return;
   		}
   		try {
-  			this.webcam.update();
+  			if (this.previewSourceElement() !== this.webcam.webcam) this.webcam.update();
   			if (this.recognizing && this.model) {
   				const model = this.model;
   				const recognitionMode = this.recognitionMode;
@@ -1894,15 +1968,73 @@
   		if (generation === this.loopGeneration && this.cameraRunning && this.webcam) requestAnimationFrame(() => void this.loop(generation));
   		else if (generation === this.loopGeneration) this.loopStarted = false;
   	}
+  	/**
+  	* The square the model looks at, as a tensor, taken straight off the camera.
+  	*
+  	* The readback canvas is 320x240 and holds a 320x320 centre crop drawn into it, so its bottom
+  	* quarter -- 25% of the subject's height -- never reaches the model. Cropping the frame itself
+  	* keeps the whole square. The crop and the mirroring both run as tensor ops, which is where the
+  	* backend can keep them on the GPU; `fromPixels` uploads a video element directly to a texture
+  	* on the WebGL and WebGPU backends, so no Canvas2D stage sits in the path.
+  	*
+  	* The pixels are mirrored here rather than through `estimatePose(sample, flipHorizontal)`:
+  	* that flag flips the returned keypoints, not the image, and the classifier head was trained on
+  	* mirrored frames. Flipping the pixels is what keeps the model's input the same as before.
+  	*
+  	* Returns null when there is no frame to take, and the canvas path answers instead.
+  	*/
+  	captureSquareFrame() {
+  		const tensorflow = globalThis.tf;
+  		const video = this.webcam?.webcam;
+  		if (typeof tensorflow?.browser?.fromPixels !== "function" || typeof tensorflow.tidy !== "function" || !video || !(video.videoWidth > 0) || !(video.videoHeight > 0)) return null;
+  		const size = Math.min(video.videoWidth, video.videoHeight);
+  		const frame = tensorflow.browser.fromPixels(video);
+  		try {
+  			const top = frame.shape[0] - size >> 1;
+  			const left = frame.shape[1] - size >> 1;
+  			return {
+  				tensor: tensorflow.tidy(() => tensorflow.reverse(tensorflow.slice(frame, [
+  					top,
+  					left,
+  					0
+  				], [
+  					size,
+  					size,
+  					3
+  				]), [1])),
+  				crop: {
+  					left,
+  					top,
+  					size
+  				}
+  			};
+  		} finally {
+  			frame.dispose();
+  		}
+  	}
   	async recognizeFrame(model, recognitionMode) {
   		if (recognitionMode === "pose") {
   			if (typeof model.estimatePose !== "function") throw new Error("Teachable Machine: The active model does not support pose recognition.");
-  			const estimate = await model.estimatePose(this.webcam.canvas);
-  			const prediction = await model.predict(estimate.posenetOutput);
-  			return {
-  				keypoints: estimate.pose?.keypoints,
-  				prediction
-  			};
+  			const square = this.captureSquareFrame();
+  			if (!square) {
+  				this.webcam.update?.();
+  				const estimate = await model.estimatePose(this.webcam.canvas);
+  				const prediction = await model.predict(estimate.posenetOutput);
+  				return {
+  					keypoints: estimate.pose?.keypoints,
+  					prediction
+  				};
+  			}
+  			try {
+  				const estimate = await model.estimatePose(square.tensor);
+  				const prediction = await model.predict(estimate.posenetOutput);
+  				return {
+  					keypoints: mapKeypointsToFrame(estimate.pose?.keypoints, square.crop),
+  					prediction
+  				};
+  			} finally {
+  				square.tensor.dispose();
+  			}
   		}
   		const source = this.webcam.webcam ?? this.webcam.canvas;
   		return {

package/dist/types/extension.d.ts

@@ -5,8 +5,27 @@ export declare const EXTENSION_ID = "kubohiroyatm";
 export declare const VERSION: string;
 export declare const DOCS_URI = "https://kubohiroya.github.io/turbowarp-teachable-machine/";
 export declare const ACCUMULATED_POSE_CHANGED_EVENT = "TM_ACCUMULATED_POSE_CHANGED";
+/** The readback canvas size every downstream frame reader is written against. */
+export declare const CAMERA_FRAME_WIDTH = 320;
+export declare const CAMERA_FRAME_HEIGHT = 240;
 export declare const BLOCK_ICON_URI: string;
 export type RecognitionMode = 'pose' | 'image' | 'audio';
+/** Where the square the model saw sits inside the camera's frame. */
+export interface PoseCropWindow {
+    readonly left: number;
+    readonly top: number;
+    readonly size: number;
+}
+/**
+ * Put keypoints where the preview shows them: in the camera frame's own space.
+ *
+ * The model sees a square centre crop with the pixels mirrored, so a keypoint comes back measured
+ * from the mirrored square's left edge. Undoing the mirror and adding the crop's offset puts it
+ * back on the frame, which is what the preview shows and what the overlay's viewBox spans. The
+ * preview's own mirroring is a CSS transform over both the video and the overlay, so these
+ * coordinates stay in frame space either way.
+ */
+export declare function mapKeypointsToFrame(keypoints: unknown, crop: PoseCropWindow): unknown;
 export interface TeachableMachineRuntime {
     Webcam: new (width: number, height: number, flipHorizontal: boolean) => any;
     load?(modelURL: string, metadataURL: string): Promise<any>;
@@ -152,6 +171,17 @@ export declare class TMExtension {
     ensureLibrariesLoaded(): Promise<void>;
     cleanupCameraResources(): void;
     startCamera(): Promise<void>;
+    /**
+     * Give the camera the readback canvas `setup()` did not build.
+     *
+     * Upstream creates that canvas inside the same branch that opens the camera, so an injected
+     * element skips the canvas with the `getUserMedia` call. 3.4.0 shipped assuming the two were
+     * separate, and every leased start then failed at the first context request with "Webcam canvas
+     * does not provide a 2D context" -- which is to say pose recognition did not start at all
+     * wherever Camera Source was present. The canvas built here is the one upstream would have
+     * built, so the mirrored centre crop every downstream reader expects is unchanged.
+     */
+    ensureCameraCanvas(): void;
     /**
      * A lease on the shared camera, or null when Camera Source is not loaded.
      *
@@ -217,12 +247,52 @@ export declare class TMExtension {
     clearPoseOverlay(): void;
     redrawPoseOverlay(): void;
     renderPoseOverlay(keypoints: unknown): void;
+    /**
+     * The element the stage shows.
+     *
+     * The camera's own video, when there is one: it carries the camera's resolution and its aspect,
+     * so the preview stops being a 320x240 readback canvas stretched to fit. The canvas remains the
+     * preview only where there is no video to show, which is the same place inference falls back to
+     * it.
+     */
+    previewSourceElement(): any;
     attachPreviewToStage(): void;
+    /**
+     * Span the overlay over what the preview is showing.
+     *
+     * Keypoints are in the camera frame's space, so the viewBox has to be the frame -- the video's
+     * own width and height, not the readback canvas's fixed 320x240. On the canvas fallback the two
+     * are the same thing.
+     */
+    updatePoseOverlayViewBox(): void;
     updatePreviewStyle(): void;
     startLoopIfNeeded(): void;
     private trackPreparedModelOperation;
     waitForPreparedModelIdle(model: object): Promise<void>;
     loop(generation?: any): Promise<void>;
+    /**
+     * The square the model looks at, as a tensor, taken straight off the camera.
+     *
+     * The readback canvas is 320x240 and holds a 320x320 centre crop drawn into it, so its bottom
+     * quarter -- 25% of the subject's height -- never reaches the model. Cropping the frame itself
+     * keeps the whole square. The crop and the mirroring both run as tensor ops, which is where the
+     * backend can keep them on the GPU; `fromPixels` uploads a video element directly to a texture
+     * on the WebGL and WebGPU backends, so no Canvas2D stage sits in the path.
+     *
+     * The pixels are mirrored here rather than through `estimatePose(sample, flipHorizontal)`:
+     * that flag flips the returned keypoints, not the image, and the classifier head was trained on
+     * mirrored frames. Flipping the pixels is what keeps the model's input the same as before.
+     *
+     * Returns null when there is no frame to take, and the canvas path answers instead.
+     */
+    captureSquareFrame(): {
+        tensor: any;
+        crop: {
+            left: number;
+            top: number;
+            size: number;
+        };
+    } | null;
     recognizeFrame(model: any, recognitionMode: RecognitionMode): Promise<{
         keypoints: any;
         prediction: any;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kubohiroya/turbowarp-teachable-machine",
-  "version": "3.4.0",
+  "version": "3.5.0",
   "description": "A TurboWarp extension for recognition using Teachable Machine pose, image, and audio models.",
   "author": "Hiroya Kubo <hiroya@cuc.ac.jp>",
   "license": "MPL-2.0",