@scratch/scratch-vm 11.0.0-beta.1

package/src/extensions/scratch3_video_sensing/view.js

@@ -0,0 +1,509 @@
+const {motionVector} = require('./math');
+
+const WIDTH = 480;
+const HEIGHT = 360;
+const WINSIZE = 8;
+const AMOUNT_SCALE = 100;
+const THRESHOLD = 10;
+
+/**
+ * Modes of debug output that can be rendered.
+ * @type {object}
+ */
+const OUTPUT = {
+    /**
+     * Render the original input.
+     * @type {number}
+     */
+    INPUT: -1,
+
+    /**
+     * Render the difference of neighboring pixels for each pixel. The
+     * horizontal difference, or x value, renders in the red output component.
+     * The vertical difference, or y value, renders in the green output
+     * component. Pixels with equal neighbors with a kind of lime green or
+     * #008080 in a RGB hex value. Colors with more red have a lower value to
+     * the right than the value to the left. Colors with less red have a higher
+     * value to the right than the value to the left. Similarly colors with
+     * more green have lower values below than above and colors with less green
+     * have higher values below than above.
+     * @type {number}
+     */
+    XY: 0,
+
+    /**
+     * Render the XY output with groups of pixels averaged together. The group
+     * shape and size matches the full frame's analysis window size.
+     * @type {number}
+     */
+    XY_CELL: 1,
+
+    /**
+     * Render three color components matching the detection algorith's values
+     * that multiple the horizontal difference, or x value, and the vertical
+     * difference, or y value together. The red component is the x value
+     * squared. The green component is the y value squared. The blue component
+     * is the x value times the y value. The detection code refers to these
+     * values as A2, B1, and A1B2.
+     * @type {number}
+     */
+    AB: 2,
+
+    /**
+     * Render the AB output of groups of pixels summarized by their combined
+     * square root. The group shape and size matches the full frame's analysis
+     * window size.
+     * @type {number}
+     */
+    AB_CELL: 3,
+
+    /**
+     * Render a single color component matching the temporal difference or the
+     * difference in color for the same pixel coordinate in the current frame
+     * and the last frame. The difference is rendered in the blue color
+     * component since x and y axis differences tend to use red and green.
+     * @type {number}
+     */
+    T: 4,
+
+    /**
+     * Render the T output of groups of pixels averaged. The group shape and
+     * size matches the full frame's analysis window.
+     * @type {number}
+     */
+    T_CELL: 5,
+
+    /**
+     * Render the XY and T outputs together. The x and y axis values use the
+     * red and green color components as they do in the XY output. The t values
+     * use the blue color component as the T output does.
+     * @type {number}
+     */
+    XYT: 6,
+
+    /**
+     * Render the XYT output of groups of pixels averaged. The group shape and
+     * size matches the full frame's analysis window.
+     * @type {number}
+     */
+    XYT_CELL: 7,
+
+    /**
+     * Render the horizontal pixel difference times the temporal difference as
+     * red and the vertical and temporal difference as green. Multiplcation of
+     * these values ends up with sharp differences in the output showing edge
+     * details where motion is happening.
+     * @type {number}
+     */
+    C: 8,
+
+    /**
+     * Render the C output of groups of pixels averaged. The group shape and
+     * size matches the full frame's analysis window.
+     * @type {number}
+     */
+    C_CELL: 9,
+
+    /**
+     * Render a per pixel version of UV_CELL. UV_CELL is a close to final step
+     * of the motion code that builds a motion amount and direction from those
+     * values. UV_CELL renders grouped summarized values, UV does the per pixel
+     * version but its can only represent one motion vector code path out of
+     * two choices. Determining the motion vector compares some of the built
+     * values but building the values with one pixel ensures this first
+     * comparison says the values are equal. Even though only one code path is
+     * used to build the values, its output is close to approximating the
+     * better solution building vectors from groups of pixels to help
+     * illustrate when the code determines the motion amount and direction to
+     * be.
+     * @type {number}
+     */
+    UV: 10,
+
+    /**
+     * Render cells of mulitple pixels at a step in the motion code that has
+     * the same cell values and turns them into motion vectors showing the
+     * amount of motion in the x axis and y axis separately. Those values are a
+     * step away from becoming a motion amount and direction through standard
+     * vector to magnitude and angle values.
+     * @type {number}
+     */
+    UV_CELL: 11
+};
+
+/**
+ * Temporary storage structure for returning values in
+ * VideoMotionView._components.
+ * @type {object}
+ */
+const _videoMotionViewComponentsTmp = {
+    A2: 0,
+    A1B2: 0,
+    B1: 0,
+    C2: 0,
+    C1: 0
+};
+
+/**
+ * Manage a debug canvas with VideoMotion input frames running parts of what
+ * VideoMotion does to visualize what it does.
+ * @param {VideoMotion} motion - VideoMotion with inputs to visualize
+ * @param {OUTPUT} output - visualization output mode
+ * @constructor
+ */
+class VideoMotionView {
+    constructor (motion, output = OUTPUT.XYT) {
+        /**
+         * VideoMotion instance to visualize.
+         * @type {VideoMotion}
+         */
+        this.motion = motion;
+
+        /**
+         * Debug canvas to render to.
+         * @type {HTMLCanvasElement}
+         */
+        const canvas = this.canvas = document.createElement('canvas');
+        canvas.width = WIDTH;
+        canvas.height = HEIGHT;
+
+        /**
+         * 2D context to draw to debug canvas.
+         * @type {CanvasRendering2DContext}
+         */
+        this.context = canvas.getContext('2d');
+
+        /**
+         * Visualization output mode.
+         * @type {OUTPUT}
+         */
+        this.output = output;
+
+        /**
+         * Pixel buffer to store output values into before they replace the last frames info in the debug canvas.
+         * @type {Uint32Array}
+         */
+        this.buffer = new Uint32Array(WIDTH * HEIGHT);
+    }
+
+    /**
+     * Modes of debug output that can be rendered.
+     * @type {object}
+     */
+    static get OUTPUT () {
+        return OUTPUT;
+    }
+
+    /**
+     * Iterate each pixel address location and call a function with that address.
+     * @param {number} xStart - start location on the x axis of the output pixel buffer
+     * @param {number} yStart - start location on the y axis of the output pixel buffer
+     * @param {nubmer} xStop - location to stop at on the x axis
+     * @param {number} yStop - location to stop at on the y axis
+     * @param {function} fn - handle to call with each iterated address
+     */
+    _eachAddress (xStart, yStart, xStop, yStop, fn) {
+        for (let i = yStart; i < yStop; i++) {
+            for (let j = xStart; j < xStop; j++) {
+                const address = (i * WIDTH) + j;
+                fn(address, j, i);
+            }
+        }
+    }
+
+    /**
+     * Iterate over cells of pixels and call a function with a function to
+     * iterate over pixel addresses.
+     * @param {number} xStart - start location on the x axis
+     * @param {number} yStart - start lcoation on the y axis
+     * @param {number} xStop - location to stop at on the x axis
+     * @param {number} yStop - location to stop at on the y axis
+     * @param {number} xStep - width of the cells
+     * @param {number} yStep - height of the cells
+     * @param {function} fn - function to call with a bound handle to _eachAddress
+     */
+    _eachCell (xStart, yStart, xStop, yStop, xStep, yStep, fn) {
+        const xStep2 = (xStep / 2) | 0;
+        const yStep2 = (yStep / 2) | 0;
+        for (let i = yStart; i < yStop; i += yStep) {
+            for (let j = xStart; j < xStop; j += xStep) {
+                fn(
+                    _fn => this._eachAddress(j - xStep2 - 1, i - yStep2 - 1, j + xStep2, i + yStep2, _fn),
+                    j - xStep2 - 1,
+                    i - yStep2 - 1,
+                    j + xStep2,
+                    i + yStep2
+                );
+            }
+        }
+    }
+
+    /**
+     * Build horizontal, vertical, and temporal difference of a pixel address.
+     * @param {number} address - address to build values for
+     * @returns {object} a object with a gradX, grady, and gradT value
+     */
+    _grads (address) {
+        const {curr, prev} = this.motion;
+        const gradX = (curr[address - 1] & 0xff) - (curr[address + 1] & 0xff);
+        const gradY = (curr[address - WIDTH] & 0xff) - (curr[address + WIDTH] & 0xff);
+        const gradT = (prev[address] & 0xff) - (curr[address] & 0xff);
+        return {gradX, gradY, gradT};
+    }
+
+    /**
+     * Build component values used in determining a motion vector for a pixel
+     * address.
+     * @param {function} eachAddress - a bound handle to _eachAddress to build
+     *   component values for
+     * @returns {object} a object with a A2, A1B2, B1, C2, C1 value
+     */
+    _components (eachAddress) {
+        let A2 = 0;
+        let A1B2 = 0;
+        let B1 = 0;
+        let C2 = 0;
+        let C1 = 0;
+
+        eachAddress(address => {
+            const {gradX, gradY, gradT} = this._grads(address);
+            A2 += gradX * gradX;
+            A1B2 += gradX * gradY;
+            B1 += gradY * gradY;
+            C2 += gradX * gradT;
+            C1 += gradY * gradT;
+        });
+
+        _videoMotionViewComponentsTmp.A2 = A2;
+        _videoMotionViewComponentsTmp.A1B2 = A1B2;
+        _videoMotionViewComponentsTmp.B1 = B1;
+        _videoMotionViewComponentsTmp.C2 = C2;
+        _videoMotionViewComponentsTmp.C1 = C1;
+        return _videoMotionViewComponentsTmp;
+    }
+
+    /**
+     * Visualize the motion code output mode selected for this view to the
+     * debug canvas.
+     */
+    draw () {
+        if (!(this.motion.prev && this.motion.curr)) {
+            return;
+        }
+
+        const {buffer} = this;
+
+        if (this.output === OUTPUT.INPUT) {
+            const {curr} = this.motion;
+            this._eachAddress(1, 1, WIDTH - 1, HEIGHT - 1, address => {
+                buffer[address] = curr[address];
+            });
+        }
+        if (this.output === OUTPUT.XYT) {
+            this._eachAddress(1, 1, WIDTH - 1, HEIGHT - 1, address => {
+                const {gradX, gradY, gradT} = this._grads(address);
+                const over1 = gradT / 0xcf;
+                buffer[address] =
+                    (0xff << 24) +
+                    (Math.floor((((gradY * over1) & 0xff) + 0xff) / 2) << 8) +
+                    Math.floor((((gradX * over1) & 0xff) + 0xff) / 2);
+            });
+        }
+        if (this.output === OUTPUT.XYT_CELL) {
+            const winStep = (WINSIZE * 2) + 1;
+            const wmax = WIDTH - WINSIZE - 1;
+            const hmax = HEIGHT - WINSIZE - 1;
+
+            this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => {
+                let C1 = 0;
+                let C2 = 0;
+                let n = 0;
+
+                eachAddress(address => {
+                    const {gradX, gradY, gradT} = this._grads(address);
+                    C2 += (Math.max(Math.min(gradX / 0x0f, 1), -1)) * (gradT / 0xff);
+                    C1 += (Math.max(Math.min(gradY / 0x0f, 1), -1)) * (gradT / 0xff);
+                    n += 1;
+                });
+
+                C1 /= n;
+                C2 /= n;
+                C1 = Math.log(C1 + (1 * Math.sign(C1))) / Math.log(2);
+                C2 = Math.log(C2 + (1 * Math.sign(C2))) / Math.log(2);
+
+                eachAddress(address => {
+                    buffer[address] = (0xff << 24) +
+                        (((((C1 * 0x7f) | 0) + 0x80) << 8) & 0xff00) +
+                        (((((C2 * 0x7f) | 0) + 0x80) << 0) & 0xff);
+                });
+            });
+        }
+        if (this.output === OUTPUT.XY) {
+            this._eachAddress(1, 1, WIDTH - 1, HEIGHT - 1, address => {
+                const {gradX, gradY} = this._grads(address);
+                buffer[address] = (0xff << 24) + (((gradY + 0xff) / 2) << 8) + ((gradX + 0xff) / 2);
+            });
+        }
+        if (this.output === OUTPUT.XY_CELL) {
+            const winStep = (WINSIZE * 2) + 1;
+            const wmax = WIDTH - WINSIZE - 1;
+            const hmax = HEIGHT - WINSIZE - 1;
+
+            this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => {
+                let C1 = 0;
+                let C2 = 0;
+                let n = 0;
+
+                eachAddress(address => {
+                    const {gradX, gradY} = this._grads(address);
+                    C2 += Math.max(Math.min(gradX / 0x1f, 1), -1);
+                    C1 += Math.max(Math.min(gradY / 0x1f, 1), -1);
+                    n += 1;
+                });
+
+                C1 /= n;
+                C2 /= n;
+                C1 = Math.log(C1 + (1 * Math.sign(C1))) / Math.log(2);
+                C2 = Math.log(C2 + (1 * Math.sign(C2))) / Math.log(2);
+
+                eachAddress(address => {
+                    buffer[address] = (0xff << 24) +
+                        (((((C1 * 0x7f) | 0) + 0x80) << 8) & 0xff00) +
+                        (((((C2 * 0x7f) | 0) + 0x80) << 0) & 0xff);
+                });
+            });
+        } else if (this.output === OUTPUT.T) {
+            this._eachAddress(1, 1, WIDTH - 1, HEIGHT - 1, address => {
+                const {gradT} = this._grads(address);
+                buffer[address] = (0xff << 24) + ((gradT + 0xff) / 2 << 16);
+            });
+        }
+        if (this.output === OUTPUT.T_CELL) {
+            const winStep = (WINSIZE * 2) + 1;
+            const wmax = WIDTH - WINSIZE - 1;
+            const hmax = HEIGHT - WINSIZE - 1;
+
+            this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => {
+                let T = 0;
+                let n = 0;
+
+                eachAddress(address => {
+                    const {gradT} = this._grads(address);
+                    T += gradT / 0xff;
+                    n += 1;
+                });
+
+                T /= n;
+
+                eachAddress(address => {
+                    buffer[address] = (0xff << 24) +
+                        (((((T * 0x7f) | 0) + 0x80) << 16) & 0xff0000);
+                });
+            });
+        } else if (this.output === OUTPUT.C) {
+            this._eachAddress(1, 1, WIDTH - 1, HEIGHT - 1, address => {
+                const {gradX, gradY, gradT} = this._grads(address);
+                buffer[address] =
+                    (0xff << 24) +
+                    (((Math.sqrt(gradY * gradT) * 0x0f) & 0xff) << 8) +
+                    ((Math.sqrt(gradX * gradT) * 0x0f) & 0xff);
+            });
+        }
+        if (this.output === OUTPUT.C_CELL) {
+            const winStep = (WINSIZE * 2) + 1;
+            const wmax = WIDTH - WINSIZE - 1;
+            const hmax = HEIGHT - WINSIZE - 1;
+
+            this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => {
+                let {C2, C1} = this._components(eachAddress);
+
+                C2 = Math.sqrt(C2);
+                C1 = Math.sqrt(C1);
+
+                eachAddress(address => {
+                    buffer[address] =
+                        (0xff << 24) +
+                        ((C1 & 0xff) << 8) +
+                        ((C2 & 0xff) << 0);
+                });
+            });
+        } else if (this.output === OUTPUT.AB) {
+            this._eachAddress(1, 1, WIDTH - 1, HEIGHT - 1, address => {
+                const {gradX, gradY} = this._grads(address);
+                buffer[address] =
+                    (0xff << 24) +
+                    (((gradX * gradY) & 0xff) << 16) +
+                    (((gradY * gradY) & 0xff) << 8) +
+                    ((gradX * gradX) & 0xff);
+            });
+        }
+        if (this.output === OUTPUT.AB_CELL) {
+            const winStep = (WINSIZE * 2) + 1;
+            const wmax = WIDTH - WINSIZE - 1;
+            const hmax = HEIGHT - WINSIZE - 1;
+
+            this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => {
+                let {A2, A1B2, B1} = this._components(eachAddress);
+
+                A2 = Math.sqrt(A2);
+                A1B2 = Math.sqrt(A1B2);
+                B1 = Math.sqrt(B1);
+
+                eachAddress(address => {
+                    buffer[address] =
+                        (0xff << 24) +
+                        ((A1B2 & 0xff) << 16) +
+                        ((B1 & 0xff) << 8) +
+                        (A2 & 0xff);
+                });
+            });
+        } else if (this.output === OUTPUT.UV) {
+            const winStep = (WINSIZE * 2) + 1;
+
+            this._eachAddress(1, 1, WIDTH - 1, HEIGHT - 1, address => {
+                const {A2, A1B2, B1, C2, C1} = this._components(fn => fn(address));
+                const {u, v} = motionVector(A2, A1B2, B1, C2, C1);
+
+                const inRange = (-winStep < u && u < winStep && -winStep < v && v < winStep);
+                const hypot = Math.hypot(u, v);
+                const amount = AMOUNT_SCALE * hypot;
+
+                buffer[address] =
+                    (0xff << 24) +
+                    (inRange && amount > THRESHOLD ?
+                        (((((v / winStep) + 1) / 2 * 0xff) << 8) & 0xff00) +
+                        (((((u / winStep) + 1) / 2 * 0xff) << 0) & 0xff) :
+                        0x8080
+                    );
+            });
+        } else if (this.output === OUTPUT.UV_CELL) {
+            const winStep = (WINSIZE * 2) + 1;
+            const wmax = WIDTH - WINSIZE - 1;
+            const hmax = HEIGHT - WINSIZE - 1;
+
+            this._eachCell(WINSIZE + 1, WINSIZE + 1, wmax, hmax, winStep, winStep, eachAddress => {
+                const {A2, A1B2, B1, C2, C1} = this._components(eachAddress);
+                const {u, v} = motionVector(A2, A1B2, B1, C2, C1);
+
+                const inRange = (-winStep < u && u < winStep && -winStep < v && v < winStep);
+                const hypot = Math.hypot(u, v);
+                const amount = AMOUNT_SCALE * hypot;
+
+                eachAddress(address => {
+                    buffer[address] =
+                        (0xff << 24) +
+                        (inRange && amount > THRESHOLD ?
+                            (((((v / winStep) + 1) / 2 * 0xff) << 8) & 0xff00) +
+                            (((((u / winStep) + 1) / 2 * 0xff) << 0) & 0xff) :
+                            0x8080
+                        );
+                });
+            });
+        }
+
+        const data = new ImageData(new Uint8ClampedArray(this.buffer.buffer), WIDTH, HEIGHT);
+        this.context.putImageData(data, 0, 0);
+    }
+}
+
+module.exports = VideoMotionView;