@yume-chan/scrcpy-decoder-webcodecs 2.5.3 → 3.0.0-beta.0

package/src/video/render/webgl.ts

@@ -1,59 +1,189 @@
-import { createCanvas } from "@yume-chan/scrcpy-decoder-tinyh264";
+// cspell: ignore highp
+// cspell: ignore mediump
+
+import { StickyEventEmitter } from "@yume-chan/event";
+import {
+    glCreateContext,
+    glIsSupported,
+    glLoseContext,
+} from "@yume-chan/scrcpy-decoder-shared";
 
 import { CanvasVideoFrameRenderer } from "./canvas.js";
 
-const Resolved = Promise.resolve();
-
-function createContext(
-    canvas: HTMLCanvasElement | OffscreenCanvas,
-    enableCapture?: boolean,
-): WebGLRenderingContext | null {
-    const attributes: WebGLContextAttributes = {
-        // Low-power GPU should be enough for video rendering.
-        powerPreference: "low-power",
-        alpha: false,
-        // Disallow software rendering.
-        // Other rendering methods are faster than software-based WebGL.
-        failIfMajorPerformanceCaveat: true,
-        preserveDrawingBuffer: !!enableCapture,
-    };
-
-    return (
-        canvas.getContext("webgl2", attributes) ||
-        canvas.getContext("webgl", attributes)
+function createShader(gl: WebGLRenderingContext, type: number, source: string) {
+    const shader = gl.createShader(type)!;
+    gl.shaderSource(shader, source);
+    gl.compileShader(shader);
+    return shader;
+}
+
+function createProgram(
+    gl: WebGLRenderingContext,
+    vertexShaderSource: string,
+    fragmentShaderSource: string,
+) {
+    const vertexShader = createShader(gl, gl.VERTEX_SHADER, vertexShaderSource);
+    const fragmentShader = createShader(
+        gl,
+        gl.FRAGMENT_SHADER,
+        fragmentShaderSource,
     );
+
+    const program = gl.createProgram();
+    gl.attachShader(program, vertexShader);
+    gl.attachShader(program, fragmentShader);
+    gl.linkProgram(program);
+
+    try {
+        if (gl.getProgramParameter(program, gl.LINK_STATUS)) {
+            return program;
+        }
+
+        // Don't check shader compile status unless linking fails
+        // https://developer.mozilla.org/en-US/docs/Web/API/WebGL_API/WebGL_best_practices#dont_check_shader_compile_status_unless_linking_fails
+        if (!gl.getShaderParameter(vertexShader, gl.COMPILE_STATUS)) {
+            throw new Error(gl.getShaderInfoLog(vertexShader)!);
+        }
+
+        if (!gl.getShaderParameter(fragmentShader, gl.COMPILE_STATUS)) {
+            throw new Error(gl.getShaderInfoLog(fragmentShader)!);
+        }
+
+        throw new Error(gl.getProgramInfoLog(program)!);
+    } finally {
+        // Delete objects eagerly
+        // https://developer.mozilla.org/en-US/docs/Web/API/WebGL_API/WebGL_best_practices#delete_objects_eagerly
+        gl.deleteShader(vertexShader);
+        gl.deleteShader(fragmentShader);
+    }
 }
 
-export class WebGLVideoFrameRenderer extends CanvasVideoFrameRenderer {
-    static vertexShaderSource = `
+export class WebGLVideoFrameRenderer extends CanvasVideoFrameRenderer<WebGLVideoFrameRenderer.Options> {
+    static VertexShader = `
         attribute vec2 xy;
 
         varying highp vec2 uv;
 
         void main(void) {
             gl_Position = vec4(xy, 0.0, 1.0);
+
             // Map vertex coordinates (-1 to +1) to UV coordinates (0 to 1).
+            uv = xy * 0.5 + 0.5;
             // UV coordinates are Y-flipped relative to vertex coordinates.
-            uv = vec2((1.0 + xy.x) / 2.0, (1.0 - xy.y) / 2.0);
+            uv.y = 1.0 - uv.y;
         }
 `;
 
-    static fragmentShaderSource = `
-        varying highp vec2 uv;
+    static FragmentShader = `
+        precision mediump float;
 
-        uniform sampler2D texture;
+        uniform sampler2D source;
+        uniform vec2 texelSize;
+        uniform float zoom;
 
-        void main(void) {
-            gl_FragColor = texture2D(texture, uv);
+        varying vec2 uv;
+
+        vec4 tent4(vec2 uv) {
+            vec2 dx = vec2(texelSize.x, 0.0);
+            vec2 dy = vec2(0.0, texelSize.y);
+
+            vec4 c0 = texture2D(source, uv);
+            vec4 c1 = texture2D(source, uv + dx);
+            vec4 c2 = texture2D(source, uv + dy);
+            vec4 c3 = texture2D(source, uv + dx + dy);
+
+            return 0.25 * (c0 + c1 + c2 + c3);
+        }
+
+        float mnWeight(float x) {
+            x = abs(x);
+            float x2 = x * x;
+            float x3 = x2 * x;
+
+            if (x < 1.0) {
+                return (1.0/6.0) * ((12.0 - 9.0 * (1.0/3.0) - 6.0 * (1.0/3.0)) * x3 +
+                                    (-18.0 + 12.0 * (1.0/3.0) + 6.0 * (1.0/3.0)) * x2 +
+                                    (6.0 - 2.0 * (1.0/3.0)));
+            } else if (x < 2.0) {
+                return (1.0/6.0) * ((- (1.0/3.0) - 6.0 * (1.0/3.0)) * x3 +
+                                    ((6.0 * (1.0/3.0) + 30.0 * (1.0/3.0)) * x2 +
+                                    (-12.0 * (1.0/3.0) - 48.0 * (1.0/3.0)) * x +
+                                    (8.0 * (1.0/3.0) + 24.0 * (1.0/3.0))));
+            }
+            return 0.0;
+        }
+
+        vec4 bicubicMN(vec2 uv, vec2 texelSize) {
+            vec2 texCoord = uv / texelSize;
+            vec2 base = floor(texCoord - 0.5);
+            vec2 f = texCoord - base - 0.5;
+
+            vec4 sum = vec4(0.0);
+            float total = 0.0;
+
+            for (int j = -1; j <= 2; j++) {
+                float wy = mnWeight(float(j) - f.y);
+                for (int i = -1; i <= 2; i++) {
+                    float wx = mnWeight(float(i) - f.x);
+                    float w = wx * wy;
+
+                    vec2 coord = (base + vec2(float(i), float(j)) + 0.5) * texelSize;
+                    sum += texture2D(source, coord) * w;
+                    total += w;
+                }
+            }
+            return sum / total;
+        }
+
+        void main() {
+            if (zoom > 0.95) {
+                gl_FragColor = texture2D(source, uv);
+            }
+            else if (zoom > 0.5) {
+                gl_FragColor = bicubicMN(uv, texelSize);
+            }
+            else {
+                gl_FragColor = tent4(uv);
+            }
         }
 `;
 
+    /**
+     * A single oversized triangle that covers the entire canvas.
+     */
+    static Vertices = new Float32Array([-1.0, -1.0, 3.0, -1.0, -1.0, 3.0]);
+
     static get isSupported() {
-        const canvas = createCanvas();
-        return !!createContext(canvas);
+        return glIsSupported({
+            // Disallow software rendering.
+            // `ImageBitmapRenderingContext` is faster than software-based WebGL.
+            failIfMajorPerformanceCaveat: true,
+        });
+    }
+
+    override get type() {
+        return "hardware" as const;
+    }
+
+    #contextLostTimer: number | undefined;
+    #onContextLost = new StickyEventEmitter<void>();
+    /**
+     * Fired when the WebGL context is lost and cannot be restored.
+     *
+     * The current renderer will be automatically disposed.
+     */
+    get onContextLost() {
+        return this.#onContextLost.event;
     }
 
     #context: WebGLRenderingContext;
+    #program!: WebGLProgram;
+    #vertexBuffer!: WebGLBuffer;
+    #texture!: WebGLTexture;
+    #texelSizeLocation!: WebGLUniformLocation;
+    #zoomLocation!: WebGLUniformLocation;
+
+    #abortController = new AbortController();
 
     /**
      * Create a new WebGL frame renderer.
@@ -62,83 +192,189 @@ export class WebGLVideoFrameRenderer extends CanvasVideoFrameRenderer {
      * Whether to allow capturing the canvas content using APIs like `readPixels` and `toDataURL`.
      * Enable this option may reduce performance.
      */
-    constructor(
-        canvas?: HTMLCanvasElement | OffscreenCanvas,
-        enableCapture?: boolean,
-    ) {
-        super(canvas);
+    constructor(options?: WebGLVideoFrameRenderer.Options) {
+        super((frame): undefined => {
+            const gl = this.#context;
+            if (gl.isContextLost()) {
+                // The base class will close the frame.
+                return;
+            }
+
+            gl.texImage2D(
+                gl.TEXTURE_2D,
+                0,
+                gl.RGBA,
+                gl.RGBA,
+                gl.UNSIGNED_BYTE,
+                frame,
+            );
 
-        const gl = createContext(this.canvas, enableCapture);
+            gl.uniform2f(
+                this.#texelSizeLocation,
+                1.0 / frame.codedWidth,
+                1.0 / frame.codedHeight,
+            );
+
+            gl.uniform1f(
+                this.#zoomLocation,
+                this.canvas.width / frame.codedWidth,
+            );
+
+            gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight);
+            gl.drawArrays(gl.TRIANGLES, 0, 3);
+
+            gl.flush();
+        }, options);
+
+        const gl = glCreateContext(this.canvas, {
+            // Low-power GPU should be enough for video rendering.
+            // Note that `OffscreenCanvas` created in Web Workers can only use `low-power` anyway.
+            powerPreference: "low-power",
+            // Avoid alpha:false, which can be expensive
+            // https://developer.mozilla.org/en-US/docs/Web/API/WebGL_API/WebGL_best_practices#avoid_alphafalse_which_can_be_expensive
+            alpha: true,
+            // Disallow software rendering.
+            // `ImageBitmapRenderingContext` is faster than software-based WebGL.
+            failIfMajorPerformanceCaveat: true,
+            preserveDrawingBuffer: !!this.options?.enableCapture,
+            // Enable desynchronized mode when not capturing to reduce latency.
+            desynchronized: !this.options?.enableCapture,
+            antialias: false,
+            depth: false,
+            premultipliedAlpha: true,
+            stencil: false,
+        });
         if (!gl) {
-            throw new Error("WebGL not supported");
+            throw new Error("WebGL not supported, check `isSupported` first");
         }
         this.#context = gl;
 
-        const vertexShader = gl.createShader(gl.VERTEX_SHADER)!;
-        gl.shaderSource(
-            vertexShader,
-            WebGLVideoFrameRenderer.vertexShaderSource,
+        this.#initialize();
+
+        this.canvas.addEventListener(
+            "webglcontextlost",
+            (e) => {
+                if (options?.contextRestoreTimeout === 0) {
+                    this.#onContextLost.fire();
+                    this.dispose();
+                    return;
+                }
+
+                // Notify WebGL we want to handle context restoration
+                e.preventDefault();
+
+                this.#contextLostTimer = globalThis.setTimeout(() => {
+                    this.#onContextLost.fire();
+                    this.dispose();
+                }, options?.contextRestoreTimeout ?? 3000);
+            },
+            { signal: this.#abortController.signal },
         );
-        gl.compileShader(vertexShader);
-        if (!gl.getShaderParameter(vertexShader, gl.COMPILE_STATUS)) {
-            throw new Error(gl.getShaderInfoLog(vertexShader)!);
-        }
+        this.canvas.addEventListener(
+            "webglcontextrestored",
+            () => {
+                if (this.#onContextLost.hasValue) {
+                    // Don't restore context if `onContextLost` has been fired,
+                    // which means the caller will handle the context loss.
+                    return;
+                }
 
-        const fragmentShader = gl.createShader(gl.FRAGMENT_SHADER)!;
-        gl.shaderSource(
-            fragmentShader,
-            WebGLVideoFrameRenderer.fragmentShaderSource,
+                globalThis.clearTimeout(this.#contextLostTimer);
+                this.#contextLostTimer = undefined;
+
+                this.#initialize();
+                void this.redraw();
+            },
+            { signal: this.#abortController.signal },
         );
-        gl.compileShader(fragmentShader);
-        if (!gl.getShaderParameter(fragmentShader, gl.COMPILE_STATUS)) {
-            throw new Error(gl.getShaderInfoLog(fragmentShader)!);
-        }
+    }
 
-        const shaderProgram = gl.createProgram();
-        gl.attachShader(shaderProgram, vertexShader);
-        gl.attachShader(shaderProgram, fragmentShader);
-        gl.linkProgram(shaderProgram);
-        if (!gl.getProgramParameter(shaderProgram, gl.LINK_STATUS)) {
-            throw new Error(gl.getProgramInfoLog(shaderProgram)!);
-        }
-        gl.useProgram(shaderProgram);
+    #initialize() {
+        const gl = this.#context;
+
+        this.#program = createProgram(
+            gl,
+            WebGLVideoFrameRenderer.VertexShader,
+            WebGLVideoFrameRenderer.FragmentShader,
+        );
+        gl.useProgram(this.#program);
 
         // Vertex coordinates, clockwise from bottom-left.
-        const vertexBuffer = gl.createBuffer();
-        gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer);
+        this.#vertexBuffer = gl.createBuffer();
+        gl.bindBuffer(gl.ARRAY_BUFFER, this.#vertexBuffer);
         gl.bufferData(
             gl.ARRAY_BUFFER,
-            new Float32Array([-1.0, -1.0, -1.0, +1.0, +1.0, +1.0, +1.0, -1.0]),
+            WebGLVideoFrameRenderer.Vertices,
             gl.STATIC_DRAW,
         );
 
-        const xyLocation = gl.getAttribLocation(shaderProgram, "xy");
+        const xyLocation = gl.getAttribLocation(this.#program, "xy");
         gl.vertexAttribPointer(xyLocation, 2, gl.FLOAT, false, 0, 0);
         gl.enableVertexAttribArray(xyLocation);
 
+        this.#texelSizeLocation = gl.getUniformLocation(
+            this.#program,
+            "texelSize",
+        )!;
+        this.#zoomLocation = gl.getUniformLocation(this.#program, "zoom")!;
+
         // Create one texture to upload frames to.
-        const texture = gl.createTexture();
-        gl.bindTexture(gl.TEXTURE_2D, texture);
+        this.#texture = gl.createTexture();
+        gl.bindTexture(gl.TEXTURE_2D, this.#texture);
         gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
         gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
         gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
         gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
     }
 
-    draw(frame: VideoFrame): Promise<void> {
-        const gl = this.#context;
-        gl.texImage2D(
-            gl.TEXTURE_2D,
-            0,
-            gl.RGBA,
-            gl.RGBA,
-            gl.UNSIGNED_BYTE,
-            frame,
-        );
+    override async snapshot(
+        options?: ImageEncodeOptions,
+    ): Promise<Blob | undefined> {
+        if (!this.options?.enableCapture) {
+            return undefined;
+        }
+        return super.snapshot(options);
+    }
+
+    override dispose(): undefined {
+        globalThis.clearTimeout(this.#contextLostTimer);
+        this.#onContextLost.dispose();
+
+        this.#abortController.abort();
+
+        this.#context.deleteBuffer(this.#vertexBuffer);
+        this.#context.deleteTexture(this.#texture);
+        this.#context.deleteProgram(this.#program);
+
+        if (!this.#context.isContextLost()) {
+            // Lose contexts eagerly
+            // https://developer.mozilla.org/en-US/docs/Web/API/WebGL_API/WebGL_best_practices#lose_contexts_eagerly
+            glLoseContext(this.#context);
+        }
+
+        super.dispose();
+    }
+}
 
-        gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight);
-        gl.drawArrays(gl.TRIANGLE_FAN, 0, 4);
+export namespace WebGLVideoFrameRenderer {
+    export interface Options extends CanvasVideoFrameRenderer.Options {
+        /**
+         * Whether to allow capturing the canvas content using APIs
+         * like `readPixels` and `toDataURL`.
+         *
+         * Enabling this option may reduce performance.
+         */
+        enableCapture?: boolean;
 
-        return Resolved;
+        /**
+         * The timeout in milliseconds to wait for context restoration
+         * before firing `onContextLost`.
+         *
+         * When set to 0, also disables automatic context restoration,
+         * `onContextLost` will be fired immediately when the context is lost.
+         *
+         * Default is 3000.
+         */
+        contextRestoreTimeout?: number;
     }
 }

package/src/video/utils/index.ts

@@ -0,0 +1,3 @@
+export * from "./snapshot.js";
+export * from "./timestamp.js";
+export * from "./video-decoder-stream.js";

package/src/video/utils/snapshot.ts

@@ -0,0 +1,22 @@
+export function canvasToBlob(
+    canvas: HTMLCanvasElement | OffscreenCanvas,
+    options?: ImageEncodeOptions,
+) {
+    if (canvas instanceof OffscreenCanvas) {
+        return canvas.convertToBlob(options);
+    } else {
+        return new Promise<Blob>((resolve, reject) => {
+            canvas.toBlob(
+                (blob) => {
+                    if (!blob) {
+                        reject(new Error("Failed to convert canvas to blob"));
+                    } else {
+                        resolve(blob);
+                    }
+                },
+                options?.type,
+                options?.quality,
+            );
+        });
+    }
+}

package/src/video/utils/timestamp.ts

@@ -0,0 +1,108 @@
+import type { ScrcpyVideoDecoderPauseController } from "@yume-chan/scrcpy-decoder-shared";
+import { TransformStream } from "@yume-chan/stream-extra";
+
+import type { CodecTransformStream } from "../codec/type.js";
+
+const view = new DataView(new ArrayBuffer(8));
+
+/**
+ * Gets the next presentable double-precision floating point value greater than x
+ * @param x A number. The caller must ensure that x is not NaN or Infinity
+ * @returns The next presentable double-precision floating point value
+ */
+function nextUp(x: number) {
+    if (x === 0) return Number.MIN_VALUE;
+
+    // Write the number as a float64
+    view.setFloat64(0, x, false);
+
+    let bits = view.getBigUint64(0, false);
+
+    // If x > 0, increment bits; if x < 0, decrement bits
+    bits += x > 0 ? 1n : -1n;
+
+    view.setBigUint64(0, bits, false);
+    return view.getFloat64(0, false);
+}
+
+let prevValue = 0;
+
+/**
+ * Gets current time.
+ *
+ * The returned value is guaranteed to be monotonically increasing.
+ */
+export function increasingNow() {
+    let now = performance.now();
+    if (now <= prevValue) {
+        now = nextUp(prevValue);
+    }
+    prevValue = now;
+    return now;
+}
+
+export class TimestampTransforms {
+    /**
+     * Timestamp of the last frame to be skipped by pause controller.
+     */
+    #skipFramesUntil = -1;
+
+    #addTimestamp = new TransformStream<
+        ScrcpyVideoDecoderPauseController.Output,
+        CodecTransformStream.Input
+    >({
+        transform: (packet, controller) => {
+            if (packet.type === "configuration") {
+                controller.enqueue(packet);
+                return;
+            }
+
+            // Use `timestamp` to convey `skipRendering` to later step
+            // and track total decoding time
+            const timestamp = increasingNow();
+
+            if (packet.skipRendering) {
+                this.#skipFramesUntil = timestamp;
+            }
+
+            controller.enqueue({
+                ...packet,
+                timestamp,
+            });
+        },
+    });
+    get addTimestamp() {
+        return this.#addTimestamp;
+    }
+
+    // This is not in `VideoDecoderStream` because
+    // this time includes all pre-processing time,
+    // and requires `EncodedVideoChunk.timestamp` to contain
+    // local time of when the frame is received,
+    // which is set by this class.
+    #totalDecodeTime = 0;
+    /**
+     * Gets the total time spent processing and decoding frames in milliseconds.
+     */
+    get totalDecodeTime() {
+        return this.#totalDecodeTime;
+    }
+
+    #consumeTimestamp = new TransformStream<VideoFrame, VideoFrame>({
+        transform: (frame, controller) => {
+            // `frame.timestamp` is the same `EncodedVideoChunk.timestamp` set above
+            this.#totalDecodeTime += performance.now() - frame.timestamp;
+
+            // Don't count these frames as skipped rendering
+            if (frame.timestamp <= this.#skipFramesUntil) {
+                frame.close();
+                return;
+            }
+
+            controller.enqueue(frame);
+        },
+    });
+    get consumeTimestamp() {
+        return this.#consumeTimestamp;
+    }
+}