@yume-chan/scrcpy-decoder-webcodecs 0.0.24 → 1.1.0

package/src/video/decoder.ts

@@ -10,11 +10,94 @@ import { WritableStream } from "@yume-chan/stream-extra";
 
 import { Av1Codec, H264Decoder, H265Decoder } from "./codec/index.js";
 import type { CodecDecoder } from "./codec/type.js";
-import type { FrameRenderer } from "./render/index.js";
-import { BitmapFrameRenderer, WebGLFrameRenderer } from "./render/index.js";
+import type { VideoFrameRenderer } from "./render/index.js";
+
+class Pool<T> {
+    #controller!: ReadableStreamDefaultController<T>;
+    #readable = new ReadableStream<T>(
+        {
+            start: (controller) => {
+                this.#controller = controller;
+            },
+            pull: (controller) => {
+                controller.enqueue(this.#initializer());
+            },
+        },
+        { highWaterMark: 0 },
+    );
+    #reader = this.#readable.getReader();
+
+    #initializer: () => T;
+
+    #size = 0;
+    #capacity: number;
+
+    constructor(initializer: () => T, capacity: number) {
+        this.#initializer = initializer;
+        this.#capacity = capacity;
+    }
+
+    async borrow() {
+        const result = await this.#reader.read();
+        return result.value!;
+    }
+
+    return(value: T) {
+        if (this.#size < this.#capacity) {
+            this.#controller.enqueue(value);
+            this.#size += 1;
+        }
+    }
+}
+
+class VideoFrameCapturer {
+    #canvas: OffscreenCanvas | HTMLCanvasElement;
+    #context: ImageBitmapRenderingContext;
+
+    constructor() {
+        if (typeof OffscreenCanvas !== "undefined") {
+            this.#canvas = new OffscreenCanvas(1, 1);
+        } else {
+            this.#canvas = document.createElement("canvas");
+            this.#canvas.width = 1;
+            this.#canvas.height = 1;
+        }
+        this.#context = this.#canvas.getContext("bitmaprenderer", {
+            alpha: false,
+        })!;
+    }
+
+    async capture(frame: VideoFrame): Promise<Blob> {
+        this.#canvas.width = frame.displayWidth;
+        this.#canvas.height = frame.displayHeight;
+
+        const bitmap = await createImageBitmap(frame);
+        this.#context.transferFromImageBitmap(bitmap);
+
+        if (this.#canvas instanceof OffscreenCanvas) {
+            return await this.#canvas.convertToBlob({
+                type: "image/png",
+            });
+        } else {
+            return new Promise((resolve, reject) => {
+                (this.#canvas as HTMLCanvasElement).toBlob((blob) => {
+                    if (!blob) {
+                        reject(new Error("Failed to convert canvas to blob"));
+                    } else {
+                        resolve(blob);
+                    }
+                }, "image/png");
+            });
+        }
+    }
+}
+
+const VideoFrameCapturerPool =
+    /* #__PURE__ */
+    new Pool(() => new VideoFrameCapturer(), 4);
 
 export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
-    static isSupported() {
+    static get isSupported() {
         return typeof globalThis.VideoDecoder !== "undefined";
     }
 
@@ -22,6 +105,7 @@ export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
         {
             h264: {},
             h265: {},
+            av1: {},
         };
 
     #codec: ScrcpyVideoCodecId;
@@ -36,19 +120,23 @@ export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
         return this.#writable;
     }
 
-    #canvas: HTMLCanvasElement;
+    #error: Error | undefined;
+    #controller!: WritableStreamDefaultController;
+
+    #renderer: VideoFrameRenderer;
     get renderer() {
-        return this.#canvas;
+        return this.#renderer;
     }
 
-    #frameRendered = 0;
+    #framesDraw = 0;
+    #framesPresented = 0;
     get framesRendered() {
-        return this.#frameRendered;
+        return this.#framesPresented;
     }
 
-    #frameSkipped = 0;
+    #framesSkipped = 0;
     get framesSkipped() {
-        return this.#frameSkipped;
+        return this.#framesSkipped;
     }
 
     #sizeChanged = new EventEmitter<{ width: number; height: number }>();
@@ -57,61 +145,40 @@ export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
     }
 
     #decoder: VideoDecoder;
-    #renderer: FrameRenderer;
 
-    #currentFrameRendered = false;
+    #drawing = false;
+    #nextFrame: VideoFrame | undefined;
+    #captureFrame: VideoFrame | undefined;
+
     #animationFrameId = 0;
 
     /**
      * Create a new WebCodecs video decoder.
-     * @param codec The video codec to decode
-     * @param enableCapture
-     * Whether to allow capturing the canvas content using APIs like `readPixels` and `toDataURL`.
-     * Enable this option may reduce performance.
      */
-    constructor(codec: ScrcpyVideoCodecId, enableCapture: boolean) {
+    constructor({ codec, renderer }: WebCodecsVideoDecoder.Options) {
         this.#codec = codec;
 
-        this.#canvas = document.createElement("canvas");
-
-        try {
-            this.#renderer = new WebGLFrameRenderer(
-                this.#canvas,
-                enableCapture,
-            );
-        } catch {
-            this.#renderer = new BitmapFrameRenderer(this.#canvas);
-        }
+        this.#renderer = renderer;
 
         this.#decoder = new VideoDecoder({
             output: (frame) => {
-                if (this.#currentFrameRendered) {
-                    this.#frameRendered += 1;
-                } else {
-                    this.#frameSkipped += 1;
-                }
-                this.#currentFrameRendered = false;
-
-                // PERF: Draw every frame to minimize latency at cost of performance.
-                // When multiple frames are drawn in one vertical sync interval,
-                // only the last one is visible to users.
-                // But this ensures users can always see the most up-to-date screen.
-                // This is also the behavior of official Scrcpy client.
-                // https://github.com/Genymobile/scrcpy/issues/3679
-                this.#updateSize(frame.displayWidth, frame.displayHeight);
-                this.#renderer.draw(frame);
-            },
-            error(e) {
-                if (controller) {
-                    try {
-                        controller.error(e);
-                    } catch {
-                        // ignore
-                        // `controller` may already in error state
+                this.#captureFrame?.close();
+                // PERF: `VideoFrame#clone` is cheap
+                this.#captureFrame = frame.clone();
+
+                if (this.#drawing) {
+                    if (this.#nextFrame) {
+                        this.#nextFrame.close();
+                        this.#framesSkipped += 1;
                     }
-                } else {
-                    error = e;
+                    this.#nextFrame = frame;
+                    return;
                 }
+
+                void this.#draw(frame);
+            },
+            error: (error) => {
+                this.#setError(error);
             },
         });
 
@@ -134,16 +201,16 @@ export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
                     this.#updateSize,
                 );
                 break;
+            default:
+                throw new Error(`Unsupported codec: ${this.#codec as number}`);
         }
 
-        let error: Error | undefined;
-        let controller: WritableStreamDefaultController | undefined;
         this.#writable = new WritableStream<ScrcpyMediaStreamPacket>({
-            start: (_controller) => {
-                if (error) {
-                    _controller.error(error);
+            start: (controller) => {
+                if (this.#error) {
+                    controller.error(this.#error);
                 } else {
-                    controller = _controller;
+                    this.#controller = controller;
                 }
             },
             write: (packet) => {
@@ -151,29 +218,90 @@ export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
             },
         });
 
-        this.#onFramePresented();
+        this.#onVerticalSync();
     }
 
-    #updateSize = (width: number, height: number) => {
-        if (width !== this.#canvas.width || height !== this.#canvas.height) {
-            this.#canvas.width = width;
-            this.#canvas.height = height;
-            this.#sizeChanged.fire({
-                width: width,
-                height: height,
-            });
+    #setError(error: Error) {
+        if (this.#controller) {
+            try {
+                this.#controller.error(error);
+            } catch {
+                // ignore
+            }
+        } else {
+            this.#error = error;
+        }
+    }
+
+    async #draw(frame: VideoFrame) {
+        try {
+            this.#drawing = true;
+            // PERF: Draw every frame to minimize latency at cost of performance.
+            // When multiple frames are drawn in one vertical sync interval,
+            // only the last one is visible to users.
+            // But this ensures users can always see the most up-to-date screen.
+            // This is also the behavior of official Scrcpy client.
+            // https://github.com/Genymobile/scrcpy/issues/3679
+            this.#updateSize(frame.displayWidth, frame.displayHeight);
+            await this.#renderer.draw(frame);
+            this.#framesDraw += 1;
+            frame.close();
+
+            if (this.#nextFrame) {
+                const frame = this.#nextFrame;
+                this.#nextFrame = undefined;
+                await this.#draw(frame);
+            }
+
+            this.#drawing = false;
+        } catch (error) {
+            this.#setError(error as Error);
         }
+    }
+
+    #updateSize = (width: number, height: number) => {
+        this.#renderer.setSize(width, height);
+        this.#sizeChanged.fire({ width, height });
     };
 
-    #onFramePresented = () => {
-        this.#currentFrameRendered = true;
-        this.#animationFrameId = requestAnimationFrame(this.#onFramePresented);
+    #onVerticalSync = () => {
+        if (this.#framesDraw > 0) {
+            this.#framesPresented += 1;
+            this.#framesSkipped += this.#framesDraw - 1;
+            this.#framesDraw = 0;
+        }
+        this.#animationFrameId = requestAnimationFrame(this.#onVerticalSync);
     };
 
+    async snapshot() {
+        const frame = this.#captureFrame;
+        if (!frame) {
+            return undefined;
+        }
+
+        const capturer = await VideoFrameCapturerPool.borrow();
+        const result = await capturer.capture(frame);
+        VideoFrameCapturerPool.return(capturer);
+        return result;
+    }
+
     dispose() {
         cancelAnimationFrame(this.#animationFrameId);
         if (this.#decoder.state !== "closed") {
             this.#decoder.close();
         }
+        this.#nextFrame?.close();
+        this.#captureFrame?.close();
+    }
+}
+
+export namespace WebCodecsVideoDecoder {
+    export interface Options {
+        /**
+         * The video codec to decode
+         */
+        codec: ScrcpyVideoCodecId;
+
+        renderer: VideoFrameRenderer;
     }
 }

package/src/video/render/bitmap.ts

@@ -1,24 +1,18 @@
-import type { FrameRenderer } from "./type.js";
+import { CanvasVideoFrameRenderer } from "./canvas.js";
 
-export class BitmapFrameRenderer implements FrameRenderer {
+export class BitmapVideoFrameRenderer extends CanvasVideoFrameRenderer {
     #context: ImageBitmapRenderingContext;
 
-    constructor(canvas: HTMLCanvasElement) {
-        this.#context = canvas.getContext("bitmaprenderer", { alpha: false })!;
+    constructor(canvas?: HTMLCanvasElement | OffscreenCanvas) {
+        super(canvas);
+
+        this.#context = this.canvas.getContext("bitmaprenderer", {
+            alpha: false,
+        })!;
     }
 
-    draw(frame: VideoFrame): void {
-        createImageBitmap(frame)
-            .then((bitmap) => {
-                this.#context.transferFromImageBitmap(bitmap);
-                frame.close();
-            })
-            .catch((e) => {
-                console.warn(
-                    "[@yume-chan/scrcpy-decoder-webcodecs]",
-                    "VideoDecoder error",
-                    e,
-                );
-            });
+    async draw(frame: VideoFrame): Promise<void> {
+        const bitmap = await createImageBitmap(frame);
+        this.#context.transferFromImageBitmap(bitmap);
     }
 }

package/src/video/render/canvas.ts

@@ -0,0 +1,27 @@
+import { createCanvas } from "@yume-chan/scrcpy-decoder-tinyh264";
+
+import type { VideoFrameRenderer } from "./type.js";
+
+export abstract class CanvasVideoFrameRenderer implements VideoFrameRenderer {
+    #canvas: HTMLCanvasElement | OffscreenCanvas;
+    get canvas() {
+        return this.#canvas;
+    }
+
+    constructor(canvas?: HTMLCanvasElement | OffscreenCanvas) {
+        if (canvas) {
+            this.#canvas = canvas;
+        } else {
+            this.#canvas = createCanvas();
+        }
+    }
+
+    setSize(width: number, height: number): void {
+        if (this.#canvas.width !== width || this.#canvas.height !== height) {
+            this.#canvas.width = width;
+            this.#canvas.height = height;
+        }
+    }
+
+    abstract draw(frame: VideoFrame): Promise<void>;
+}

package/src/video/render/index.ts

@@ -1,3 +1,7 @@
+// cspell: ignore insertable
+
 export * from "./bitmap.js";
+export * from "./canvas.js";
+export * from "./insertable-stream.js";
 export * from "./type.js";
 export * from "./webgl.js";

package/src/video/render/insertable-stream.ts

@@ -0,0 +1,60 @@
+// cspell: ignore insertable
+
+import type { VideoFrameRenderer } from "./type.js";
+
+declare class MediaStreamTrackGenerator extends MediaStreamTrack {
+    constructor(options: { kind: "audio" | "video" });
+
+    writable: WritableStream<VideoFrame>;
+}
+
+export class InsertableStreamVideoFrameRenderer implements VideoFrameRenderer {
+    static get isSupported() {
+        return typeof MediaStreamTrackGenerator !== "undefined";
+    }
+
+    #element: HTMLVideoElement;
+    get element() {
+        return this.#element;
+    }
+
+    #generator: MediaStreamTrackGenerator;
+    #writer: WritableStreamDefaultWriter<VideoFrame>;
+    #stream: MediaStream;
+
+    constructor(element?: HTMLVideoElement) {
+        if (element) {
+            this.#element = element;
+        } else if (typeof document !== "undefined") {
+            this.#element = document.createElement("video");
+        } else {
+            throw new Error(
+                "no video element input found nor any video element can be created",
+            );
+        }
+        this.#element.muted = true;
+        this.#element.autoplay = true;
+        this.#element.disablePictureInPicture = true;
+        this.#element.disableRemotePlayback = true;
+
+        // The spec replaced `MediaStreamTrackGenerator` with `VideoTrackGenerator`.
+        // But Chrome has not implemented it yet.
+        // https://issues.chromium.org/issues/40058895
+        this.#generator = new MediaStreamTrackGenerator({ kind: "video" });
+        this.#writer = this.#generator.writable.getWriter();
+
+        this.#stream = new MediaStream([this.#generator]);
+        this.#element.srcObject = this.#stream;
+    }
+
+    setSize(width: number, height: number): void {
+        if (this.#element.width !== width || this.#element.height !== height) {
+            this.#element.width = width;
+            this.#element.height = height;
+        }
+    }
+
+    async draw(frame: VideoFrame): Promise<void> {
+        await this.#writer.write(frame);
+    }
+}

package/src/video/render/type.ts

@@ -1,3 +1,7 @@
-export interface FrameRenderer {
-    draw(frame: VideoFrame): void;
+import type { MaybePromiseLike } from "@yume-chan/async";
+
+export interface VideoFrameRenderer {
+    setSize(width: number, height: number): void;
+
+    draw(frame: VideoFrame): MaybePromiseLike<void>;
 }

package/src/video/render/webgl.ts

@@ -1,6 +1,30 @@
-import type { FrameRenderer } from "./type.js";
+import { createCanvas } from "@yume-chan/scrcpy-decoder-tinyh264";
+
+import { CanvasVideoFrameRenderer } from "./canvas.js";
+
+const Resolved = Promise.resolve();
+
+function createContext(
+    canvas: HTMLCanvasElement | OffscreenCanvas,
+    enableCapture?: boolean,
+): WebGLRenderingContext | null {
+    const attributes: WebGLContextAttributes = {
+        // Low-power GPU should be enough for video rendering.
+        powerPreference: "low-power",
+        alpha: false,
+        // Disallow software rendering.
+        // Other rendering methods are faster than software-based WebGL.
+        failIfMajorPerformanceCaveat: true,
+        preserveDrawingBuffer: !!enableCapture,
+    };
+
+    return (
+        canvas.getContext("webgl2", attributes) ||
+        canvas.getContext("webgl", attributes)
+    );
+}
 
-export class WebGLFrameRenderer implements FrameRenderer {
+export class WebGLVideoFrameRenderer extends CanvasVideoFrameRenderer {
     static vertexShaderSource = `
         attribute vec2 xy;
 
@@ -24,6 +48,11 @@ export class WebGLFrameRenderer implements FrameRenderer {
         }
 `;
 
+    static get isSupported() {
+        const canvas = createCanvas();
+        return !!createContext(canvas);
+    }
+
     #context: WebGLRenderingContext;
 
     /**
@@ -33,25 +62,23 @@ export class WebGLFrameRenderer implements FrameRenderer {
      * Whether to allow capturing the canvas content using APIs like `readPixels` and `toDataURL`.
      * Enable this option may reduce performance.
      */
-    constructor(canvas: HTMLCanvasElement, enableCapture: boolean) {
-        const gl =
-            canvas.getContext("webgl2", {
-                alpha: false,
-                failIfMajorPerformanceCaveat: true,
-                preserveDrawingBuffer: enableCapture,
-            }) ||
-            canvas.getContext("webgl", {
-                alpha: false,
-                failIfMajorPerformanceCaveat: true,
-                preserveDrawingBuffer: enableCapture,
-            });
+    constructor(
+        canvas?: HTMLCanvasElement | OffscreenCanvas,
+        enableCapture?: boolean,
+    ) {
+        super(canvas);
+
+        const gl = createContext(this.canvas, enableCapture);
         if (!gl) {
             throw new Error("WebGL not supported");
         }
         this.#context = gl;
 
         const vertexShader = gl.createShader(gl.VERTEX_SHADER)!;
-        gl.shaderSource(vertexShader, WebGLFrameRenderer.vertexShaderSource);
+        gl.shaderSource(
+            vertexShader,
+            WebGLVideoFrameRenderer.vertexShaderSource,
+        );
         gl.compileShader(vertexShader);
         if (!gl.getShaderParameter(vertexShader, gl.COMPILE_STATUS)) {
             throw new Error(gl.getShaderInfoLog(vertexShader)!);
@@ -60,14 +87,14 @@ export class WebGLFrameRenderer implements FrameRenderer {
         const fragmentShader = gl.createShader(gl.FRAGMENT_SHADER)!;
         gl.shaderSource(
             fragmentShader,
-            WebGLFrameRenderer.fragmentShaderSource,
+            WebGLVideoFrameRenderer.fragmentShaderSource,
         );
         gl.compileShader(fragmentShader);
         if (!gl.getShaderParameter(fragmentShader, gl.COMPILE_STATUS)) {
             throw new Error(gl.getShaderInfoLog(fragmentShader)!);
         }
 
-        const shaderProgram = gl.createProgram()!;
+        const shaderProgram = gl.createProgram();
         gl.attachShader(shaderProgram, vertexShader);
         gl.attachShader(shaderProgram, fragmentShader);
         gl.linkProgram(shaderProgram);
@@ -98,7 +125,7 @@ export class WebGLFrameRenderer implements FrameRenderer {
         gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
     }
 
-    draw(frame: VideoFrame) {
+    draw(frame: VideoFrame): Promise<void> {
         const gl = this.#context;
         gl.texImage2D(
             gl.TEXTURE_2D,
@@ -108,9 +135,10 @@ export class WebGLFrameRenderer implements FrameRenderer {
             gl.UNSIGNED_BYTE,
             frame,
         );
-        frame.close();
 
         gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight);
         gl.drawArrays(gl.TRIANGLE_FAN, 0, 4);
+
+        return Resolved;
     }
 }

package/tsconfig.build.json

@@ -9,11 +9,14 @@
     },
     "references": [
         {
-            "path": "../scrcpy/tsconfig.build.json"
+            "path": "../no-data-view/tsconfig.build.json"
         },
         {
             "path": "../scrcpy-decoder-tinyh264/tsconfig.build.json"
         },
+        {
+            "path": "../scrcpy/tsconfig.build.json"
+        },
         {
             "path": "../stream-extra/tsconfig.build.json"
         }