@yume-chan/scrcpy-decoder-webcodecs 0.0.23 → 1.0.0

package/src/video/decoder.ts

@@ -0,0 +1,307 @@
+import { EventEmitter } from "@yume-chan/event";
+import type { ScrcpyMediaStreamPacket } from "@yume-chan/scrcpy";
+import { ScrcpyVideoCodecId } from "@yume-chan/scrcpy";
+import type {
+    ScrcpyVideoDecoder,
+    ScrcpyVideoDecoderCapability,
+} from "@yume-chan/scrcpy-decoder-tinyh264";
+import type { WritableStreamDefaultController } from "@yume-chan/stream-extra";
+import { WritableStream } from "@yume-chan/stream-extra";
+
+import { Av1Codec, H264Decoder, H265Decoder } from "./codec/index.js";
+import type { CodecDecoder } from "./codec/type.js";
+import type { WebCodecsVideoDecoderRenderer } from "./render/index.js";
+
+class Pool<T> {
+    #controller!: ReadableStreamDefaultController<T>;
+    #readable = new ReadableStream<T>(
+        {
+            start: (controller) => {
+                this.#controller = controller;
+            },
+            pull: (controller) => {
+                controller.enqueue(this.#initializer());
+            },
+        },
+        { highWaterMark: 0 },
+    );
+    #reader = this.#readable.getReader();
+
+    #initializer: () => T;
+
+    #size = 0;
+    #capacity: number;
+
+    constructor(initializer: () => T, capacity: number) {
+        this.#initializer = initializer;
+        this.#capacity = capacity;
+    }
+
+    async borrow() {
+        const result = await this.#reader.read();
+        return result.value!;
+    }
+
+    return(value: T) {
+        if (this.#size < this.#capacity) {
+            this.#controller.enqueue(value);
+            this.#size += 1;
+        }
+    }
+}
+
+class VideoFrameCapturer {
+    #canvas: OffscreenCanvas | HTMLCanvasElement;
+    #context: ImageBitmapRenderingContext;
+
+    constructor() {
+        if (typeof OffscreenCanvas !== "undefined") {
+            this.#canvas = new OffscreenCanvas(1, 1);
+        } else {
+            this.#canvas = document.createElement("canvas");
+            this.#canvas.width = 1;
+            this.#canvas.height = 1;
+        }
+        this.#context = this.#canvas.getContext("bitmaprenderer", {
+            alpha: false,
+        })!;
+    }
+
+    async capture(frame: VideoFrame): Promise<Blob> {
+        this.#canvas.width = frame.displayWidth;
+        this.#canvas.height = frame.displayHeight;
+
+        const bitmap = await createImageBitmap(frame);
+        this.#context.transferFromImageBitmap(bitmap);
+
+        if (this.#canvas instanceof OffscreenCanvas) {
+            return await this.#canvas.convertToBlob({
+                type: "image/png",
+            });
+        } else {
+            return new Promise((resolve, reject) => {
+                (this.#canvas as HTMLCanvasElement).toBlob((blob) => {
+                    if (!blob) {
+                        reject(new Error("Failed to convert canvas to blob"));
+                    } else {
+                        resolve(blob);
+                    }
+                }, "image/png");
+            });
+        }
+    }
+}
+
+const VideoFrameCapturerPool =
+    /* #__PURE__ */
+    new Pool(() => new VideoFrameCapturer(), 4);
+
+export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
+    static get isSupported() {
+        return typeof globalThis.VideoDecoder !== "undefined";
+    }
+
+    static readonly capabilities: Record<string, ScrcpyVideoDecoderCapability> =
+        {
+            h264: {},
+            h265: {},
+            av1: {},
+        };
+
+    #codec: ScrcpyVideoCodecId;
+    get codec() {
+        return this.#codec;
+    }
+
+    #codecDecoder: CodecDecoder;
+
+    #writable: WritableStream<ScrcpyMediaStreamPacket>;
+    get writable() {
+        return this.#writable;
+    }
+
+    #error: Error | undefined;
+    #controller!: WritableStreamDefaultController;
+
+    #renderer: WebCodecsVideoDecoderRenderer;
+    get renderer() {
+        return this.#renderer;
+    }
+
+    #framesDraw = 0;
+    #framesPresented = 0;
+    get framesRendered() {
+        return this.#framesPresented;
+    }
+
+    #framesSkipped = 0;
+    get framesSkipped() {
+        return this.#framesSkipped;
+    }
+
+    #sizeChanged = new EventEmitter<{ width: number; height: number }>();
+    get sizeChanged() {
+        return this.#sizeChanged.event;
+    }
+
+    #decoder: VideoDecoder;
+
+    #drawing = false;
+    #nextFrame: VideoFrame | undefined;
+    #captureFrame: VideoFrame | undefined;
+
+    #animationFrameId = 0;
+
+    /**
+     * Create a new WebCodecs video decoder.
+     */
+    constructor({ codec, renderer }: WebCodecsVideoDecoder.Options) {
+        this.#codec = codec;
+
+        this.#renderer = renderer;
+
+        this.#decoder = new VideoDecoder({
+            output: (frame) => {
+                this.#captureFrame?.close();
+                // PERF: `VideoFrame#clone` is cheap
+                this.#captureFrame = frame.clone();
+
+                if (this.#drawing) {
+                    if (this.#nextFrame) {
+                        this.#nextFrame.close();
+                        this.#framesSkipped += 1;
+                    }
+                    this.#nextFrame = frame;
+                    return;
+                }
+
+                void this.#draw(frame);
+            },
+            error: (error) => {
+                this.#setError(error);
+            },
+        });
+
+        switch (this.#codec) {
+            case ScrcpyVideoCodecId.H264:
+                this.#codecDecoder = new H264Decoder(
+                    this.#decoder,
+                    this.#updateSize,
+                );
+                break;
+            case ScrcpyVideoCodecId.H265:
+                this.#codecDecoder = new H265Decoder(
+                    this.#decoder,
+                    this.#updateSize,
+                );
+                break;
+            case ScrcpyVideoCodecId.AV1:
+                this.#codecDecoder = new Av1Codec(
+                    this.#decoder,
+                    this.#updateSize,
+                );
+                break;
+            default:
+                throw new Error(`Unsupported codec: ${this.#codec as number}`);
+        }
+
+        this.#writable = new WritableStream<ScrcpyMediaStreamPacket>({
+            start: (controller) => {
+                if (this.#error) {
+                    controller.error(this.#error);
+                } else {
+                    this.#controller = controller;
+                }
+            },
+            write: (packet) => {
+                this.#codecDecoder.decode(packet);
+            },
+        });
+
+        this.#onVerticalSync();
+    }
+
+    #setError(error: Error) {
+        if (this.#controller) {
+            try {
+                this.#controller.error(error);
+            } catch {
+                // ignore
+            }
+        } else {
+            this.#error = error;
+        }
+    }
+
+    async #draw(frame: VideoFrame) {
+        try {
+            this.#drawing = true;
+            // PERF: Draw every frame to minimize latency at cost of performance.
+            // When multiple frames are drawn in one vertical sync interval,
+            // only the last one is visible to users.
+            // But this ensures users can always see the most up-to-date screen.
+            // This is also the behavior of official Scrcpy client.
+            // https://github.com/Genymobile/scrcpy/issues/3679
+            this.#updateSize(frame.displayWidth, frame.displayHeight);
+            await this.#renderer.draw(frame);
+            this.#framesDraw += 1;
+            frame.close();
+
+            if (this.#nextFrame) {
+                const frame = this.#nextFrame;
+                this.#nextFrame = undefined;
+                await this.#draw(frame);
+            }
+
+            this.#drawing = false;
+        } catch (error) {
+            this.#setError(error as Error);
+        }
+    }
+
+    #updateSize = (width: number, height: number) => {
+        this.#renderer.setSize(width, height);
+        this.#sizeChanged.fire({ width, height });
+    };
+
+    #onVerticalSync = () => {
+        if (this.#framesDraw > 0) {
+            this.#framesPresented += 1;
+            this.#framesSkipped += this.#framesDraw - 1;
+            this.#framesDraw = 0;
+        }
+        this.#animationFrameId = requestAnimationFrame(this.#onVerticalSync);
+    };
+
+    async snapshot() {
+        const frame = this.#captureFrame;
+        if (!frame) {
+            return undefined;
+        }
+
+        const capturer = await VideoFrameCapturerPool.borrow();
+        const result = await capturer.capture(frame);
+        VideoFrameCapturerPool.return(capturer);
+        return result;
+    }
+
+    dispose() {
+        cancelAnimationFrame(this.#animationFrameId);
+        if (this.#decoder.state !== "closed") {
+            this.#decoder.close();
+        }
+        this.#nextFrame?.close();
+        this.#captureFrame?.close();
+    }
+}
+
+export namespace WebCodecsVideoDecoder {
+    export interface Options {
+        /**
+         * The video codec to decode
+         */
+        codec: ScrcpyVideoCodecId;
+
+        renderer: WebCodecsVideoDecoderRenderer;
+    }
+}

package/src/video/index.ts

@@ -0,0 +1,3 @@
+export * from "./codec/index.js";
+export * from "./decoder.js";
+export * from "./render/index.js";

package/src/video/render/bitmap.ts

@@ -0,0 +1,18 @@
+import { CanvasWebCodecsDecoderRenderer } from "./canvas.js";
+
+export class BitmapWebCodecsDecoderRenderer extends CanvasWebCodecsDecoderRenderer {
+    #context: ImageBitmapRenderingContext;
+
+    constructor(canvas?: HTMLCanvasElement | OffscreenCanvas) {
+        super(canvas);
+
+        this.#context = this.canvas.getContext("bitmaprenderer", {
+            alpha: false,
+        })!;
+    }
+
+    async draw(frame: VideoFrame): Promise<void> {
+        const bitmap = await createImageBitmap(frame);
+        this.#context.transferFromImageBitmap(bitmap);
+    }
+}

package/src/video/render/canvas.ts

@@ -0,0 +1,29 @@
+import { createCanvas } from "@yume-chan/scrcpy-decoder-tinyh264";
+
+import type { WebCodecsVideoDecoderRenderer } from "./type.js";
+
+export abstract class CanvasWebCodecsDecoderRenderer
+    implements WebCodecsVideoDecoderRenderer
+{
+    #canvas: HTMLCanvasElement | OffscreenCanvas;
+    get canvas() {
+        return this.#canvas;
+    }
+
+    constructor(canvas?: HTMLCanvasElement | OffscreenCanvas) {
+        if (canvas) {
+            this.#canvas = canvas;
+        } else {
+            this.#canvas = createCanvas();
+        }
+    }
+
+    setSize(width: number, height: number): void {
+        if (this.#canvas.width !== width || this.#canvas.height !== height) {
+            this.#canvas.width = width;
+            this.#canvas.height = height;
+        }
+    }
+
+    abstract draw(frame: VideoFrame): Promise<void>;
+}

package/src/video/render/index.ts

@@ -0,0 +1,7 @@
+// cspell: ignore insertable
+
+export * from "./bitmap.js";
+export * from "./canvas.js";
+export * from "./insertable-stream.js";
+export * from "./type.js";
+export * from "./webgl.js";

package/src/video/render/insertable-stream.ts

@@ -0,0 +1,62 @@
+// cspell: ignore insertable
+
+import type { WebCodecsVideoDecoderRenderer } from "./type.js";
+
+declare class MediaStreamTrackGenerator extends MediaStreamTrack {
+    constructor(options: { kind: "audio" | "video" });
+
+    writable: WritableStream<VideoFrame>;
+}
+
+export class InsertableStreamWebCodecsDecoderRenderer
+    implements WebCodecsVideoDecoderRenderer
+{
+    static get isSupported() {
+        return typeof MediaStreamTrackGenerator !== "undefined";
+    }
+
+    #element: HTMLVideoElement;
+    get element() {
+        return this.#element;
+    }
+
+    #generator: MediaStreamTrackGenerator;
+    #writer: WritableStreamDefaultWriter<VideoFrame>;
+    #stream: MediaStream;
+
+    constructor(element?: HTMLVideoElement) {
+        if (element) {
+            this.#element = element;
+        } else if (typeof document !== "undefined") {
+            this.#element = document.createElement("video");
+        } else {
+            throw new Error(
+                "no video element input found nor any video element can be created",
+            );
+        }
+        this.#element.muted = true;
+        this.#element.autoplay = true;
+        this.#element.disablePictureInPicture = true;
+        this.#element.disableRemotePlayback = true;
+
+        // The spec replaced `MediaStreamTrackGenerator` with `VideoTrackGenerator`.
+        // But Chrome has not implemented it yet.
+        // https://issues.chromium.org/issues/40058895
+        this.#generator = new MediaStreamTrackGenerator({ kind: "video" });
+        this.#writer = this.#generator.writable.getWriter();
+
+        this.#stream = new MediaStream([this.#generator]);
+        this.#element.srcObject = this.#stream;
+    }
+
+    setSize(width: number, height: number): void {
+        if (this.#element.width !== width || this.#element.height !== height) {
+            this.#element.width = width;
+            this.#element.height = height;
+        }
+    }
+
+    async draw(frame: VideoFrame): Promise<void> {
+        await this.#writer.write(frame);
+    }
+}

package/src/video/render/type.ts

@@ -0,0 +1,7 @@
+import type { MaybePromiseLike } from "@yume-chan/async";
+
+export interface WebCodecsVideoDecoderRenderer {
+    setSize(width: number, height: number): void;
+
+    draw(frame: VideoFrame): MaybePromiseLike<void>;
+}

package/src/{webgl.ts → video/render/webgl.ts}

@@ -1,48 +1,84 @@
-import type { FrameRenderer } from "./renderer.js";
+import { createCanvas } from "@yume-chan/scrcpy-decoder-tinyh264";
+
+import { CanvasWebCodecsDecoderRenderer } from "./canvas.js";
+
+const Resolved = Promise.resolve();
+
+function createContext(
+    canvas: HTMLCanvasElement | OffscreenCanvas,
+    enableCapture?: boolean,
+): WebGLRenderingContext | null {
+    const attributes: WebGLContextAttributes = {
+        // Low-power GPU should be enough for video rendering.
+        powerPreference: "low-power",
+        alpha: false,
+        // Disallow software rendering.
+        // Other rendering methods are faster than software-based WebGL.
+        failIfMajorPerformanceCaveat: true,
+        preserveDrawingBuffer: !!enableCapture,
+    };
+
+    return (
+        canvas.getContext("webgl2", attributes) ||
+        canvas.getContext("webgl", attributes)
+    );
+}
 
-export class WebGLFrameRenderer implements FrameRenderer {
+export class WebGLWebCodecsDecoderRenderer extends CanvasWebCodecsDecoderRenderer {
     static vertexShaderSource = `
-attribute vec2 xy;
+        attribute vec2 xy;
 
-varying highp vec2 uv;
+        varying highp vec2 uv;
 
-void main(void) {
-  gl_Position = vec4(xy, 0.0, 1.0);
-  // Map vertex coordinates (-1 to +1) to UV coordinates (0 to 1).
-  // UV coordinates are Y-flipped relative to vertex coordinates.
-  uv = vec2((1.0 + xy.x) / 2.0, (1.0 - xy.y) / 2.0);
-}
+        void main(void) {
+            gl_Position = vec4(xy, 0.0, 1.0);
+            // Map vertex coordinates (-1 to +1) to UV coordinates (0 to 1).
+            // UV coordinates are Y-flipped relative to vertex coordinates.
+            uv = vec2((1.0 + xy.x) / 2.0, (1.0 - xy.y) / 2.0);
+        }
 `;
 
     static fragmentShaderSource = `
-varying highp vec2 uv;
+        varying highp vec2 uv;
 
-uniform sampler2D texture;
+        uniform sampler2D texture;
 
-void main(void) {
-  gl_FragColor = texture2D(texture, uv);
-}
+        void main(void) {
+            gl_FragColor = texture2D(texture, uv);
+        }
 `;
 
+    static get isSupported() {
+        const canvas = createCanvas();
+        return !!createContext(canvas);
+    }
+
     #context: WebGLRenderingContext;
 
-    constructor(canvas: HTMLCanvasElement) {
-        const gl =
-            canvas.getContext("webgl2", {
-                alpha: false,
-                failIfMajorPerformanceCaveat: true,
-            }) ||
-            canvas.getContext("webgl", {
-                alpha: false,
-                failIfMajorPerformanceCaveat: true,
-            });
+    /**
+     * Create a new WebGL frame renderer.
+     * @param canvas The canvas to render frames to.
+     * @param enableCapture
+     * Whether to allow capturing the canvas content using APIs like `readPixels` and `toDataURL`.
+     * Enable this option may reduce performance.
+     */
+    constructor(
+        canvas?: HTMLCanvasElement | OffscreenCanvas,
+        enableCapture?: boolean,
+    ) {
+        super(canvas);
+
+        const gl = createContext(this.canvas, enableCapture);
         if (!gl) {
             throw new Error("WebGL not supported");
         }
         this.#context = gl;
 
         const vertexShader = gl.createShader(gl.VERTEX_SHADER)!;
-        gl.shaderSource(vertexShader, WebGLFrameRenderer.vertexShaderSource);
+        gl.shaderSource(
+            vertexShader,
+            WebGLWebCodecsDecoderRenderer.vertexShaderSource,
+        );
         gl.compileShader(vertexShader);
         if (!gl.getShaderParameter(vertexShader, gl.COMPILE_STATUS)) {
             throw new Error(gl.getShaderInfoLog(vertexShader)!);
@@ -51,14 +87,14 @@ void main(void) {
         const fragmentShader = gl.createShader(gl.FRAGMENT_SHADER)!;
         gl.shaderSource(
             fragmentShader,
-            WebGLFrameRenderer.fragmentShaderSource,
+            WebGLWebCodecsDecoderRenderer.fragmentShaderSource,
         );
         gl.compileShader(fragmentShader);
         if (!gl.getShaderParameter(fragmentShader, gl.COMPILE_STATUS)) {
             throw new Error(gl.getShaderInfoLog(fragmentShader)!);
         }
 
-        const shaderProgram = gl.createProgram()!;
+        const shaderProgram = gl.createProgram();
         gl.attachShader(shaderProgram, vertexShader);
         gl.attachShader(shaderProgram, fragmentShader);
         gl.linkProgram(shaderProgram);
@@ -89,7 +125,7 @@ void main(void) {
         gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
     }
 
-    draw(frame: VideoFrame) {
+    draw(frame: VideoFrame): Promise<void> {
         const gl = this.#context;
         gl.texImage2D(
             gl.TEXTURE_2D,
@@ -99,9 +135,10 @@ void main(void) {
             gl.UNSIGNED_BYTE,
             frame,
         );
-        frame.close();
 
         gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight);
         gl.drawArrays(gl.TRIANGLE_FAN, 0, 4);
+
+        return Resolved;
     }
 }

package/tsconfig.build.json

@@ -9,11 +9,14 @@
     },
     "references": [
         {
-            "path": "../scrcpy/tsconfig.build.json"
+            "path": "../no-data-view/tsconfig.build.json"
         },
         {
             "path": "../scrcpy-decoder-tinyh264/tsconfig.build.json"
         },
+        {
+            "path": "../scrcpy/tsconfig.build.json"
+        },
         {
             "path": "../stream-extra/tsconfig.build.json"
         }