@yume-chan/scrcpy-decoder-webcodecs 2.5.3 → 3.0.0-beta.1

package/src/video/decoder.ts

@@ -1,22 +1,27 @@
-import { StickyEventEmitter } from "@yume-chan/event";
-import type { ScrcpyMediaStreamPacket } from "@yume-chan/scrcpy";
-import { ScrcpyVideoCodecId } from "@yume-chan/scrcpy";
+import { pipe } from "@yume-chan/event";
+import { ScrcpyVideoCodecId, ScrcpyVideoSizeImpl } from "@yume-chan/scrcpy";
 import type {
     ScrcpyVideoDecoder,
     ScrcpyVideoDecoderCapability,
-} from "@yume-chan/scrcpy-decoder-tinyh264";
-import type { WritableStreamDefaultController } from "@yume-chan/stream-extra";
-import { WritableStream } from "@yume-chan/stream-extra";
-
-import { Av1Codec, H264Decoder, H265Decoder } from "./codec/index.js";
-import type { CodecDecoder, CodecDecoderOptions } from "./codec/type.js";
-import { Pool } from "./pool.js";
+} from "@yume-chan/scrcpy-decoder-shared";
+import { ScrcpyVideoDecoderPauseController } from "@yume-chan/scrcpy-decoder-shared";
+import { InspectStream } from "@yume-chan/stream-extra";
+
+import {
+    Av1TransformStream,
+    H264TransformStream,
+    H265TransformStream,
+    Vp8TransformStream,
+    Vp9TransformStream,
+} from "./codec/index.js";
+import type { CodecTransformStream } from "./codec/type.js";
 import type { VideoFrameRenderer } from "./render/index.js";
-import { VideoFrameCapturer } from "./snapshot.js";
-
-const VideoFrameCapturerPool =
-    /* #__PURE__ */
-    new Pool(() => new VideoFrameCapturer(), 4);
+import {
+    AutoCanvasRenderer,
+    BitmapVideoFrameRenderer,
+    RendererController,
+} from "./render/index.js";
+import { TimestampTransforms, VideoDecoderStream } from "./utils/index.js";
 
 export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
     static get isSupported() {
@@ -30,6 +35,8 @@ export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
             av1: {},
         };
 
+    // #region parameters
+
     #codec: ScrcpyVideoCodecId;
     get codec() {
         return this.#codec;
@@ -39,214 +46,253 @@ export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
     get renderer() {
         return this.#renderer;
     }
+    get rendererType() {
+        return this.#renderer.type;
+    }
+    get onRendererTypeChange() {
+        return this.#renderer.onTypeChanged;
+    }
 
-    #options: CodecDecoderOptions;
+    // #endregion parameters
 
-    #codecDecoder: CodecDecoder;
+    // #region pause controller
 
-    #writable: WritableStream<ScrcpyMediaStreamPacket>;
+    #pause = new ScrcpyVideoDecoderPauseController();
+    get paused() {
+        return this.#pause.paused;
+    }
     get writable() {
-        return this.#writable;
+        return this.#pause.writable;
     }
 
-    #error: Error | undefined;
-    #controller!: WritableStreamDefaultController;
+    // #endregion pause controller
 
-    #framesDraw = 0;
-    #framesPresented = 0;
-    get framesRendered() {
-        return this.#framesPresented;
-    }
+    // #region size
 
-    #framesSkipped = 0;
-    get framesSkipped() {
-        return this.#framesSkipped;
+    #size = new ScrcpyVideoSizeImpl();
+    get width() {
+        return this.#size.width;
+    }
+    get height() {
+        return this.#size.height;
     }
-
-    #sizeChanged = new StickyEventEmitter<{ width: number; height: number }>();
     get sizeChanged() {
-        return this.#sizeChanged.event;
+        return this.#size.sizeChanged;
     }
 
-    #width: number = 0;
-    get width() {
-        return this.#width;
+    // #endregion size
+
+    // #region raw decoder
+
+    #rawDecoder = new VideoDecoderStream();
+    get type() {
+        return this.#rawDecoder.hardwareAcceleration !== "prefer-software"
+            ? "hardware"
+            : "software";
+    }
+    #onTypeChange = pipe(
+        this.#rawDecoder.onHardwareAccelerationChange,
+        (value) => (value !== "prefer-software" ? "hardware" : "software"),
+    );
+    get onTypeChange() {
+        return this.#onTypeChange;
+    }
+    /**
+     * Gets the number of frames waiting to be decoded.
+     */
+    get decodeQueueSize() {
+        return this.#rawDecoder.decodeQueueSize;
+    }
+    /**
+     * Gets an event when a frame is dequeued (either decoded or discarded).
+     */
+    get onDequeue() {
+        return this.#rawDecoder.onDequeue;
+    }
+    /**
+     * Gets the number of times the decoder has been reset to catch up new keyframes.
+     */
+    get decoderResetCount() {
+        return this.#rawDecoder.decoderResetCount;
+    }
+    /**
+     * Gets the number of frames decoded by the decoder.
+     */
+    get framesDecoded() {
+        return this.#rawDecoder.framesDecoded;
+    }
+    /**
+     * Gets the number of frames skipped by the decoder.
+     */
+    get framesSkippedDecoding() {
+        return this.#rawDecoder.framesSkippedDecoding;
     }
 
-    #height: number = 0;
-    get height() {
-        return this.#height;
+    // #endregion raw decoder
+
+    #timestampTransforms = new TimestampTransforms();
+    /**
+     * Gets the total time spent processing and decoding frames in milliseconds.
+     */
+    get totalDecodeTime() {
+        return this.#timestampTransforms.totalDecodeTime;
     }
 
-    #decoder: VideoDecoder;
+    // #region renderer
 
-    #drawing = false;
-    #nextFrame: VideoFrame | undefined;
-    #captureFrame: VideoFrame | undefined;
+    #renderController = new RendererController();
+    /**
+     * Gets the number of frames that have been drawn on the renderer.
+     */
+    get framesRendered() {
+        return this.#renderController.framesRendered;
+    }
+    /**
+     * Gets the number of frames that's visible to the user.
+     *
+     * Multiple frames might be rendered during one vertical sync interval,
+     * but only the last of them is represented to the user.
+     * This costs some performance but reduces latency by 1 frame.
+     *
+     * Might be `0` if the renderer is in a nested Web Worker on Chrome due to a Chrome bug.
+     * https://issues.chromium.org/issues/41483010
+     */
+    get framesDisplayed() {
+        return this.#renderController.framesDisplayed;
+    }
+    /**
+     * Gets the number of frames that wasn't drawn on the renderer
+     * because the renderer can't keep up
+     */
+    get framesSkippedRendering() {
+        return this.#renderController.framesSkippedRendering;
+    }
 
-    #animationFrameId = 0;
+    // #endregion renderer
 
     /**
      * Create a new WebCodecs video decoder.
      */
     constructor({
         codec,
-        renderer,
-        ...options
+        renderer = new AutoCanvasRenderer(),
+        hardwareAcceleration = "no-preference",
+        optimizeForLatency = true,
     }: WebCodecsVideoDecoder.Options) {
         this.#codec = codec;
         this.#renderer = renderer;
-        this.#options = options;
-
-        this.#decoder = new VideoDecoder({
-            output: (frame) => {
-                this.#captureFrame?.close();
-                // PERF: `VideoFrame#clone` is cheap
-                this.#captureFrame = frame.clone();
-
-                if (this.#drawing) {
-                    if (this.#nextFrame) {
-                        this.#nextFrame.close();
-                        this.#framesSkipped += 1;
-                    }
-                    this.#nextFrame = frame;
-                    return;
-                }
-
-                void this.#draw(frame);
-            },
-            error: (error) => {
-                this.#setError(error);
-            },
-        });
 
+        let codecTransform: CodecTransformStream;
         switch (this.#codec) {
             case ScrcpyVideoCodecId.H264:
-                this.#codecDecoder = new H264Decoder(
-                    this.#decoder,
-                    this.#updateSize,
-                    this.#options,
-                );
+                codecTransform = new H264TransformStream();
                 break;
             case ScrcpyVideoCodecId.H265:
-                this.#codecDecoder = new H265Decoder(
-                    this.#decoder,
-                    this.#updateSize,
-                    this.#options,
-                );
+                codecTransform = new H265TransformStream();
                 break;
-            case ScrcpyVideoCodecId.AV1:
-                this.#codecDecoder = new Av1Codec(
-                    this.#decoder,
-                    this.#updateSize,
-                    this.#options,
-                );
+            case ScrcpyVideoCodecId.Av1:
+                codecTransform = new Av1TransformStream();
+                break;
+            case ScrcpyVideoCodecId.Vp8:
+                codecTransform = new Vp8TransformStream();
+                break;
+            case ScrcpyVideoCodecId.Vp9:
+                codecTransform = new Vp9TransformStream();
                 break;
             default:
-                throw new Error(`Unsupported codec: ${this.#codec as number}`);
+                // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+                throw new Error(`Unknown codec: ${this.#codec}`);
         }
 
-        this.#writable = new WritableStream<ScrcpyMediaStreamPacket>({
-            start: (controller) => {
-                if (this.#error) {
-                    controller.error(this.#error);
-                } else {
-                    this.#controller = controller;
-                }
-            },
-            write: (packet) => {
-                this.#codecDecoder.decode(packet);
-            },
-        });
-
-        this.#handleAnimationFrame();
-    }
-
-    #setError(error: Error) {
-        if (this.#controller) {
-            try {
-                this.#controller.error(error);
-            } catch {
-                // ignore
-            }
-        } else {
-            this.#error = error;
-        }
+        void this.#pause.readable
+            // Add timestamp
+            .pipeThrough(this.#timestampTransforms.addTimestamp)
+            // Convert Scrcpy packets to `VideoDecoder` config/chunk
+            .pipeThrough(codecTransform)
+            // Insert extra `VideoDecoder` config and intercept size changes
+            .pipeThrough(
+                new InspectStream((chunk): undefined => {
+                    if ("codec" in chunk) {
+                        chunk.hardwareAcceleration = hardwareAcceleration;
+                        chunk.optimizeForLatency = optimizeForLatency;
+
+                        this.#size.setSize(chunk.codedWidth, chunk.codedHeight);
+                    }
+                }),
+            )
+            // Decode `VideoDecoder` config/chunk to `VideoFrame`s
+            .pipeThrough(this.#rawDecoder)
+            // Track decoding time and filter skipped frames
+            .pipeThrough(this.#timestampTransforms.consumeTimestamp)
+            // Skip frames if renderer can't keep up
+            .pipeThrough(this.#renderController)
+            // Render
+            .pipeTo(renderer.writable)
+            // Errors will be handled by source stream
+            .catch(() => {});
     }
 
-    async #draw(frame: VideoFrame) {
-        try {
-            this.#drawing = true;
-            // PERF: Draw every frame to minimize latency at cost of performance.
-            // When multiple frames are drawn in one vertical sync interval,
-            // only the last one is visible to users.
-            // But this ensures users can always see the most up-to-date screen.
-            // This is also the behavior of official Scrcpy client.
-            // https://github.com/Genymobile/scrcpy/issues/3679
-            this.#updateSize(frame.displayWidth, frame.displayHeight);
-            await this.#renderer.draw(frame);
-            this.#framesDraw += 1;
-            frame.close();
-
-            if (this.#nextFrame) {
-                const frame = this.#nextFrame;
-                this.#nextFrame = undefined;
-                await this.#draw(frame);
-            }
-
-            this.#drawing = false;
-        } catch (error) {
-            this.#setError(error as Error);
-        }
+    pause(): void {
+        this.#pause.pause();
     }
 
-    #updateSize = (width: number, height: number) => {
-        this.#renderer.setSize(width, height);
-        this.#width = width;
-        this.#height = height;
-        this.#sizeChanged.fire({ width, height });
-    };
+    resume(): undefined {
+        this.#pause.resume();
+    }
 
-    #handleAnimationFrame = () => {
-        if (this.#framesDraw > 0) {
-            this.#framesPresented += 1;
-            this.#framesSkipped += this.#framesDraw - 1;
-            this.#framesDraw = 0;
-        }
-        this.#animationFrameId = requestAnimationFrame(
-            this.#handleAnimationFrame,
-        );
-    };
+    trackDocumentVisibility(document: Document): () => undefined {
+        return this.#pause.trackDocumentVisibility(document);
+    }
 
-    async snapshot() {
-        const frame = this.#captureFrame;
+    async snapshot(options?: ImageEncodeOptions) {
+        const frame = this.#renderController.captureFrame;
         if (!frame) {
             return undefined;
         }
 
-        const capturer = await VideoFrameCapturerPool.borrow();
-        const result = await capturer.capture(frame);
-        VideoFrameCapturerPool.return(capturer);
-        return result;
-    }
+        // First check if the renderer can provide a snapshot natively
+        const blob = await this.#renderer.snapshot?.(options);
+        if (blob) {
+            return blob;
+        }
 
-    dispose() {
-        cancelAnimationFrame(this.#animationFrameId);
-        if (this.#decoder.state !== "closed") {
-            this.#decoder.close();
+        // Create a BitmapVideoFrameRenderer to draw the frame
+        const renderer = new BitmapVideoFrameRenderer();
+        try {
+            const writer = renderer.writable.getWriter();
+            // Draw the frame
+            // `renderer` will close the passed in `frame`, so make a clone
+            await writer.write(frame.clone());
+            await writer.close();
+            // BitmapVideoFrameRenderer.snapshot will definitely return a value
+            return await renderer.snapshot(options);
+        } finally {
+            renderer.dispose();
         }
-        this.#nextFrame?.close();
-        this.#captureFrame?.close();
+    }
+
+    dispose(): undefined {
+        // Most cleanup happens automatically when `writable` ends
+        // (in each stream's `close` callback).
+        // This method cleanup things that still available after `writable` ends
+
+        this.#pause.dispose();
+        this.#size.dispose();
+        this.#renderController.dispose();
+        this.#renderer.dispose?.();
     }
 }
 
 export namespace WebCodecsVideoDecoder {
-    export interface Options extends CodecDecoderOptions {
+    export interface Options extends Pick<
+        VideoDecoderConfig,
+        "hardwareAcceleration" | "optimizeForLatency"
+    > {
         /**
          * The video codec to decode
          */
         codec: ScrcpyVideoCodecId;
 
-        renderer: VideoFrameRenderer;
+        renderer?: VideoFrameRenderer | undefined;
     }
 }

package/src/video/index.ts

@@ -1,3 +1,4 @@
 export * from "./codec/index.js";
 export * from "./decoder.js";
 export * from "./render/index.js";
+export * from "./utils/index.js";

package/src/video/render/auto-canvas.ts

@@ -0,0 +1,105 @@
+import { StickyEventEmitter } from "@yume-chan/event";
+import type { ScrcpyVideoDecoder } from "@yume-chan/scrcpy-decoder-shared";
+import { createCanvas } from "@yume-chan/scrcpy-decoder-shared";
+import { WritableStream } from "@yume-chan/stream-extra";
+
+import { BitmapVideoFrameRenderer } from "./bitmap.js";
+import type { CanvasVideoFrameRenderer } from "./canvas.js";
+import type { VideoFrameRenderer } from "./type.js";
+import { WebGLVideoFrameRenderer } from "./webgl.js";
+
+export class AutoCanvasRenderer implements VideoFrameRenderer {
+    #inner!: CanvasVideoFrameRenderer;
+    #innerWriter!: WritableStreamDefaultWriter<VideoFrame>;
+
+    get type() {
+        return this.#inner.type;
+    }
+    #onTypeChanged = new StickyEventEmitter<ScrcpyVideoDecoder.RendererType>();
+    get onTypeChanged() {
+        return this.#onTypeChanged.event;
+    }
+
+    #canvas!: HTMLCanvasElement | OffscreenCanvas;
+    /**
+     * Gets the canvas used by the renderer.
+     *
+     * The `canvas` property might change when the renderer switches between WebGL and 2D rendering.
+     * Specify a `createCanvas` option, or listen to the `onTypeChanged` event to handle the canvas change.
+     */
+    get canvas() {
+        return this.#canvas;
+    }
+
+    #lastFrame: VideoFrame | undefined;
+    #writableController!: WritableStreamDefaultController;
+    #writable = new WritableStream<VideoFrame>({
+        start: (controller) => {
+            this.#writableController = controller;
+        },
+        write: async (frame) => {
+            this.#lastFrame?.close();
+            this.#lastFrame = frame.clone();
+
+            await this.#innerWriter.write(frame);
+        },
+    });
+    get writable() {
+        return this.#writable;
+    }
+
+    constructor(options?: AutoCanvasRenderer.Options) {
+        this.#createInner(options, WebGLVideoFrameRenderer.isSupported);
+    }
+
+    #createInner(
+        options: AutoCanvasRenderer.Options | undefined,
+        webgl: boolean,
+    ) {
+        this.#canvas = options?.createCanvas?.() ?? createCanvas();
+        if (webgl) {
+            const webgl = new WebGLVideoFrameRenderer({
+                ...options,
+                canvas: this.#canvas,
+            });
+            webgl.onContextLost(() => {
+                this.#createInner(options, false);
+                if (this.#lastFrame) {
+                    this.#innerWriter
+                        .write(this.#lastFrame.clone())
+                        .catch((e) => this.#writableController.error(e));
+                }
+            });
+            this.#inner = webgl;
+        } else {
+            this.#inner = new BitmapVideoFrameRenderer({
+                ...options,
+                canvas: this.#canvas,
+            });
+        }
+        this.#innerWriter = this.#inner.writable.getWriter();
+        this.#onTypeChanged.fire(this.#inner.type);
+    }
+
+    snapshot(options?: ImageEncodeOptions): Promise<Blob | undefined> {
+        return this.#inner.snapshot(options);
+    }
+
+    dispose() {
+        this.#onTypeChanged.dispose();
+
+        this.#lastFrame?.close();
+        this.#lastFrame = undefined;
+
+        return this.#inner.dispose();
+    }
+}
+
+export namespace AutoCanvasRenderer {
+    export interface Options extends Omit<
+        WebGLVideoFrameRenderer.Options,
+        "canvas"
+    > {
+        createCanvas?: () => HTMLCanvasElement | OffscreenCanvas;
+    }
+}

package/src/video/render/bitmap.ts

@@ -1,18 +1,32 @@
 import { CanvasVideoFrameRenderer } from "./canvas.js";
 
 export class BitmapVideoFrameRenderer extends CanvasVideoFrameRenderer {
+    override get type() {
+        return "software" as const;
+    }
+
     #context: ImageBitmapRenderingContext;
 
-    constructor(canvas?: HTMLCanvasElement | OffscreenCanvas) {
-        super(canvas);
+    constructor(options?: CanvasVideoFrameRenderer.Options) {
+        super(async (frame) => {
+            const bitmap = await createImageBitmap(frame);
+            this.#context.transferFromImageBitmap(bitmap);
+            bitmap.close();
+        }, options);
 
-        this.#context = this.canvas.getContext("bitmaprenderer", {
-            alpha: false,
-        })!;
-    }
+        const context = (this.canvas as HTMLCanvasElement).getContext(
+            "bitmaprenderer",
+            {
+                // Avoid alpha:false, which can be expensive
+                // https://developer.mozilla.org/en-US/docs/Web/API/WebGL_API/WebGL_best_practices#avoid_alphafalse_which_can_be_expensive
+                alpha: true,
+            },
+        );
+        if (!context) {
+            // This can happen if the canvas already has a context of a different type
+            throw new Error("Failed to create rendering context");
+        }
 
-    async draw(frame: VideoFrame): Promise<void> {
-        const bitmap = await createImageBitmap(frame);
-        this.#context.transferFromImageBitmap(bitmap);
+        this.#context = context;
     }
 }