@yume-chan/scrcpy-decoder-webcodecs 2.5.3 → 3.0.0-beta.0

package/src/video/render/canvas.ts

@@ -1,27 +1,172 @@
-import { createCanvas } from "@yume-chan/scrcpy-decoder-tinyh264";
+import type { MaybePromiseLike } from "@yume-chan/async";
+import type { ScrcpyVideoDecoder } from "@yume-chan/scrcpy-decoder-shared";
+import { createCanvas } from "@yume-chan/scrcpy-decoder-shared";
+import { WritableStream } from "@yume-chan/stream-extra";
 
+import { canvasToBlob } from "../utils/index.js";
+
+import { RedrawController } from "./redraw.js";
 import type { VideoFrameRenderer } from "./type.js";
 
-export abstract class CanvasVideoFrameRenderer implements VideoFrameRenderer {
+export abstract class CanvasVideoFrameRenderer<
+    TOptions extends CanvasVideoFrameRenderer.Options =
+        CanvasVideoFrameRenderer.Options,
+> implements VideoFrameRenderer {
+    abstract get type(): ScrcpyVideoDecoder.RendererType;
+
     #canvas: HTMLCanvasElement | OffscreenCanvas;
     get canvas() {
         return this.#canvas;
     }
 
-    constructor(canvas?: HTMLCanvasElement | OffscreenCanvas) {
-        if (canvas) {
-            this.#canvas = canvas;
-        } else {
-            this.#canvas = createCanvas();
+    #options: TOptions | undefined;
+    #canvasSize: CanvasVideoFrameRenderer.Options["canvasSize"];
+    get options(): Readonly<TOptions> | undefined {
+        return this.#options;
+    }
+
+    #resizeObserver: ResizeObserver | undefined;
+    #displayWidth = Infinity;
+    #displayHeight = Infinity;
+
+    #draw: (frame: VideoFrame) => MaybePromiseLike<undefined>;
+    #controller = new RedrawController((frame) => {
+        if (this.#canvasSize !== "external") {
+            this.#updateSize(frame);
+        }
+
+        return this.#draw(frame);
+    });
+    get lastFrame() {
+        return this.#controller.lastFrame;
+    }
+
+    #writable = new WritableStream<VideoFrame>({
+        write: (frame) => this.#controller.draw(frame),
+    });
+    get writable() {
+        return this.#writable;
+    }
+
+    constructor(
+        draw: (frame: VideoFrame) => MaybePromiseLike<undefined>,
+        options?: TOptions,
+    ) {
+        this.#draw = draw;
+        this.#canvas = options?.canvas ?? createCanvas();
+        this.#options = options;
+        this.#canvasSize = options?.canvasSize ?? "video";
+
+        if (this.#canvasSize === "display") {
+            if (
+                typeof HTMLCanvasElement === "undefined" ||
+                !(this.#canvas instanceof HTMLCanvasElement)
+            ) {
+                throw new Error(
+                    "`canvasSize: display` is only supported for HTMLCanvasElement",
+                );
+            }
+
+            this.#resizeObserver = new ResizeObserver((entries) => {
+                const entry = entries[0];
+                if (!entry) {
+                    return;
+                }
+
+                const devicePixelSize = entry.devicePixelContentBoxSize?.[0];
+                if (devicePixelSize) {
+                    this.#setDisplaySize(
+                        devicePixelSize.inlineSize,
+                        devicePixelSize.blockSize,
+                    );
+                    return;
+                }
+
+                const cssSize = entry.contentBoxSize[0];
+                if (cssSize) {
+                    this.#setDisplaySize(
+                        Math.round(cssSize.inlineSize * devicePixelRatio),
+                        Math.round(cssSize.blockSize * devicePixelRatio),
+                    );
+                }
+            });
+            this.#resizeObserver.observe(this.#canvas);
         }
     }
 
-    setSize(width: number, height: number): void {
-        if (this.#canvas.width !== width || this.#canvas.height !== height) {
-            this.#canvas.width = width;
-            this.#canvas.height = height;
+    #setDisplaySize(width: number, height: number) {
+        if (this.#displayWidth === width && this.#displayHeight === height) {
+            return;
         }
+
+        this.#displayWidth = width;
+        this.#displayHeight = height;
+
+        void this.#controller.redraw();
     }
 
-    abstract draw(frame: VideoFrame): Promise<void>;
+    #updateSize(frame: VideoFrame) {
+        let { codedWidth: width, codedHeight: height } = frame;
+        if (this.#canvasSize === "display") {
+            width = Math.min(width, this.#displayWidth);
+            height = Math.min(height, this.#displayHeight);
+        }
+
+        if (this.#canvas.width === width && this.#canvas.height === height) {
+            return false;
+        }
+
+        this.#canvas.width = width;
+        this.#canvas.height = height;
+        return true;
+    }
+
+    /**
+     * Redraws the last drawn frame.
+     *
+     * If a draw or redraw is in progress, it waits for them to finish before redrawing.
+     *
+     * If a redraw is in progress and another one is in queue,
+     * it cancels the queued redraw and redraws the latest frame instead.
+     */
+    redraw() {
+        return this.#controller.redraw();
+    }
+
+    async snapshot(options?: ImageEncodeOptions): Promise<Blob | undefined> {
+        if (this.#canvasSize !== "video") {
+            return undefined;
+        }
+        return canvasToBlob(this.#canvas, options);
+    }
+
+    dispose(): undefined {
+        if (this.#canvasSize !== "external") {
+            this.#canvas.width = 0;
+            this.#canvas.height = 0;
+        }
+
+        this.#resizeObserver?.disconnect();
+        this.#controller.dispose();
+    }
+}
+
+export namespace CanvasVideoFrameRenderer {
+    export interface Options {
+        canvas?: HTMLCanvasElement | OffscreenCanvas;
+
+        /**
+         * Whether to update the canvas size (rendering resolution) automatically.
+         *
+         * * `"video"` (default): update the canvas size to match the video resolution
+         * * `"display"` (only for `HTMLCanvasElement`):
+         *    update the canvas size to match the display size.
+         *    The display size can be set using `canvas.style.width/height`,
+         *    and must be in correct aspect ratio.
+         * * `"external"`: use the canvas size as it is.
+         *    The size must be manually set using `canvas.width/height`,
+         *    and must be in correct aspect ratio.
+         */
+        canvasSize?: "video" | "display" | "external";
+    }
 }

package/src/video/render/flow-control.ts

@@ -0,0 +1,182 @@
+import type { ScrcpyVideoRendererPerformanceCounterInterface } from "@yume-chan/scrcpy-decoder-shared";
+import { ScrcpyVideoRendererPerformanceCounter } from "@yume-chan/scrcpy-decoder-shared";
+import type {
+    PushReadableStreamController,
+    ReadableStream,
+    TransformStream,
+    WritableStreamDefaultController,
+} from "@yume-chan/stream-extra";
+import {
+    PushReadableStream,
+    tryClose,
+    WritableStream,
+} from "@yume-chan/stream-extra";
+
+export class RendererController
+    implements
+        TransformStream<VideoFrame, VideoFrame>,
+        ScrcpyVideoRendererPerformanceCounterInterface
+{
+    #readable: ReadableStream<VideoFrame>;
+    #readableController!: PushReadableStreamController<VideoFrame>;
+    get readable() {
+        return this.#readable;
+    }
+
+    #writable: WritableStream<VideoFrame>;
+    #writableController!: WritableStreamDefaultController;
+    get writable() {
+        return this.#writable;
+    }
+
+    #captureFrame: VideoFrame | undefined;
+    get captureFrame() {
+        return this.#captureFrame;
+    }
+
+    #nextFrame: VideoFrame | undefined;
+
+    #drawTask: Promise<undefined> | undefined;
+
+    #counter = new ScrcpyVideoRendererPerformanceCounter();
+    /**
+     * Gets the number of frames that have been drawn on the renderer.
+     */
+    get framesRendered() {
+        return this.#counter.framesRendered;
+    }
+    /**
+     * Gets the number of frames that's visible to the user.
+     *
+     * Multiple frames might be rendered during one vertical sync interval,
+     * but only the last of them is represented to the user.
+     * This costs some performance but reduces latency by 1 frame.
+     *
+     * Might be `0` if the renderer is in a nested Web Worker on Chrome due to a Chrome bug.
+     * https://issues.chromium.org/issues/41483010
+     */
+    get framesDisplayed() {
+        return this.#counter.framesDisplayed;
+    }
+    /**
+     * Gets the number of frames that wasn't drawn on the renderer
+     * because the renderer can't keep up
+     */
+    get framesSkippedRendering() {
+        return this.#counter.framesSkippedRendering;
+    }
+
+    constructor() {
+        this.#readable = new PushReadableStream((controller) => {
+            this.#readableController = controller;
+        });
+
+        this.#writable = new WritableStream({
+            start: (controller) => {
+                this.#writableController = controller;
+
+                // Propagate `readable` error back to `writable`
+                const signal = this.#readableController.abortSignal;
+                signal.addEventListener("abort", () =>
+                    controller.error(signal.reason),
+                );
+            },
+            write: (frame) => {
+                this.#captureFrame?.close();
+                // `#captureFrame` and `#nextFrame` must not be the same object
+                // because they need to be closed at different times
+                this.#captureFrame = frame.clone();
+
+                // Frame A is drawing, frame B (`#nextFrame`) is waiting,
+                // then frame C (`frame`) arrives.
+                // Skip frame B and queue frame C
+                if (this.#nextFrame) {
+                    this.#nextFrame.close();
+                    this.#counter.increaseFramesSkippedRendering();
+                }
+                this.#nextFrame = frame;
+
+                // Don't `await` because this writable needs to
+                // accept incoming frames as fast as produced.
+                // The `#draw` method then draws the frames
+                // as fast as the renderer can keep up
+                void this.#tryStartDrawing();
+            },
+            close: async () => {
+                // Normally `WritableStream` only calls `close` after `write` finishes,
+                // but because our `write` doesn't wait for `#draw`,
+                // we might still need to send `#nextFrame` to `readable`.
+                // So wait for `#drawTask` before closing `readable`
+                await this.#drawTask;
+                // `#nextFrame` must be `undefined` at this point
+
+                // Propagate `writable` close to `readable`
+                this.#readableController.close();
+
+                this.#counter.dispose();
+
+                // Don't close `#captureFrame` to allow using `snapshot` on the last frame
+            },
+            abort: async (reason) => {
+                // See `close` above
+                await this.#drawTask;
+
+                // Propagate `writable` error to `readable`
+                this.#readableController.error(reason);
+
+                this.#counter.dispose();
+            },
+        });
+    }
+
+    async #draw(): Promise<undefined> {
+        // PERF: Draw every frame to minimize latency at cost of performance.
+        // When multiple frames are drawn in one vertical sync interval,
+        // only the last one is visible to users.
+        // But this ensures users can always see the most up-to-date screen.
+        // This is also the behavior of official Scrcpy client.
+        // https://github.com/Genymobile/scrcpy/issues/3679
+
+        let frame: VideoFrame | undefined;
+        while ((frame = this.#nextFrame)) {
+            this.#nextFrame = undefined;
+            if (await this.#readableController.enqueue(frame)) {
+                // The consumer is responsible for closing `frame`
+                this.#counter.increaseFramesRendered();
+            } else {
+                // `enqueue` returning `false` means the consumer has called `readable.cancel()`,
+                // before consuming the `frame`. So close it here.
+                frame.close();
+
+                // Continue looping to close all `#nextFrame`s
+                // In the mean time the `writable` is erroring out its source
+                // so no more frames will arrive
+            }
+        }
+
+        this.#drawTask = undefined;
+    }
+
+    #tryStartDrawing() {
+        if (!this.#drawTask) {
+            this.#drawTask = this.#draw();
+        }
+    }
+
+    dispose() {
+        this.#captureFrame?.close();
+        this.#captureFrame = undefined;
+
+        this.#nextFrame?.close();
+        this.#nextFrame = undefined;
+
+        this.#counter.dispose();
+
+        tryClose(this.#readableController);
+
+        // Throw an error similar to native TransformStream
+        this.#writableController.error(
+            new TypeError("The transform stream has been terminated"),
+        );
+    }
+}

package/src/video/render/index.ts

@@ -2,6 +2,8 @@
 
 export * from "./bitmap.js";
 export * from "./canvas.js";
+export * from "./auto-canvas.js";
+export * from "./flow-control.js";
 export * from "./insertable-stream.js";
 export * from "./type.js";
 export * from "./webgl.js";

package/src/video/render/insertable-stream.ts

@@ -1,5 +1,7 @@
 // cspell: ignore insertable
 
+import type { WritableStream } from "@yume-chan/stream-extra";
+
 import type { VideoFrameRenderer } from "./type.js";
 
 declare class MediaStreamTrackGenerator extends MediaStreamTrack {
@@ -13,16 +15,36 @@ export class InsertableStreamVideoFrameRenderer implements VideoFrameRenderer {
         return typeof MediaStreamTrackGenerator !== "undefined";
     }
 
+    get type() {
+        return "hardware" as const;
+    }
+
     #element: HTMLVideoElement;
     get element() {
         return this.#element;
     }
 
+    #options: InsertableStreamVideoFrameRenderer.Options | undefined;
+    get options() {
+        return this.#options;
+    }
+
     #generator: MediaStreamTrackGenerator;
-    #writer: WritableStreamDefaultWriter<VideoFrame>;
+    get writable() {
+        return this.#generator.writable;
+    }
+
     #stream: MediaStream;
+    get stream() {
+        return this.#stream;
+    }
+
+    #abortController = new AbortController();
 
-    constructor(element?: HTMLVideoElement) {
+    constructor(
+        element?: HTMLVideoElement,
+        options?: InsertableStreamVideoFrameRenderer.Options,
+    ) {
         if (element) {
             this.#element = element;
         } else if (typeof document !== "undefined") {
@@ -34,27 +56,45 @@ export class InsertableStreamVideoFrameRenderer implements VideoFrameRenderer {
         }
         this.#element.muted = true;
         this.#element.autoplay = true;
+        this.#element.playsInline = true;
         this.#element.disablePictureInPicture = true;
         this.#element.disableRemotePlayback = true;
 
+        this.#options = options;
+        if (options?.updateSize) {
+            this.#element.addEventListener(
+                "resize",
+                () => {
+                    this.#element.width = this.#element.videoWidth;
+                    this.#element.height = this.#element.videoHeight;
+                },
+                { signal: this.#abortController.signal },
+            );
+        }
+
         // The spec replaced `MediaStreamTrackGenerator` with `VideoTrackGenerator`.
         // But Chrome has not implemented it yet.
         // https://issues.chromium.org/issues/40058895
         this.#generator = new MediaStreamTrackGenerator({ kind: "video" });
-        this.#writer = this.#generator.writable.getWriter();
+        this.#generator.contentHint = "motion";
 
         this.#stream = new MediaStream([this.#generator]);
         this.#element.srcObject = this.#stream;
     }
 
-    setSize(width: number, height: number): void {
-        if (this.#element.width !== width || this.#element.height !== height) {
-            this.#element.width = width;
-            this.#element.height = height;
-        }
+    dispose(): undefined {
+        this.#abortController.abort();
+        this.#generator.stop();
+        this.#stream.removeTrack(this.#generator);
+        this.#element.srcObject = null;
     }
+}
 
-    async draw(frame: VideoFrame): Promise<void> {
-        await this.#writer.write(frame);
+export namespace InsertableStreamVideoFrameRenderer {
+    export interface Options {
+        /**
+         * Whether to update the size of the video element when the size of the video frame changes.
+         */
+        updateSize?: boolean;
     }
 }

package/src/video/render/redraw.ts

@@ -0,0 +1,88 @@
+import type { MaybePromiseLike } from "@yume-chan/async";
+import { TaskQueue } from "@yume-chan/stream-extra";
+
+/**
+ * Manages drawing and redrawing of video frames.
+ */
+export class RedrawController {
+    #draw: (frame: VideoFrame) => MaybePromiseLike<undefined>;
+
+    #queue = new TaskQueue();
+    #pendingRedraw: AbortController | undefined;
+
+    #lastFrame: VideoFrame | undefined;
+    get lastFrame() {
+        return this.#lastFrame;
+    }
+
+    constructor(draw: (frame: VideoFrame) => MaybePromiseLike<undefined>) {
+        this.#draw = draw;
+    }
+
+    /**
+     * Draws a new frame.
+     *
+     * If a redraw is in progress, it waits for the redraw to finish before drawing the new frame.
+     *
+     * If a redraw is in progress and another one is in queue,
+     * it cancels the queued redraw and draws the new frame instead.
+     *
+     * @param frame A `VideoFrame` to draw. The frame will be closed after drawing.
+     */
+    draw(frame: VideoFrame) {
+        this.#pendingRedraw?.abort();
+        this.#pendingRedraw = undefined;
+
+        this.#lastFrame?.close();
+        this.#lastFrame = frame.clone();
+
+        return this.#queue.enqueue(async (): Promise<undefined> => {
+            try {
+                await this.#draw(frame);
+            } finally {
+                frame.close();
+            }
+        }, true);
+    }
+
+    /**
+     * Redraws the last drawn frame.
+     *
+     * If a draw or redraw is in progress, it waits for them to finish before redrawing.
+     *
+     * If a redraw is in progress and another one is in queue,
+     * it cancels the queued redraw and redraws the latest frame instead.
+     */
+    async redraw(): Promise<undefined> {
+        if (!this.#lastFrame || this.#pendingRedraw) {
+            return;
+        }
+
+        const abortController = new AbortController();
+        this.#pendingRedraw = abortController;
+
+        return await this.#queue.enqueue(async (): Promise<undefined> => {
+            if (abortController.signal.aborted) {
+                return;
+            }
+
+            this.#pendingRedraw = undefined;
+
+            const frame = this.#lastFrame!.clone();
+            try {
+                await this.#draw(frame);
+            } finally {
+                frame.close();
+            }
+        }, true);
+    }
+
+    dispose() {
+        this.#pendingRedraw?.abort();
+
+        this.#lastFrame?.close();
+        this.#lastFrame = undefined;
+
+        this.#queue.dispose();
+    }
+}

package/src/video/render/type.ts

@@ -1,7 +1,15 @@
 import type { MaybePromiseLike } from "@yume-chan/async";
+import type { Event } from "@yume-chan/event";
+import type { ScrcpyVideoDecoder } from "@yume-chan/scrcpy-decoder-shared";
+import type { WritableStream } from "@yume-chan/stream-extra";
 
 export interface VideoFrameRenderer {
-    setSize(width: number, height: number): void;
+    readonly type: ScrcpyVideoDecoder.RendererType;
+    readonly onTypeChanged?: Event<ScrcpyVideoDecoder.RendererType>;
 
-    draw(frame: VideoFrame): MaybePromiseLike<void>;
+    readonly writable: WritableStream<VideoFrame>;
+
+    snapshot?(options?: ImageEncodeOptions): Promise<Blob | undefined>;
+
+    dispose?(): MaybePromiseLike<undefined>;
 }