@yume-chan/scrcpy-decoder-webcodecs 0.0.22 → 0.0.24

package/src/video/codec/av1.ts

@@ -0,0 +1,101 @@
+import type { ScrcpyMediaStreamPacket } from "@yume-chan/scrcpy";
+import { Av1 } from "@yume-chan/scrcpy";
+
+import type { CodecDecoder } from "./type.js";
+import { decimalTwoDigits } from "./utils.js";
+
+export class Av1Codec implements CodecDecoder {
+    #decoder: VideoDecoder;
+    #updateSize: (width: number, height: number) => void;
+
+    constructor(
+        decoder: VideoDecoder,
+        updateSize: (width: number, height: number) => void,
+    ) {
+        this.#decoder = decoder;
+        this.#updateSize = updateSize;
+    }
+
+    #configure(data: Uint8Array) {
+        const parser = new Av1(data);
+        const sequenceHeader = parser.searchSequenceHeaderObu();
+
+        if (!sequenceHeader) {
+            return;
+        }
+
+        const {
+            seq_profile: seqProfile,
+            seq_level_idx: [seqLevelIdx = 0],
+            max_frame_width_minus_1,
+            max_frame_height_minus_1,
+            color_config: {
+                BitDepth,
+                mono_chrome: monoChrome,
+                subsampling_x: subsamplingX,
+                subsampling_y: subsamplingY,
+                chroma_sample_position: chromaSamplePosition,
+                color_description_present_flag,
+            },
+        } = sequenceHeader;
+
+        let colorPrimaries: Av1.ColorPrimaries;
+        let transferCharacteristics: Av1.TransferCharacteristics;
+        let matrixCoefficients: Av1.MatrixCoefficients;
+        let colorRange: boolean;
+        if (color_description_present_flag) {
+            ({
+                color_primaries: colorPrimaries,
+                transfer_characteristics: transferCharacteristics,
+                matrix_coefficients: matrixCoefficients,
+                color_range: colorRange,
+            } = sequenceHeader.color_config);
+        } else {
+            colorPrimaries = Av1.ColorPrimaries.Bt709;
+            transferCharacteristics = Av1.TransferCharacteristics.Bt709;
+            matrixCoefficients = Av1.MatrixCoefficients.Bt709;
+            colorRange = false;
+        }
+
+        const width = max_frame_width_minus_1 + 1;
+        const height = max_frame_height_minus_1 + 1;
+
+        this.#updateSize(width, height);
+
+        const codec = [
+            "av01",
+            seqProfile.toString(16),
+            decimalTwoDigits(seqLevelIdx) +
+                (sequenceHeader.seq_tier[0] ? "H" : "M"),
+            decimalTwoDigits(BitDepth),
+            monoChrome ? "1" : "0",
+            (subsamplingX ? "1" : "0") +
+                (subsamplingY ? "1" : "0") +
+                chromaSamplePosition.toString(),
+            decimalTwoDigits(colorPrimaries),
+            decimalTwoDigits(transferCharacteristics),
+            decimalTwoDigits(matrixCoefficients),
+            colorRange ? "1" : "0",
+        ].join(".");
+        this.#decoder.configure({
+            codec,
+            optimizeForLatency: true,
+        });
+    }
+
+    decode(packet: ScrcpyMediaStreamPacket): void {
+        if (packet.type === "configuration") {
+            return;
+        }
+
+        this.#configure(packet.data);
+        this.#decoder.decode(
+            new EncodedVideoChunk({
+                // Treat `undefined` as `key`, otherwise it won't decode.
+                type: packet.keyframe === false ? "delta" : "key",
+                timestamp: 0,
+                data: packet.data,
+            }),
+        );
+    }
+}

package/src/video/codec/h264.ts

@@ -0,0 +1,42 @@
+import { h264ParseConfiguration } from "@yume-chan/scrcpy";
+
+import { H26xDecoder } from "./h26x.js";
+import { hexTwoDigits } from "./utils.js";
+
+export class H264Decoder extends H26xDecoder {
+    #decoder: VideoDecoder;
+    #updateSize: (width: number, height: number) => void;
+
+    constructor(
+        decoder: VideoDecoder,
+        updateSize: (width: number, height: number) => void,
+    ) {
+        super(decoder);
+        this.#decoder = decoder;
+        this.#updateSize = updateSize;
+    }
+
+    override configure(data: Uint8Array): void {
+        const {
+            profileIndex,
+            constraintSet,
+            levelIndex,
+            croppedWidth,
+            croppedHeight,
+        } = h264ParseConfiguration(data);
+
+        this.#updateSize(croppedWidth, croppedHeight);
+
+        // https://www.rfc-editor.org/rfc/rfc6381#section-3.3
+        // ISO Base Media File Format Name Space
+        const codec =
+            "avc1." +
+            hexTwoDigits(profileIndex) +
+            hexTwoDigits(constraintSet) +
+            hexTwoDigits(levelIndex);
+        this.#decoder.configure({
+            codec: codec,
+            optimizeForLatency: true,
+        });
+    }
+}

package/src/video/codec/h265.ts

@@ -0,0 +1,47 @@
+import { getUint32LittleEndian } from "@yume-chan/no-data-view";
+import { h265ParseConfiguration } from "@yume-chan/scrcpy";
+
+import { H26xDecoder } from "./h26x.js";
+import { hexDigits } from "./utils.js";
+
+export class H265Decoder extends H26xDecoder {
+    #decoder: VideoDecoder;
+    #updateSize: (width: number, height: number) => void;
+
+    constructor(
+        decoder: VideoDecoder,
+        updateSize: (width: number, height: number) => void,
+    ) {
+        super(decoder);
+        this.#decoder = decoder;
+        this.#updateSize = updateSize;
+    }
+
+    override configure(data: Uint8Array): void {
+        const {
+            generalProfileSpace,
+            generalProfileIndex,
+            generalProfileCompatibilitySet,
+            generalTierFlag,
+            generalLevelIndex,
+            generalConstraintSet,
+            croppedWidth,
+            croppedHeight,
+        } = h265ParseConfiguration(data);
+
+        this.#updateSize(croppedWidth, croppedHeight);
+
+        const codec = [
+            "hev1",
+            ["", "A", "B", "C"][generalProfileSpace]! +
+                generalProfileIndex.toString(),
+            hexDigits(getUint32LittleEndian(generalProfileCompatibilitySet, 0)),
+            (generalTierFlag ? "H" : "L") + generalLevelIndex.toString(),
+            ...Array.from(generalConstraintSet, hexDigits),
+        ].join(".");
+        this.#decoder.configure({
+            codec,
+            optimizeForLatency: true,
+        });
+    }
+}

package/src/video/codec/h26x.ts

@@ -0,0 +1,45 @@
+import type { ScrcpyMediaStreamPacket } from "@yume-chan/scrcpy";
+
+import type { CodecDecoder } from "./type.js";
+
+export abstract class H26xDecoder implements CodecDecoder {
+    #config: Uint8Array | undefined;
+    #decoder: VideoDecoder;
+
+    constructor(decoder: VideoDecoder) {
+        this.#decoder = decoder;
+    }
+
+    abstract configure(data: Uint8Array): void;
+
+    decode(packet: ScrcpyMediaStreamPacket): void {
+        if (packet.type === "configuration") {
+            this.#config = packet.data;
+            this.configure(packet.data);
+            return;
+        }
+
+        // For H.264 and H.265, when the stream is in Annex B format
+        // (which Scrcpy uses, as Android MediaCodec produces),
+        // configuration data needs to be combined with the first frame data.
+        // https://www.w3.org/TR/webcodecs-avc-codec-registration/#encodedvideochunk-type
+        let data: Uint8Array;
+        if (this.#config !== undefined) {
+            data = new Uint8Array(this.#config.length + packet.data.length);
+            data.set(this.#config, 0);
+            data.set(packet.data, this.#config.length);
+            this.#config = undefined;
+        } else {
+            data = packet.data;
+        }
+
+        this.#decoder.decode(
+            new EncodedVideoChunk({
+                // Treat `undefined` as `key`, otherwise won't decode.
+                type: packet.keyframe === false ? "delta" : "key",
+                timestamp: 0,
+                data,
+            }),
+        );
+    }
+}

package/src/video/codec/index.ts

@@ -0,0 +1,6 @@
+export * from "./av1.js";
+export * from "./h264.js";
+export * from "./h265.js";
+export * from "./h26x.js";
+export * from "./type.js";
+export * from "./utils.js";

package/src/video/codec/type.ts

@@ -0,0 +1,12 @@
+import type { ScrcpyMediaStreamPacket } from "@yume-chan/scrcpy";
+
+export interface CodecDecoder {
+    decode(packet: ScrcpyMediaStreamPacket): void;
+}
+
+export interface CodecDecoderConstructor {
+    new (
+        decoder: VideoDecoder,
+        updateSize: (width: number, height: number) => void,
+    ): CodecDecoder;
+}

package/src/video/codec/utils.ts

@@ -0,0 +1,11 @@
+export function hexDigits(value: number) {
+    return value.toString(16).toUpperCase();
+}
+
+export function hexTwoDigits(value: number) {
+    return value.toString(16).toUpperCase().padStart(2, "0");
+}
+
+export function decimalTwoDigits(value: number) {
+    return value.toString(10).padStart(2, "0");
+}

package/src/video/decoder.ts

@@ -0,0 +1,179 @@
+import { EventEmitter } from "@yume-chan/event";
+import type { ScrcpyMediaStreamPacket } from "@yume-chan/scrcpy";
+import { ScrcpyVideoCodecId } from "@yume-chan/scrcpy";
+import type {
+    ScrcpyVideoDecoder,
+    ScrcpyVideoDecoderCapability,
+} from "@yume-chan/scrcpy-decoder-tinyh264";
+import type { WritableStreamDefaultController } from "@yume-chan/stream-extra";
+import { WritableStream } from "@yume-chan/stream-extra";
+
+import { Av1Codec, H264Decoder, H265Decoder } from "./codec/index.js";
+import type { CodecDecoder } from "./codec/type.js";
+import type { FrameRenderer } from "./render/index.js";
+import { BitmapFrameRenderer, WebGLFrameRenderer } from "./render/index.js";
+
+export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
+    static isSupported() {
+        return typeof globalThis.VideoDecoder !== "undefined";
+    }
+
+    static readonly capabilities: Record<string, ScrcpyVideoDecoderCapability> =
+        {
+            h264: {},
+            h265: {},
+        };
+
+    #codec: ScrcpyVideoCodecId;
+    get codec() {
+        return this.#codec;
+    }
+
+    #codecDecoder: CodecDecoder;
+
+    #writable: WritableStream<ScrcpyMediaStreamPacket>;
+    get writable() {
+        return this.#writable;
+    }
+
+    #canvas: HTMLCanvasElement;
+    get renderer() {
+        return this.#canvas;
+    }
+
+    #frameRendered = 0;
+    get framesRendered() {
+        return this.#frameRendered;
+    }
+
+    #frameSkipped = 0;
+    get framesSkipped() {
+        return this.#frameSkipped;
+    }
+
+    #sizeChanged = new EventEmitter<{ width: number; height: number }>();
+    get sizeChanged() {
+        return this.#sizeChanged.event;
+    }
+
+    #decoder: VideoDecoder;
+    #renderer: FrameRenderer;
+
+    #currentFrameRendered = false;
+    #animationFrameId = 0;
+
+    /**
+     * Create a new WebCodecs video decoder.
+     * @param codec The video codec to decode
+     * @param enableCapture
+     * Whether to allow capturing the canvas content using APIs like `readPixels` and `toDataURL`.
+     * Enable this option may reduce performance.
+     */
+    constructor(codec: ScrcpyVideoCodecId, enableCapture: boolean) {
+        this.#codec = codec;
+
+        this.#canvas = document.createElement("canvas");
+
+        try {
+            this.#renderer = new WebGLFrameRenderer(
+                this.#canvas,
+                enableCapture,
+            );
+        } catch {
+            this.#renderer = new BitmapFrameRenderer(this.#canvas);
+        }
+
+        this.#decoder = new VideoDecoder({
+            output: (frame) => {
+                if (this.#currentFrameRendered) {
+                    this.#frameRendered += 1;
+                } else {
+                    this.#frameSkipped += 1;
+                }
+                this.#currentFrameRendered = false;
+
+                // PERF: Draw every frame to minimize latency at cost of performance.
+                // When multiple frames are drawn in one vertical sync interval,
+                // only the last one is visible to users.
+                // But this ensures users can always see the most up-to-date screen.
+                // This is also the behavior of official Scrcpy client.
+                // https://github.com/Genymobile/scrcpy/issues/3679
+                this.#updateSize(frame.displayWidth, frame.displayHeight);
+                this.#renderer.draw(frame);
+            },
+            error(e) {
+                if (controller) {
+                    try {
+                        controller.error(e);
+                    } catch {
+                        // ignore
+                        // `controller` may already in error state
+                    }
+                } else {
+                    error = e;
+                }
+            },
+        });
+
+        switch (this.#codec) {
+            case ScrcpyVideoCodecId.H264:
+                this.#codecDecoder = new H264Decoder(
+                    this.#decoder,
+                    this.#updateSize,
+                );
+                break;
+            case ScrcpyVideoCodecId.H265:
+                this.#codecDecoder = new H265Decoder(
+                    this.#decoder,
+                    this.#updateSize,
+                );
+                break;
+            case ScrcpyVideoCodecId.AV1:
+                this.#codecDecoder = new Av1Codec(
+                    this.#decoder,
+                    this.#updateSize,
+                );
+                break;
+        }
+
+        let error: Error | undefined;
+        let controller: WritableStreamDefaultController | undefined;
+        this.#writable = new WritableStream<ScrcpyMediaStreamPacket>({
+            start: (_controller) => {
+                if (error) {
+                    _controller.error(error);
+                } else {
+                    controller = _controller;
+                }
+            },
+            write: (packet) => {
+                this.#codecDecoder.decode(packet);
+            },
+        });
+
+        this.#onFramePresented();
+    }
+
+    #updateSize = (width: number, height: number) => {
+        if (width !== this.#canvas.width || height !== this.#canvas.height) {
+            this.#canvas.width = width;
+            this.#canvas.height = height;
+            this.#sizeChanged.fire({
+                width: width,
+                height: height,
+            });
+        }
+    };
+
+    #onFramePresented = () => {
+        this.#currentFrameRendered = true;
+        this.#animationFrameId = requestAnimationFrame(this.#onFramePresented);
+    };
+
+    dispose() {
+        cancelAnimationFrame(this.#animationFrameId);
+        if (this.#decoder.state !== "closed") {
+            this.#decoder.close();
+        }
+    }
+}

package/src/video/index.ts

@@ -0,0 +1,3 @@
+export * from "./codec/index.js";
+export * from "./decoder.js";
+export * from "./render/index.js";

package/src/video/render/bitmap.ts

@@ -0,0 +1,24 @@
+import type { FrameRenderer } from "./type.js";
+
+export class BitmapFrameRenderer implements FrameRenderer {
+    #context: ImageBitmapRenderingContext;
+
+    constructor(canvas: HTMLCanvasElement) {
+        this.#context = canvas.getContext("bitmaprenderer", { alpha: false })!;
+    }
+
+    draw(frame: VideoFrame): void {
+        createImageBitmap(frame)
+            .then((bitmap) => {
+                this.#context.transferFromImageBitmap(bitmap);
+                frame.close();
+            })
+            .catch((e) => {
+                console.warn(
+                    "[@yume-chan/scrcpy-decoder-webcodecs]",
+                    "VideoDecoder error",
+                    e,
+                );
+            });
+    }
+}

package/src/video/render/index.ts

@@ -0,0 +1,3 @@
+export * from "./bitmap.js";
+export * from "./type.js";
+export * from "./webgl.js";

package/src/video/render/type.ts

@@ -0,0 +1,3 @@
+export interface FrameRenderer {
+    draw(frame: VideoFrame): void;
+}

package/src/video/render/webgl.ts

@@ -0,0 +1,116 @@
+import type { FrameRenderer } from "./type.js";
+
+export class WebGLFrameRenderer implements FrameRenderer {
+    static vertexShaderSource = `
+        attribute vec2 xy;
+
+        varying highp vec2 uv;
+
+        void main(void) {
+            gl_Position = vec4(xy, 0.0, 1.0);
+            // Map vertex coordinates (-1 to +1) to UV coordinates (0 to 1).
+            // UV coordinates are Y-flipped relative to vertex coordinates.
+            uv = vec2((1.0 + xy.x) / 2.0, (1.0 - xy.y) / 2.0);
+        }
+`;
+
+    static fragmentShaderSource = `
+        varying highp vec2 uv;
+
+        uniform sampler2D texture;
+
+        void main(void) {
+            gl_FragColor = texture2D(texture, uv);
+        }
+`;
+
+    #context: WebGLRenderingContext;
+
+    /**
+     * Create a new WebGL frame renderer.
+     * @param canvas The canvas to render frames to.
+     * @param enableCapture
+     * Whether to allow capturing the canvas content using APIs like `readPixels` and `toDataURL`.
+     * Enable this option may reduce performance.
+     */
+    constructor(canvas: HTMLCanvasElement, enableCapture: boolean) {
+        const gl =
+            canvas.getContext("webgl2", {
+                alpha: false,
+                failIfMajorPerformanceCaveat: true,
+                preserveDrawingBuffer: enableCapture,
+            }) ||
+            canvas.getContext("webgl", {
+                alpha: false,
+                failIfMajorPerformanceCaveat: true,
+                preserveDrawingBuffer: enableCapture,
+            });
+        if (!gl) {
+            throw new Error("WebGL not supported");
+        }
+        this.#context = gl;
+
+        const vertexShader = gl.createShader(gl.VERTEX_SHADER)!;
+        gl.shaderSource(vertexShader, WebGLFrameRenderer.vertexShaderSource);
+        gl.compileShader(vertexShader);
+        if (!gl.getShaderParameter(vertexShader, gl.COMPILE_STATUS)) {
+            throw new Error(gl.getShaderInfoLog(vertexShader)!);
+        }
+
+        const fragmentShader = gl.createShader(gl.FRAGMENT_SHADER)!;
+        gl.shaderSource(
+            fragmentShader,
+            WebGLFrameRenderer.fragmentShaderSource,
+        );
+        gl.compileShader(fragmentShader);
+        if (!gl.getShaderParameter(fragmentShader, gl.COMPILE_STATUS)) {
+            throw new Error(gl.getShaderInfoLog(fragmentShader)!);
+        }
+
+        const shaderProgram = gl.createProgram()!;
+        gl.attachShader(shaderProgram, vertexShader);
+        gl.attachShader(shaderProgram, fragmentShader);
+        gl.linkProgram(shaderProgram);
+        if (!gl.getProgramParameter(shaderProgram, gl.LINK_STATUS)) {
+            throw new Error(gl.getProgramInfoLog(shaderProgram)!);
+        }
+        gl.useProgram(shaderProgram);
+
+        // Vertex coordinates, clockwise from bottom-left.
+        const vertexBuffer = gl.createBuffer();
+        gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer);
+        gl.bufferData(
+            gl.ARRAY_BUFFER,
+            new Float32Array([-1.0, -1.0, -1.0, +1.0, +1.0, +1.0, +1.0, -1.0]),
+            gl.STATIC_DRAW,
+        );
+
+        const xyLocation = gl.getAttribLocation(shaderProgram, "xy");
+        gl.vertexAttribPointer(xyLocation, 2, gl.FLOAT, false, 0, 0);
+        gl.enableVertexAttribArray(xyLocation);
+
+        // Create one texture to upload frames to.
+        const texture = gl.createTexture();
+        gl.bindTexture(gl.TEXTURE_2D, texture);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+    }
+
+    draw(frame: VideoFrame) {
+        const gl = this.#context;
+        gl.texImage2D(
+            gl.TEXTURE_2D,
+            0,
+            gl.RGBA,
+            gl.RGBA,
+            gl.UNSIGNED_BYTE,
+            frame,
+        );
+        frame.close();
+
+        gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight);
+        gl.drawArrays(gl.TRIANGLE_FAN, 0, 4);
+    }
+}