@yume-chan/scrcpy-decoder-webcodecs 0.0.0-20240714132542

package/esm/video/decoder.js

@@ -0,0 +1,144 @@
+import { EventEmitter } from "@yume-chan/event";
+import { ScrcpyVideoCodecId } from "@yume-chan/scrcpy";
+import { createCanvas } from "@yume-chan/scrcpy-decoder-tinyh264";
+import { WritableStream } from "@yume-chan/stream-extra";
+import { Av1Codec, H264Decoder, H265Decoder } from "./codec/index.js";
+import { BitmapFrameSink, WebGLFrameSink } from "./render/index.js";
+export class WebCodecsVideoDecoder {
+    static isSupported() {
+        return typeof globalThis.VideoDecoder !== "undefined";
+    }
+    static capabilities = {
+        h264: {},
+        h265: {},
+    };
+    #codec;
+    get codec() {
+        return this.#codec;
+    }
+    #codecDecoder;
+    #writable;
+    get writable() {
+        return this.#writable;
+    }
+    #renderer;
+    get renderer() {
+        return this.#renderer;
+    }
+    #frameRendered = 0;
+    get framesRendered() {
+        return this.#frameRendered;
+    }
+    #frameSkipped = 0;
+    get framesSkipped() {
+        return this.#frameSkipped;
+    }
+    #sizeChanged = new EventEmitter();
+    get sizeChanged() {
+        return this.#sizeChanged.event;
+    }
+    #decoder;
+    #frameSink;
+    #currentFrameRendered = false;
+    #animationFrameId = 0;
+    /**
+     * Create a new WebCodecs video decoder.
+     */
+    constructor({ codec, canvas, enableCapture }) {
+        this.#codec = codec;
+        if (canvas) {
+            this.#renderer = canvas;
+        }
+        else {
+            this.#renderer = createCanvas();
+        }
+        try {
+            this.#frameSink = new WebGLFrameSink(this.#renderer, !!enableCapture);
+        }
+        catch {
+            this.#frameSink = new BitmapFrameSink(this.#renderer);
+        }
+        this.#decoder = new VideoDecoder({
+            output: (frame) => {
+                if (this.#currentFrameRendered) {
+                    this.#frameRendered += 1;
+                }
+                else {
+                    this.#frameSkipped += 1;
+                }
+                this.#currentFrameRendered = false;
+                // PERF: Draw every frame to minimize latency at cost of performance.
+                // When multiple frames are drawn in one vertical sync interval,
+                // only the last one is visible to users.
+                // But this ensures users can always see the most up-to-date screen.
+                // This is also the behavior of official Scrcpy client.
+                // https://github.com/Genymobile/scrcpy/issues/3679
+                this.#updateSize(frame.displayWidth, frame.displayHeight);
+                this.#frameSink.draw(frame);
+            },
+            error(e) {
+                if (controller) {
+                    try {
+                        controller.error(e);
+                    }
+                    catch {
+                        // ignore
+                        // `controller` may already in error state
+                    }
+                }
+                else {
+                    error = e;
+                }
+            },
+        });
+        switch (this.#codec) {
+            case ScrcpyVideoCodecId.H264:
+                this.#codecDecoder = new H264Decoder(this.#decoder, this.#updateSize);
+                break;
+            case ScrcpyVideoCodecId.H265:
+                this.#codecDecoder = new H265Decoder(this.#decoder, this.#updateSize);
+                break;
+            case ScrcpyVideoCodecId.AV1:
+                this.#codecDecoder = new Av1Codec(this.#decoder, this.#updateSize);
+                break;
+        }
+        let error;
+        let controller;
+        this.#writable = new WritableStream({
+            start: (_controller) => {
+                if (error) {
+                    _controller.error(error);
+                }
+                else {
+                    controller = _controller;
+                }
+            },
+            write: (packet) => {
+                this.#codecDecoder.decode(packet);
+            },
+        });
+        this.#onFramePresented();
+    }
+    #updateSize = (width, height) => {
+        if (width !== this.#renderer.width ||
+            height !== this.#renderer.height) {
+            this.#renderer.width = width;
+            this.#renderer.height = height;
+            this.#sizeChanged.fire({
+                width: width,
+                height: height,
+            });
+        }
+    };
+    #onFramePresented = () => {
+        this.#currentFrameRendered = true;
+        this.#animationFrameId = requestAnimationFrame(this.#onFramePresented);
+    };
+    dispose() {
+        cancelAnimationFrame(this.#animationFrameId);
+        if (this.#decoder.state !== "closed") {
+            this.#decoder.close();
+        }
+    }
+}
+//# sourceMappingURL=decoder.js.map

package/esm/video/decoder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"decoder.js","sourceRoot":"","sources":["../../src/video/decoder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAEhD,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAMvD,OAAO,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AAElE,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAGtE,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAepE,MAAM,OAAO,qBAAqB;IAC9B,MAAM,CAAC,WAAW;QACd,OAAO,OAAO,UAAU,CAAC,YAAY,KAAK,WAAW,CAAC;IAC1D,CAAC;IAED,MAAM,CAAU,YAAY,GACxB;QACI,IAAI,EAAE,EAAE;QACR,IAAI,EAAE,EAAE;KACX,CAAC;IAEN,MAAM,CAAqB;IAC3B,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAED,aAAa,CAAe;IAE5B,SAAS,CAA0C;IACnD,IAAI,QAAQ;QACR,OAAO,IAAI,CAAC,SAAS,CAAC;IAC1B,CAAC;IAED,SAAS,CAAsC;IAC/C,IAAI,QAAQ;QACR,OAAO,IAAI,CAAC,SAAS,CAAC;IAC1B,CAAC;IAED,cAAc,GAAG,CAAC,CAAC;IACnB,IAAI,cAAc;QACd,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED,aAAa,GAAG,CAAC,CAAC;IAClB,IAAI,aAAa;QACb,OAAO,IAAI,CAAC,aAAa,CAAC;IAC9B,CAAC;IAED,YAAY,GAAG,IAAI,YAAY,EAAqC,CAAC;IACrE,IAAI,WAAW;QACX,OAAO,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC;IACnC,CAAC;IAED,QAAQ,CAAe;IACvB,UAAU,CAAY;IAEtB,qBAAqB,GAAG,KAAK,CAAC;IAC9B,iBAAiB,GAAG,CAAC,CAAC;IAEtB;;OAEG;IACH,YAAY,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAA6B;QACnE,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QAEpB,IAAI,MAAM,EAAE,CAAC;YACT,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC;QAC5B,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,SAAS,GAAG,YAAY,EAAE,CAAC;QACpC,CAAC;QAED,IAAI,CAAC;YACD,IAAI,CAAC,UAAU,GAAG,IAAI,cAAc,CAChC,IAAI,CAAC,SAAS,EACd,CAAC,CAAC,aAAa,CAClB,CAAC;QACN,CAAC;QAAC,MAAM,CAAC;YACL,IAAI,CAAC,UAAU,GAAG,IAAI,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC1D,CAAC;QAED,IAAI,CAAC,QAAQ,GAAG,IAAI,YAAY,CAAC;YAC7B,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;gBACd,IAAI,IAAI,CAAC,qBAAqB,EAAE,CAAC;oBAC7B,IAAI,CAAC,cAAc,IAAI,CAAC,CAAC;gBAC7B,CAAC;qBAAM,CAAC;oBACJ,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC;gBAC5B,CAAC;gBACD,IAAI,CAAC,qBAAqB,GAAG,KAAK,CAAC;gBAEnC,qEAAqE;gBACrE,gEAAgE;gBAChE,yCAAyC;gBACzC,oEAAoE;gBACpE,uDAAuD;gBACvD,mDAAmD;gBACnD,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC;gBAC1D,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChC,CAAC;YACD,KAAK,CAAC,CAAC;gBACH,IAAI,UAAU,EAAE,CAAC;oBACb,IAAI,CAAC;wBACD,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;oBACxB,CAAC;oBAAC,MAAM,CAAC;wBACL,SAAS;wBACT,0CAA0C;oBAC9C,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACJ,KAAK,GAAG,CAAC,CAAC;gBACd,CAAC;YACL,CAAC;SACJ,CAAC,CAAC;QAEH,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC;YAClB,KAAK,kBAAkB,CAAC,IAAI;gBACxB,IAAI,CAAC,aAAa,GAAG,IAAI,WAAW,CAChC,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,WAAW,CACnB,CAAC;gBACF,MAAM;YACV,KAAK,kBAAkB,CAAC,IAAI;gBACxB,IAAI,CAAC,aAAa,GAAG,IAAI,WAAW,CAChC,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,WAAW,CACnB,CAAC;gBACF,MAAM;YACV,KAAK,kBAAkB,CAAC,GAAG;gBACvB,IAAI,CAAC,aAAa,GAAG,IAAI,QAAQ,CAC7B,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,WAAW,CACnB,CAAC;gBACF,MAAM;QACd,CAAC;QAED,IAAI,KAAwB,CAAC;QAC7B,IAAI,UAAuD,CAAC;QAC5D,IAAI,CAAC,SAAS,GAAG,IAAI,cAAc,CAA0B;YACzD,KAAK,EAAE,CAAC,WAAW,EAAE,EAAE;gBACnB,IAAI,KAAK,EAAE,CAAC;oBACR,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC7B,CAAC;qBAAM,CAAC;oBACJ,UAAU,GAAG,WAAW,CAAC;gBAC7B,CAAC;YACL,CAAC;YACD,KAAK,EAAE,CAAC,MAAM,EAAE,EAAE;gBACd,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACtC,CAAC;SACJ,CAAC,CAAC;QAEH,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC7B,CAAC;IAED,WAAW,GAAG,CAAC,KAAa,EAAE,MAAc,EAAE,EAAE;QAC5C,IACI,KAAK,KAAK,IAAI,CAAC,SAAS,CAAC,KAAK;YAC9B,MAAM,KAAK,IAAI,CAAC,SAAS,CAAC,MAAM,EAClC,CAAC;YACC,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG,KAAK,CAAC;YAC7B,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,MAAM,CAAC;YAC/B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC;gBACnB,KAAK,EAAE,KAAK;gBACZ,MAAM,EAAE,MAAM;aACjB,CAAC,CAAC;QACP,CAAC;IACL,CAAC,CAAC;IAEF,iBAAiB,GAAG,GAAG,EAAE;QACrB,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;QAClC,IAAI,CAAC,iBAAiB,GAAG,qBAAqB,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC3E,CAAC,CAAC;IAEF,OAAO;QACH,oBAAoB,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC7C,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YACnC,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QAC1B,CAAC;IACL,CAAC"}

package/esm/video/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./codec/index.js";
+export * from "./decoder.js";
+export * from "./render/index.js";
+//# sourceMappingURL=index.d.ts.map

package/esm/video/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/video/index.ts"],"names":[],"mappings":"AAAA,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,mBAAmB,CAAC"}

package/esm/video/index.js

@@ -0,0 +1,4 @@
+export * from "./codec/index.js";
+export * from "./decoder.js";
+export * from "./render/index.js";
+//# sourceMappingURL=index.js.map

package/esm/video/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/video/index.ts"],"names":[],"mappings":"AAAA,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,mBAAmB,CAAC"}

package/esm/video/render/bitmap.d.ts

@@ -0,0 +1,7 @@
+import type { FrameSink } from "./type.js";
+export declare class BitmapFrameSink implements FrameSink {
+    #private;
+    constructor(canvas: HTMLCanvasElement | OffscreenCanvas);
+    draw(frame: VideoFrame): void;
+}
+//# sourceMappingURL=bitmap.d.ts.map

package/esm/video/render/bitmap.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bitmap.d.ts","sourceRoot":"","sources":["../../../src/video/render/bitmap.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAE3C,qBAAa,eAAgB,YAAW,SAAS;;gBAGjC,MAAM,EAAE,iBAAiB,GAAG,eAAe;IAIvD,IAAI,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;CAchC"}

package/esm/video/render/bitmap.js

@@ -0,0 +1,17 @@
+export class BitmapFrameSink {
+    #context;
+    constructor(canvas) {
+        this.#context = canvas.getContext("bitmaprenderer", { alpha: false });
+    }
+    draw(frame) {
+        createImageBitmap(frame)
+            .then((bitmap) => {
+            this.#context.transferFromImageBitmap(bitmap);
+            frame.close();
+        })
+            .catch((e) => {
+            console.warn("[@yume-chan/scrcpy-decoder-webcodecs]", "VideoDecoder error", e);
+        });
+    }
+}
+//# sourceMappingURL=bitmap.js.map

package/esm/video/render/bitmap.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bitmap.js","sourceRoot":"","sources":["../../../src/video/render/bitmap.ts"],"names":[],"mappings":"AAEA,MAAM,OAAO,eAAe;IACxB,QAAQ,CAA8B;IAEtC,YAAY,MAA2C;QACnD,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,UAAU,CAAC,gBAAgB,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAE,CAAC;IAC3E,CAAC;IAED,IAAI,CAAC,KAAiB;QAClB,iBAAiB,CAAC,KAAK,CAAC;aACnB,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;YACb,IAAI,CAAC,QAAQ,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC;YAC9C,KAAK,CAAC,KAAK,EAAE,CAAC;QAClB,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACT,OAAO,CAAC,IAAI,CACR,uCAAuC,EACvC,oBAAoB,EACpB,CAAC,CACJ,CAAC;QACN,CAAC,CAAC,CAAC;IACX,CAAC;CACJ"}

package/esm/video/render/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./bitmap.js";
+export * from "./type.js";
+export * from "./webgl.js";
+//# sourceMappingURL=index.d.ts.map

package/esm/video/render/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/video/render/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,WAAW,CAAC;AAC1B,cAAc,YAAY,CAAC"}

package/esm/video/render/index.js

@@ -0,0 +1,4 @@
+export * from "./bitmap.js";
+export * from "./type.js";
+export * from "./webgl.js";
+//# sourceMappingURL=index.js.map

package/esm/video/render/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/video/render/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,WAAW,CAAC;AAC1B,cAAc,YAAY,CAAC"}

package/esm/video/render/type.d.ts

@@ -0,0 +1,4 @@
+export interface FrameSink {
+    draw(frame: VideoFrame): void;
+}
+//# sourceMappingURL=type.d.ts.map

package/esm/video/render/type.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"type.d.ts","sourceRoot":"","sources":["../../../src/video/render/type.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,SAAS;IACtB,IAAI,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI,CAAC;CACjC"}

package/esm/video/render/type.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=type.js.map

package/esm/video/render/type.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"type.js","sourceRoot":"","sources":["../../../src/video/render/type.ts"],"names":[],"mappings":""}

package/esm/video/render/webgl.d.ts

@@ -0,0 +1,16 @@
+import type { FrameSink } from "./type.js";
+export declare class WebGLFrameSink implements FrameSink {
+    #private;
+    static vertexShaderSource: string;
+    static fragmentShaderSource: string;
+    /**
+     * Create a new WebGL frame renderer.
+     * @param canvas The canvas to render frames to.
+     * @param enableCapture
+     * Whether to allow capturing the canvas content using APIs like `readPixels` and `toDataURL`.
+     * Enable this option may reduce performance.
+     */
+    constructor(canvas: HTMLCanvasElement | OffscreenCanvas, enableCapture: boolean);
+    draw(frame: VideoFrame): void;
+}
+//# sourceMappingURL=webgl.d.ts.map

package/esm/video/render/webgl.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"webgl.d.ts","sourceRoot":"","sources":["../../../src/video/render/webgl.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAE3C,qBAAa,cAAe,YAAW,SAAS;;IAC5C,MAAM,CAAC,kBAAkB,SAW3B;IAEE,MAAM,CAAC,oBAAoB,SAQ7B;IAIE;;;;;;OAMG;gBAEC,MAAM,EAAE,iBAAiB,GAAG,eAAe,EAC3C,aAAa,EAAE,OAAO;IAiE1B,IAAI,CAAC,KAAK,EAAE,UAAU;CAezB"}

package/esm/video/render/webgl.js

@@ -0,0 +1,90 @@
+export class WebGLFrameSink {
+    static vertexShaderSource = `
+        attribute vec2 xy;
+
+        varying highp vec2 uv;
+
+        void main(void) {
+            gl_Position = vec4(xy, 0.0, 1.0);
+            // Map vertex coordinates (-1 to +1) to UV coordinates (0 to 1).
+            // UV coordinates are Y-flipped relative to vertex coordinates.
+            uv = vec2((1.0 + xy.x) / 2.0, (1.0 - xy.y) / 2.0);
+        }
+`;
+    static fragmentShaderSource = `
+        varying highp vec2 uv;
+
+        uniform sampler2D texture;
+
+        void main(void) {
+            gl_FragColor = texture2D(texture, uv);
+        }
+`;
+    #context;
+    /**
+     * Create a new WebGL frame renderer.
+     * @param canvas The canvas to render frames to.
+     * @param enableCapture
+     * Whether to allow capturing the canvas content using APIs like `readPixels` and `toDataURL`.
+     * Enable this option may reduce performance.
+     */
+    constructor(canvas, enableCapture) {
+        const attributes = {
+            // Low-power GPU should be enough for video rendering.
+            powerPreference: "low-power",
+            alpha: false,
+            // Disallow software rendering.
+            // Other rendering methods are faster than software-based WebGL.
+            failIfMajorPerformanceCaveat: true,
+            preserveDrawingBuffer: enableCapture,
+        };
+        const gl = canvas.getContext("webgl2", attributes) ||
+            canvas.getContext("webgl", attributes);
+        if (!gl) {
+            throw new Error("WebGL not supported");
+        }
+        this.#context = gl;
+        const vertexShader = gl.createShader(gl.VERTEX_SHADER);
+        gl.shaderSource(vertexShader, WebGLFrameSink.vertexShaderSource);
+        gl.compileShader(vertexShader);
+        if (!gl.getShaderParameter(vertexShader, gl.COMPILE_STATUS)) {
+            throw new Error(gl.getShaderInfoLog(vertexShader));
+        }
+        const fragmentShader = gl.createShader(gl.FRAGMENT_SHADER);
+        gl.shaderSource(fragmentShader, WebGLFrameSink.fragmentShaderSource);
+        gl.compileShader(fragmentShader);
+        if (!gl.getShaderParameter(fragmentShader, gl.COMPILE_STATUS)) {
+            throw new Error(gl.getShaderInfoLog(fragmentShader));
+        }
+        const shaderProgram = gl.createProgram();
+        gl.attachShader(shaderProgram, vertexShader);
+        gl.attachShader(shaderProgram, fragmentShader);
+        gl.linkProgram(shaderProgram);
+        if (!gl.getProgramParameter(shaderProgram, gl.LINK_STATUS)) {
+            throw new Error(gl.getProgramInfoLog(shaderProgram));
+        }
+        gl.useProgram(shaderProgram);
+        // Vertex coordinates, clockwise from bottom-left.
+        const vertexBuffer = gl.createBuffer();
+        gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer);
+        gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1.0, -1.0, -1.0, +1.0, +1.0, +1.0, +1.0, -1.0]), gl.STATIC_DRAW);
+        const xyLocation = gl.getAttribLocation(shaderProgram, "xy");
+        gl.vertexAttribPointer(xyLocation, 2, gl.FLOAT, false, 0, 0);
+        gl.enableVertexAttribArray(xyLocation);
+        // Create one texture to upload frames to.
+        const texture = gl.createTexture();
+        gl.bindTexture(gl.TEXTURE_2D, texture);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+    }
+    draw(frame) {
+        const gl = this.#context;
+        gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, frame);
+        frame.close();
+        gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight);
+        gl.drawArrays(gl.TRIANGLE_FAN, 0, 4);
+    }
+}
+//# sourceMappingURL=webgl.js.map

package/esm/video/render/webgl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"webgl.js","sourceRoot":"","sources":["../../../src/video/render/webgl.ts"],"names":[],"mappings":"AAEA,MAAM,OAAO,cAAc;IACvB,MAAM,CAAC,kBAAkB,GAAG;;;;;;;;;;;CAW/B,CAAC;IAEE,MAAM,CAAC,oBAAoB,GAAG;;;;;;;;CAQjC,CAAC;IAEE,QAAQ,CAAwB;IAEhC;;;;;;OAMG;IACH,YACI,MAA2C,EAC3C,aAAsB;QAEtB,MAAM,UAAU,GAA2B;YACvC,sDAAsD;YACtD,eAAe,EAAE,WAAW;YAC5B,KAAK,EAAE,KAAK;YACZ,+BAA+B;YAC/B,gEAAgE;YAChE,4BAA4B,EAAE,IAAI;YAClC,qBAAqB,EAAE,aAAa;SACvC,CAAC;QAEF,MAAM,EAAE,GACJ,MAAM,CAAC,UAAU,CAAC,QAAQ,EAAE,UAAU,CAAC;YACvC,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QAC3C,IAAI,CAAC,EAAE,EAAE,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;QAC3C,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;QAEnB,MAAM,YAAY,GAAG,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC,aAAa,CAAE,CAAC;QACxD,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,cAAc,CAAC,kBAAkB,CAAC,CAAC;QACjE,EAAE,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC;QAC/B,IAAI,CAAC,EAAE,CAAC,kBAAkB,CAAC,YAAY,EAAE,EAAE,CAAC,cAAc,CAAC,EAAE,CAAC;YAC1D,MAAM,IAAI,KAAK,CAAC,EAAE,CAAC,gBAAgB,CAAC,YAAY,CAAE,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,cAAc,GAAG,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC,eAAe,CAAE,CAAC;QAC5D,EAAE,CAAC,YAAY,CAAC,cAAc,EAAE,cAAc,CAAC,oBAAoB,CAAC,CAAC;QACrE,EAAE,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC;QACjC,IAAI,CAAC,EAAE,CAAC,kBAAkB,CAAC,cAAc,EAAE,EAAE,CAAC,cAAc,CAAC,EAAE,CAAC;YAC5D,MAAM,IAAI,KAAK,CAAC,EAAE,CAAC,gBAAgB,CAAC,cAAc,CAAE,CAAC,CAAC;QAC1D,CAAC;QAED,MAAM,aAAa,GAAG,EAAE,CAAC,aAAa,EAAG,CAAC;QAC1C,EAAE,CAAC,YAAY,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;QAC7C,EAAE,CAAC,YAAY,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC;QAC/C,EAAE,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;QAC9B,IAAI,CAAC,EAAE,CAAC,mBAAmB,CAAC,aAAa,EAAE,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC;YACzD,MAAM,IAAI,KAAK,CAAC,EAAE,CAAC,iBAAiB,CAAC,aAAa,CAAE,CAAC,CAAC;QAC1D,CAAC;QACD,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAE7B,kDAAkD;QAClD,MAAM,YAAY,GAAG,EAAE,CAAC,YAAY,EAAE,CAAC;QACvC,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;QAC7C,EAAE,CAAC,UAAU,CACT,EAAE,CAAC,YAAY,EACf,IAAI,YAAY,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,EAClE,EAAE,CAAC,WAAW,CACjB,CAAC;QAEF,MAAM,UAAU,GAAG,EAAE,CAAC,iBAAiB,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;QAC7D,EAAE,CAAC,mBAAmB,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7D,EAAE,CAAC,uBAAuB,CAAC,UAAU,CAAC,CAAC;QAEvC,0CAA0C;QAC1C,MAAM,OAAO,GAAG,EAAE,CAAC,aAAa,EAAE,CAAC;QACnC,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACvC,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,kBAAkB,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC;QACnE,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,kBAAkB,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC;QACnE,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,cAAc,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC;QACrE,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,cAAc,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC;IACzE,CAAC;IAED,IAAI,CAAC,KAAiB;QAClB,MAAM,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC;QACzB,EAAE,CAAC,UAAU,CACT,EAAE,CAAC,UAAU,EACb,CAAC,EACD,EAAE,CAAC,IAAI,EACP,EAAE,CAAC,IAAI,EACP,EAAE,CAAC,aAAa,EAChB,KAAK,CACR,CAAC;QACF,KAAK,CAAC,KAAK,EAAE,CAAC;QAEd,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,EAAE,EAAE,CAAC,mBAAmB,CAAC,CAAC;QACjE,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IACzC,CAAC"}

package/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "@yume-chan/scrcpy-decoder-webcodecs",
+  "version": "0.0.0-20240714132542",
+  "description": "Raw H.264 stream decoder and renderer using WebCodecs API (requires modern browser).",
+  "keywords": [
+    "adb",
+    "android-phone",
+    "scrcpy",
+    "scrcpy-decoder"
+  ],
+  "license": "MIT",
+  "author": {
+    "name": "Simon Chan",
+    "email": "cnsimonchan@live.com",
+    "url": "https://chensi.moe/blog"
+  },
+  "homepage": "https://github.com/yume-chan/ya-webadb/tree/main/packages/scrcpy-decoder-webcodecs#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/yume-chan/ya-webadb.git",
+    "directory": "packages/scrcpy-decoder-webcodecs"
+  },
+  "bugs": {
+    "url": "https://github.com/yume-chan/ya-webadb/issues"
+  },
+  "type": "module",
+  "main": "esm/index.js",
+  "types": "esm/index.d.ts",
+  "sideEffects": false,
+  "dependencies": {
+    "@yume-chan/event": "0.0.0-20240714132542",
+    "@yume-chan/no-data-view": "0.0.0-20240714132542",
+    "@yume-chan/scrcpy": "0.0.0-20240714132542",
+    "@yume-chan/scrcpy-decoder-tinyh264": "0.0.0-20240714132542",
+    "@yume-chan/stream-extra": "0.0.0-20240714132542"
+  },
+  "devDependencies": {
+    "@jest/globals": "^30.0.0-alpha.4",
+    "@yume-chan/eslint-config": "^1.0.0",
+    "@yume-chan/tsconfig": "^1.0.0",
+    "cross-env": "^7.0.3",
+    "jest": "^30.0.0-alpha.4",
+    "prettier": "^3.3.3",
+    "ts-jest": "^29.2.2",
+    "typescript": "^5.5.3"
+  },
+  "scripts": {
+    "build": "tsc -b tsconfig.build.json",
+    "build:watch": "tsc -b tsconfig.build.json",
+    "lint": "run-eslint && prettier src/**/*.ts --write --tab-width 4"
+  }
+}

package/src/index.ts

@@ -0,0 +1 @@
+export * from "./video/index.js";

package/src/video/codec/av1.ts

@@ -0,0 +1,101 @@
+import type { ScrcpyMediaStreamPacket } from "@yume-chan/scrcpy";
+import { Av1 } from "@yume-chan/scrcpy";
+
+import type { CodecDecoder } from "./type.js";
+import { decimalTwoDigits } from "./utils.js";
+
+export class Av1Codec implements CodecDecoder {
+    #decoder: VideoDecoder;
+    #updateSize: (width: number, height: number) => void;
+
+    constructor(
+        decoder: VideoDecoder,
+        updateSize: (width: number, height: number) => void,
+    ) {
+        this.#decoder = decoder;
+        this.#updateSize = updateSize;
+    }
+
+    #configure(data: Uint8Array) {
+        const parser = new Av1(data);
+        const sequenceHeader = parser.searchSequenceHeaderObu();
+
+        if (!sequenceHeader) {
+            return;
+        }
+
+        const {
+            seq_profile: seqProfile,
+            seq_level_idx: [seqLevelIdx = 0],
+            max_frame_width_minus_1,
+            max_frame_height_minus_1,
+            color_config: {
+                BitDepth,
+                mono_chrome: monoChrome,
+                subsampling_x: subsamplingX,
+                subsampling_y: subsamplingY,
+                chroma_sample_position: chromaSamplePosition,
+                color_description_present_flag,
+            },
+        } = sequenceHeader;
+
+        let colorPrimaries: Av1.ColorPrimaries;
+        let transferCharacteristics: Av1.TransferCharacteristics;
+        let matrixCoefficients: Av1.MatrixCoefficients;
+        let colorRange: boolean;
+        if (color_description_present_flag) {
+            ({
+                color_primaries: colorPrimaries,
+                transfer_characteristics: transferCharacteristics,
+                matrix_coefficients: matrixCoefficients,
+                color_range: colorRange,
+            } = sequenceHeader.color_config);
+        } else {
+            colorPrimaries = Av1.ColorPrimaries.Bt709;
+            transferCharacteristics = Av1.TransferCharacteristics.Bt709;
+            matrixCoefficients = Av1.MatrixCoefficients.Bt709;
+            colorRange = false;
+        }
+
+        const width = max_frame_width_minus_1 + 1;
+        const height = max_frame_height_minus_1 + 1;
+
+        this.#updateSize(width, height);
+
+        const codec = [
+            "av01",
+            seqProfile.toString(16),
+            decimalTwoDigits(seqLevelIdx) +
+                (sequenceHeader.seq_tier[0] ? "H" : "M"),
+            decimalTwoDigits(BitDepth),
+            monoChrome ? "1" : "0",
+            (subsamplingX ? "1" : "0") +
+                (subsamplingY ? "1" : "0") +
+                chromaSamplePosition.toString(),
+            decimalTwoDigits(colorPrimaries),
+            decimalTwoDigits(transferCharacteristics),
+            decimalTwoDigits(matrixCoefficients),
+            colorRange ? "1" : "0",
+        ].join(".");
+        this.#decoder.configure({
+            codec,
+            optimizeForLatency: true,
+        });
+    }
+
+    decode(packet: ScrcpyMediaStreamPacket): void {
+        if (packet.type === "configuration") {
+            return;
+        }
+
+        this.#configure(packet.data);
+        this.#decoder.decode(
+            new EncodedVideoChunk({
+                // Treat `undefined` as `key`, otherwise it won't decode.
+                type: packet.keyframe === false ? "delta" : "key",
+                timestamp: 0,
+                data: packet.data,
+            }),
+        );
+    }
+}

package/src/video/codec/h264.ts

@@ -0,0 +1,42 @@
+import { h264ParseConfiguration } from "@yume-chan/scrcpy";
+
+import { H26xDecoder } from "./h26x.js";
+import { hexTwoDigits } from "./utils.js";
+
+export class H264Decoder extends H26xDecoder {
+    #decoder: VideoDecoder;
+    #updateSize: (width: number, height: number) => void;
+
+    constructor(
+        decoder: VideoDecoder,
+        updateSize: (width: number, height: number) => void,
+    ) {
+        super(decoder);
+        this.#decoder = decoder;
+        this.#updateSize = updateSize;
+    }
+
+    override configure(data: Uint8Array): void {
+        const {
+            profileIndex,
+            constraintSet,
+            levelIndex,
+            croppedWidth,
+            croppedHeight,
+        } = h264ParseConfiguration(data);
+
+        this.#updateSize(croppedWidth, croppedHeight);
+
+        // https://www.rfc-editor.org/rfc/rfc6381#section-3.3
+        // ISO Base Media File Format Name Space
+        const codec =
+            "avc1." +
+            hexTwoDigits(profileIndex) +
+            hexTwoDigits(constraintSet) +
+            hexTwoDigits(levelIndex);
+        this.#decoder.configure({
+            codec: codec,
+            optimizeForLatency: true,
+        });
+    }
+}

package/src/video/codec/h265.ts

@@ -0,0 +1,47 @@
+import { getUint32LittleEndian } from "@yume-chan/no-data-view";
+import { h265ParseConfiguration } from "@yume-chan/scrcpy";
+
+import { H26xDecoder } from "./h26x.js";
+import { hexDigits } from "./utils.js";
+
+export class H265Decoder extends H26xDecoder {
+    #decoder: VideoDecoder;
+    #updateSize: (width: number, height: number) => void;
+
+    constructor(
+        decoder: VideoDecoder,
+        updateSize: (width: number, height: number) => void,
+    ) {
+        super(decoder);
+        this.#decoder = decoder;
+        this.#updateSize = updateSize;
+    }
+
+    override configure(data: Uint8Array): void {
+        const {
+            generalProfileSpace,
+            generalProfileIndex,
+            generalProfileCompatibilitySet,
+            generalTierFlag,
+            generalLevelIndex,
+            generalConstraintSet,
+            croppedWidth,
+            croppedHeight,
+        } = h265ParseConfiguration(data);
+
+        this.#updateSize(croppedWidth, croppedHeight);
+
+        const codec = [
+            "hev1",
+            ["", "A", "B", "C"][generalProfileSpace]! +
+                generalProfileIndex.toString(),
+            hexDigits(getUint32LittleEndian(generalProfileCompatibilitySet, 0)),
+            (generalTierFlag ? "H" : "L") + generalLevelIndex.toString(),
+            ...Array.from(generalConstraintSet, hexDigits),
+        ].join(".");
+        this.#decoder.configure({
+            codec,
+            optimizeForLatency: true,
+        });
+    }
+}

package/src/video/codec/h26x.ts

@@ -0,0 +1,45 @@
+import type { ScrcpyMediaStreamPacket } from "@yume-chan/scrcpy";
+
+import type { CodecDecoder } from "./type.js";
+
+export abstract class H26xDecoder implements CodecDecoder {
+    #config: Uint8Array | undefined;
+    #decoder: VideoDecoder;
+
+    constructor(decoder: VideoDecoder) {
+        this.#decoder = decoder;
+    }
+
+    abstract configure(data: Uint8Array): void;
+
+    decode(packet: ScrcpyMediaStreamPacket): void {
+        if (packet.type === "configuration") {
+            this.#config = packet.data;
+            this.configure(packet.data);
+            return;
+        }
+
+        // For H.264 and H.265, when the stream is in Annex B format
+        // (which Scrcpy uses, as Android MediaCodec produces),
+        // configuration data needs to be combined with the first frame data.
+        // https://www.w3.org/TR/webcodecs-avc-codec-registration/#encodedvideochunk-type
+        let data: Uint8Array;
+        if (this.#config !== undefined) {
+            data = new Uint8Array(this.#config.length + packet.data.length);
+            data.set(this.#config, 0);
+            data.set(packet.data, this.#config.length);
+            this.#config = undefined;
+        } else {
+            data = packet.data;
+        }
+
+        this.#decoder.decode(
+            new EncodedVideoChunk({
+                // Treat `undefined` as `key`, otherwise won't decode.
+                type: packet.keyframe === false ? "delta" : "key",
+                timestamp: 0,
+                data,
+            }),
+        );
+    }
+}