@yume-chan/scrcpy-decoder-webcodecs 2.1.0 → 2.5.3

package/src/video/codec/h264.ts

@@ -1,22 +1,24 @@
 import { h264ParseConfiguration } from "@yume-chan/scrcpy";
 
 import { H26xDecoder } from "./h26x.js";
+import type { CodecDecoderOptions } from "./type.js";
 import { hexTwoDigits } from "./utils.js";
 
 export class H264Decoder extends H26xDecoder {
-    #decoder: VideoDecoder;
     #updateSize: (width: number, height: number) => void;
+    #options: CodecDecoderOptions | undefined;
 
     constructor(
         decoder: VideoDecoder,
         updateSize: (width: number, height: number) => void,
+        options?: CodecDecoderOptions,
     ) {
         super(decoder);
-        this.#decoder = decoder;
         this.#updateSize = updateSize;
+        this.#options = options;
     }
 
-    override configure(data: Uint8Array): void {
+    override configure(data: Uint8Array): VideoDecoderConfig {
         const {
             profileIndex,
             constraintSet,
@@ -34,9 +36,11 @@ export class H264Decoder extends H26xDecoder {
             hexTwoDigits(profileIndex) +
             hexTwoDigits(constraintSet) +
             hexTwoDigits(levelIndex);
-        this.#decoder.configure({
+        return {
             codec: codec,
+            hardwareAcceleration:
+                this.#options?.hardwareAcceleration ?? "no-preference",
             optimizeForLatency: true,
-        });
+        };
     }
 }

package/src/video/codec/h265.ts

@@ -2,22 +2,24 @@ import { getUint32LittleEndian } from "@yume-chan/no-data-view";
 import { h265ParseConfiguration } from "@yume-chan/scrcpy";
 
 import { H26xDecoder } from "./h26x.js";
+import type { CodecDecoderOptions } from "./type.js";
 import { hexDigits } from "./utils.js";
 
 export class H265Decoder extends H26xDecoder {
-    #decoder: VideoDecoder;
     #updateSize: (width: number, height: number) => void;
+    #options: CodecDecoderOptions | undefined;
 
     constructor(
         decoder: VideoDecoder,
         updateSize: (width: number, height: number) => void,
+        options?: CodecDecoderOptions,
     ) {
         super(decoder);
-        this.#decoder = decoder;
         this.#updateSize = updateSize;
+        this.#options = options;
     }
 
-    override configure(data: Uint8Array): void {
+    override configure(data: Uint8Array): VideoDecoderConfig {
         const {
             generalProfileSpace,
             generalProfileIndex,
@@ -39,12 +41,14 @@ export class H265Decoder extends H26xDecoder {
             (generalTierFlag ? "H" : "L") + generalLevelIndex.toString(),
             ...Array.from(generalConstraintSet, hexDigits),
         ].join(".");
-        this.#decoder.configure({
+        return {
             codec,
             // Microsoft Edge requires explicit size to work
             codedWidth: croppedWidth,
             codedHeight: croppedHeight,
+            hardwareAcceleration:
+                this.#options?.hardwareAcceleration ?? "no-preference",
             optimizeForLatency: true,
-        });
+        };
     }
 }

package/src/video/codec/h26x.ts

@@ -1,36 +1,92 @@
-import type { ScrcpyMediaStreamPacket } from "@yume-chan/scrcpy";
+import type {
+    ScrcpyMediaStreamDataPacket,
+    ScrcpyMediaStreamPacket,
+} from "@yume-chan/scrcpy";
 
 import type { CodecDecoder } from "./type.js";
 
 export abstract class H26xDecoder implements CodecDecoder {
-    #config: Uint8Array | undefined;
     #decoder: VideoDecoder;
 
+    #config: (VideoDecoderConfig & { raw: Uint8Array }) | undefined;
+    #configured = false;
+
     constructor(decoder: VideoDecoder) {
         this.#decoder = decoder;
     }
 
-    abstract configure(data: Uint8Array): void;
+    abstract configure(data: Uint8Array): VideoDecoderConfig;
 
-    decode(packet: ScrcpyMediaStreamPacket): void {
-        if (packet.type === "configuration") {
-            this.#config = packet.data;
-            this.configure(packet.data);
-            return;
-        }
+    #configureAndDecodeFirstKeyframe(
+        config: VideoDecoderConfig & { raw: Uint8Array },
+        packet: ScrcpyMediaStreamDataPacket,
+    ) {
+        this.#decoder.configure(config);
+        this.#configured = true;
 
         // For H.264 and H.265, when the stream is in Annex B format
         // (which Scrcpy uses, as Android MediaCodec produces),
         // configuration data needs to be combined with the first frame data.
         // https://www.w3.org/TR/webcodecs-avc-codec-registration/#encodedvideochunk-type
-        let data: Uint8Array;
-        if (this.#config !== undefined) {
-            data = new Uint8Array(this.#config.length + packet.data.length);
-            data.set(this.#config, 0);
-            data.set(packet.data, this.#config.length);
-            this.#config = undefined;
-        } else {
-            data = packet.data;
+        const { raw } = config;
+        const data = new Uint8Array(raw.length + packet.data.length);
+        data.set(raw, 0);
+        data.set(packet.data, raw.length);
+
+        this.#decoder.decode(
+            new EncodedVideoChunk({
+                type: "key",
+                timestamp: 0,
+                data,
+            }),
+        );
+    }
+
+    decode(packet: ScrcpyMediaStreamPacket): void {
+        if (packet.type === "configuration") {
+            this.#config = {
+                ...this.configure(packet.data),
+                raw: packet.data,
+            };
+            this.#configured = false;
+            return;
+        }
+
+        if (!this.#config) {
+            throw new Error("Decoder not configured");
+        }
+
+        if (packet.keyframe) {
+            if (this.#decoder.decodeQueueSize) {
+                // If the device is too slow to decode all frames,
+                // discard queued frames when next keyframe arrives.
+                // (can only do this for keyframes because decoding must start from a keyframe)
+                // This limits the maximum latency to 1 keyframe interval
+                // (60 frames by default).
+                this.#decoder.reset();
+
+                // `reset` also resets the decoder configuration
+                // so we need to re-configure it again.
+                this.#configureAndDecodeFirstKeyframe(this.#config, packet);
+                return;
+            }
+
+            if (!this.#configured) {
+                this.#configureAndDecodeFirstKeyframe(this.#config, packet);
+                return;
+            }
+        }
+
+        if (!this.#configured) {
+            if (packet.keyframe === undefined) {
+                // Scrcpy <1.23 doesn't send `keyframe` flag
+                // Infer the first frame after configuration as keyframe
+                // (`VideoDecoder` will throw error if it's not)
+                this.#configureAndDecodeFirstKeyframe(this.#config, packet);
+                return;
+            }
+
+            throw new Error("Expect a keyframe but got a delta frame");
         }
 
         this.#decoder.decode(
@@ -38,7 +94,7 @@ export abstract class H26xDecoder implements CodecDecoder {
                 // Treat `undefined` as `key`, otherwise won't decode.
                 type: packet.keyframe === false ? "delta" : "key",
                 timestamp: 0,
-                data,
+                data: packet.data,
             }),
         );
     }

package/src/video/codec/type.ts

@@ -4,9 +4,14 @@ export interface CodecDecoder {
     decode(packet: ScrcpyMediaStreamPacket): void;
 }
 
+export interface CodecDecoderOptions {
+    hardwareAcceleration?: HardwareAcceleration | undefined;
+}
+
 export interface CodecDecoderConstructor {
     new (
         decoder: VideoDecoder,
         updateSize: (width: number, height: number) => void,
+        options?: CodecDecoderOptions,
     ): CodecDecoder;
 }

package/src/video/decoder.ts

@@ -9,7 +9,7 @@ import type { WritableStreamDefaultController } from "@yume-chan/stream-extra";
 import { WritableStream } from "@yume-chan/stream-extra";
 
 import { Av1Codec, H264Decoder, H265Decoder } from "./codec/index.js";
-import type { CodecDecoder } from "./codec/type.js";
+import type { CodecDecoder, CodecDecoderOptions } from "./codec/type.js";
 import { Pool } from "./pool.js";
 import type { VideoFrameRenderer } from "./render/index.js";
 import { VideoFrameCapturer } from "./snapshot.js";
@@ -35,6 +35,13 @@ export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
         return this.#codec;
     }
 
+    #renderer: VideoFrameRenderer;
+    get renderer() {
+        return this.#renderer;
+    }
+
+    #options: CodecDecoderOptions;
+
     #codecDecoder: CodecDecoder;
 
     #writable: WritableStream<ScrcpyMediaStreamPacket>;
@@ -45,11 +52,6 @@ export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
     #error: Error | undefined;
     #controller!: WritableStreamDefaultController;
 
-    #renderer: VideoFrameRenderer;
-    get renderer() {
-        return this.#renderer;
-    }
-
     #framesDraw = 0;
     #framesPresented = 0;
     get framesRendered() {
@@ -87,10 +89,14 @@ export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
     /**
      * Create a new WebCodecs video decoder.
      */
-    constructor({ codec, renderer }: WebCodecsVideoDecoder.Options) {
+    constructor({
+        codec,
+        renderer,
+        ...options
+    }: WebCodecsVideoDecoder.Options) {
         this.#codec = codec;
-
         this.#renderer = renderer;
+        this.#options = options;
 
         this.#decoder = new VideoDecoder({
             output: (frame) => {
@@ -119,18 +125,21 @@ export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
                 this.#codecDecoder = new H264Decoder(
                     this.#decoder,
                     this.#updateSize,
+                    this.#options,
                 );
                 break;
             case ScrcpyVideoCodecId.H265:
                 this.#codecDecoder = new H265Decoder(
                     this.#decoder,
                     this.#updateSize,
+                    this.#options,
                 );
                 break;
             case ScrcpyVideoCodecId.AV1:
                 this.#codecDecoder = new Av1Codec(
                     this.#decoder,
                     this.#updateSize,
+                    this.#options,
                 );
                 break;
             default:
@@ -232,7 +241,7 @@ export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
 }
 
 export namespace WebCodecsVideoDecoder {
-    export interface Options {
+    export interface Options extends CodecDecoderOptions {
         /**
          * The video codec to decode
          */