@yume-chan/scrcpy-decoder-webcodecs 2.5.3 → 3.0.0-beta.0

package/src/video/codec/h264.ts

@@ -1,46 +1,24 @@
-import { h264ParseConfiguration } from "@yume-chan/scrcpy";
+import { H264 } from "@yume-chan/media-codec";
 
-import { H26xDecoder } from "./h26x.js";
-import type { CodecDecoderOptions } from "./type.js";
-import { hexTwoDigits } from "./utils.js";
+import { H26xTransformStream } from "./h26x.js";
 
-export class H264Decoder extends H26xDecoder {
-    #updateSize: (width: number, height: number) => void;
-    #options: CodecDecoderOptions | undefined;
-
-    constructor(
-        decoder: VideoDecoder,
-        updateSize: (width: number, height: number) => void,
-        options?: CodecDecoderOptions,
-    ) {
-        super(decoder);
-        this.#updateSize = updateSize;
-        this.#options = options;
+export class H264TransformStream extends H26xTransformStream {
+    override convertFrameType(
+        keyframe: boolean | undefined,
+        data: Uint8Array,
+    ): EncodedVideoChunkType {
+        // Older versions of Scrcpy doesn't have `keyframe` property,
+        // so detect it from the frame data.
+        return (keyframe ?? H264.containsKeyFrame(data)) ? "key" : "delta";
     }
 
-    override configure(data: Uint8Array): VideoDecoderConfig {
-        const {
-            profileIndex,
-            constraintSet,
-            levelIndex,
-            croppedWidth,
-            croppedHeight,
-        } = h264ParseConfiguration(data);
-
-        this.#updateSize(croppedWidth, croppedHeight);
+    override configure(data: Uint8Array): H26xTransformStream.Config {
+        const configuration = H264.parseConfiguration(data);
 
-        // https://www.rfc-editor.org/rfc/rfc6381#section-3.3
-        // ISO Base Media File Format Name Space
-        const codec =
-            "avc1." +
-            hexTwoDigits(profileIndex) +
-            hexTwoDigits(constraintSet) +
-            hexTwoDigits(levelIndex);
         return {
-            codec: codec,
-            hardwareAcceleration:
-                this.#options?.hardwareAcceleration ?? "no-preference",
-            optimizeForLatency: true,
+            codec: H264.toCodecString(configuration),
+            codedHeight: configuration.croppedHeight,
+            codedWidth: configuration.croppedWidth,
         };
     }
 }

package/src/video/codec/h265.ts

@@ -1,54 +1,25 @@
-import { getUint32LittleEndian } from "@yume-chan/no-data-view";
-import { h265ParseConfiguration } from "@yume-chan/scrcpy";
+import { H265 } from "@yume-chan/media-codec";
 
-import { H26xDecoder } from "./h26x.js";
-import type { CodecDecoderOptions } from "./type.js";
-import { hexDigits } from "./utils.js";
+import { H26xTransformStream } from "./h26x.js";
 
-export class H265Decoder extends H26xDecoder {
-    #updateSize: (width: number, height: number) => void;
-    #options: CodecDecoderOptions | undefined;
-
-    constructor(
-        decoder: VideoDecoder,
-        updateSize: (width: number, height: number) => void,
-        options?: CodecDecoderOptions,
-    ) {
-        super(decoder);
-        this.#updateSize = updateSize;
-        this.#options = options;
+export class H265TransformStream extends H26xTransformStream {
+    override convertFrameType(
+        keyframe: boolean | undefined,
+    ): EncodedVideoChunkType {
+        // H.265 was added in Scrcpy 2.0 which must have `keyframe` property.
+        return keyframe ? "key" : "delta";
     }
 
-    override configure(data: Uint8Array): VideoDecoderConfig {
-        const {
-            generalProfileSpace,
-            generalProfileIndex,
-            generalProfileCompatibilitySet,
-            generalTierFlag,
-            generalLevelIndex,
-            generalConstraintSet,
-            croppedWidth,
-            croppedHeight,
-        } = h265ParseConfiguration(data);
-
-        this.#updateSize(croppedWidth, croppedHeight);
+    override configure(data: Uint8Array): H26xTransformStream.Config {
+        const configuration = H265.parseConfiguration(data);
 
-        const codec = [
-            "hev1",
-            ["", "A", "B", "C"][generalProfileSpace]! +
-                generalProfileIndex.toString(),
-            hexDigits(getUint32LittleEndian(generalProfileCompatibilitySet, 0)),
-            (generalTierFlag ? "H" : "L") + generalLevelIndex.toString(),
-            ...Array.from(generalConstraintSet, hexDigits),
-        ].join(".");
         return {
-            codec,
-            // Microsoft Edge requires explicit size to work
-            codedWidth: croppedWidth,
-            codedHeight: croppedHeight,
-            hardwareAcceleration:
-                this.#options?.hardwareAcceleration ?? "no-preference",
-            optimizeForLatency: true,
+            codec: H265.toCodecString(configuration),
+            // Microsoft Edge on Windows requires explicit size,
+            // otherwise it returns frames in incorrect size.
+            // And it needs cropped size, as opposed to the option name.
+            codedWidth: configuration.croppedWidth,
+            codedHeight: configuration.croppedHeight,
         };
     }
 }

package/src/video/codec/h26x.ts

@@ -1,101 +1,50 @@
-import type {
-    ScrcpyMediaStreamDataPacket,
-    ScrcpyMediaStreamPacket,
-} from "@yume-chan/scrcpy";
-
-import type { CodecDecoder } from "./type.js";
-
-export abstract class H26xDecoder implements CodecDecoder {
-    #decoder: VideoDecoder;
-
-    #config: (VideoDecoderConfig & { raw: Uint8Array }) | undefined;
-    #configured = false;
-
-    constructor(decoder: VideoDecoder) {
-        this.#decoder = decoder;
+import { TransformStream } from "@yume-chan/stream-extra";
+
+import type { CodecTransformStream } from "./type.js";
+
+export abstract class H26xTransformStream
+    extends TransformStream<
+        CodecTransformStream.Input,
+        CodecTransformStream.Output
+    >
+    implements CodecTransformStream
+{
+    constructor() {
+        super({
+            transform: (packet, controller) => {
+                if (packet.type === "configuration") {
+                    controller.enqueue(this.#configure(packet.data));
+                    return;
+                }
+
+                controller.enqueue({
+                    type: this.convertFrameType(packet.keyframe, packet.data),
+                    timestamp: packet.timestamp,
+                    data: packet.data,
+                });
+            },
+        });
     }
 
-    abstract configure(data: Uint8Array): VideoDecoderConfig;
-
-    #configureAndDecodeFirstKeyframe(
-        config: VideoDecoderConfig & { raw: Uint8Array },
-        packet: ScrcpyMediaStreamDataPacket,
-    ) {
-        this.#decoder.configure(config);
-        this.#configured = true;
-
-        // For H.264 and H.265, when the stream is in Annex B format
-        // (which Scrcpy uses, as Android MediaCodec produces),
-        // configuration data needs to be combined with the first frame data.
-        // https://www.w3.org/TR/webcodecs-avc-codec-registration/#encodedvideochunk-type
-        const { raw } = config;
-        const data = new Uint8Array(raw.length + packet.data.length);
-        data.set(raw, 0);
-        data.set(packet.data, raw.length);
-
-        this.#decoder.decode(
-            new EncodedVideoChunk({
-                type: "key",
-                timestamp: 0,
-                data,
-            }),
-        );
+    abstract convertFrameType(
+        keyframe: boolean | undefined,
+        data: Uint8Array,
+    ): EncodedVideoChunkType;
+
+    abstract configure(data: Uint8Array): H26xTransformStream.Config;
+
+    #configure(data: Uint8Array): CodecTransformStream.Config {
+        return {
+            ...this.configure(data),
+            // For H.264 and H.265, when the stream is in Annex B format
+            // (which Scrcpy uses, as Android MediaCodec produces),
+            // configuration data needs to be combined with the first frame data.
+            // https://www.w3.org/TR/webcodecs-avc-codec-registration/#encodedvideochunk-type
+            raw: data,
+        };
     }
+}
 
-    decode(packet: ScrcpyMediaStreamPacket): void {
-        if (packet.type === "configuration") {
-            this.#config = {
-                ...this.configure(packet.data),
-                raw: packet.data,
-            };
-            this.#configured = false;
-            return;
-        }
-
-        if (!this.#config) {
-            throw new Error("Decoder not configured");
-        }
-
-        if (packet.keyframe) {
-            if (this.#decoder.decodeQueueSize) {
-                // If the device is too slow to decode all frames,
-                // discard queued frames when next keyframe arrives.
-                // (can only do this for keyframes because decoding must start from a keyframe)
-                // This limits the maximum latency to 1 keyframe interval
-                // (60 frames by default).
-                this.#decoder.reset();
-
-                // `reset` also resets the decoder configuration
-                // so we need to re-configure it again.
-                this.#configureAndDecodeFirstKeyframe(this.#config, packet);
-                return;
-            }
-
-            if (!this.#configured) {
-                this.#configureAndDecodeFirstKeyframe(this.#config, packet);
-                return;
-            }
-        }
-
-        if (!this.#configured) {
-            if (packet.keyframe === undefined) {
-                // Scrcpy <1.23 doesn't send `keyframe` flag
-                // Infer the first frame after configuration as keyframe
-                // (`VideoDecoder` will throw error if it's not)
-                this.#configureAndDecodeFirstKeyframe(this.#config, packet);
-                return;
-            }
-
-            throw new Error("Expect a keyframe but got a delta frame");
-        }
-
-        this.#decoder.decode(
-            new EncodedVideoChunk({
-                // Treat `undefined` as `key`, otherwise won't decode.
-                type: packet.keyframe === false ? "delta" : "key",
-                timestamp: 0,
-                data: packet.data,
-            }),
-        );
-    }
+export namespace H26xTransformStream {
+    export type Config = Omit<CodecTransformStream.Config, "raw">;
 }

package/src/video/codec/index.ts

@@ -3,4 +3,3 @@ export * from "./h264.js";
 export * from "./h265.js";
 export * from "./h26x.js";
 export * from "./type.js";
-export * from "./utils.js";

package/src/video/codec/type.ts

@@ -1,17 +1,37 @@
-import type { ScrcpyMediaStreamPacket } from "@yume-chan/scrcpy";
+import type {
+    ScrcpyMediaStreamConfigurationPacket,
+    ScrcpyMediaStreamDataPacket,
+} from "@yume-chan/scrcpy";
+import type { TransformStream } from "@yume-chan/stream-extra";
 
-export interface CodecDecoder {
-    decode(packet: ScrcpyMediaStreamPacket): void;
-}
+export type CodecTransformStream = TransformStream<
+    CodecTransformStream.Input,
+    CodecTransformStream.Output
+>;
+
+export namespace CodecTransformStream {
+    export type Input =
+        | ScrcpyMediaStreamConfigurationPacket
+        | (ScrcpyMediaStreamDataPacket & { timestamp: number });
+
+    export type Config = VideoDecoderConfig & {
+        codedWidth: number;
+        codedHeight: number;
+        /**
+         * Sets an optional raw buffer what will be prepended with the first key frame for decoding.
+         *
+         * Some codecs (e.g. H.264 and H.265 in Annex B format)
+         * send configuration in separate packet,
+         * but the configuration also needs to be feed into the decoder.
+         */
+        raw?: AllowSharedBufferSource;
+    };
+
+    export type VideoChunk = EncodedVideoChunkInit;
 
-export interface CodecDecoderOptions {
-    hardwareAcceleration?: HardwareAcceleration | undefined;
+    export type Output = Config | VideoChunk;
 }
 
-export interface CodecDecoderConstructor {
-    new (
-        decoder: VideoDecoder,
-        updateSize: (width: number, height: number) => void,
-        options?: CodecDecoderOptions,
-    ): CodecDecoder;
+export interface CodecTransformStreamConstructor {
+    new (): CodecTransformStream;
 }