@yume-chan/scrcpy-decoder-webcodecs 2.5.0 → 3.0.0-beta.0

package/esm/video/utils/video-decoder-stream.js

@@ -0,0 +1,284 @@
+import { EventEmitter, StickyEventEmitter } from "@yume-chan/event";
+import { ScrcpyVideoDecoderPerformanceCounter } from "@yume-chan/scrcpy-decoder-shared";
+import { concatBuffers, TransformStream } from "@yume-chan/stream-extra";
+function decodeFirstKeyFrame(decoder, config, chunk) {
+    decoder.decode(new EncodedVideoChunk({
+        type: "key",
+        timestamp: chunk.timestamp,
+        // `lib.dom.d.ts` doesn't allow `duration` to be undefined.
+        duration: chunk.duration,
+        data: config.raw
+            ? concatBuffers([config.raw, chunk.data])
+            : chunk.data,
+    }));
+}
+function decodeFrame(decoder, chunk) {
+    decoder.decode(new EncodedVideoChunk({
+        type: chunk.type,
+        timestamp: chunk.timestamp,
+        // `lib.dom.d.ts` doesn't allow `duration` to be undefined.
+        duration: chunk.duration,
+        data: chunk.data,
+    }));
+}
+export class VideoDecoderStream extends TransformStream {
+    #controller;
+    #state = { type: "unconfigured" };
+    #hardwareAcceleration = new StickyEventEmitter({
+        initialValue: "no-preference",
+        equals: (a, b) => a === b,
+    });
+    get hardwareAcceleration() {
+        return this.#hardwareAcceleration.value;
+    }
+    get onHardwareAccelerationChange() {
+        return this.#hardwareAcceleration.event;
+    }
+    /**
+     * Gets the number of frames waiting to be decoded.
+     */
+    get decodeQueueSize() {
+        if ("decoder" in this.#state) {
+            return this.#state.decoder.decodeQueueSize;
+        }
+        return 0;
+    }
+    #onDequeue = new EventEmitter();
+    /**
+     * Gets an event when a frame is dequeued (either decoded or discarded).
+     */
+    get onDequeue() {
+        return this.#onDequeue.event;
+    }
+    #counter = new ScrcpyVideoDecoderPerformanceCounter();
+    /**
+     * Gets the number of frames decoded by the decoder.
+     */
+    get framesDecoded() {
+        return this.#counter.framesDecoded;
+    }
+    /**
+     * Gets the number of frames skipped by the decoder.
+     */
+    get framesSkippedDecoding() {
+        return this.#counter.framesSkippedDecoding;
+    }
+    /**
+     * Gets the number of times the decoder has been reset to catch up new keyframes.
+     */
+    get decoderResetCount() {
+        return this.#counter.decoderResetCount;
+    }
+    #abortController = new AbortController();
+    #skipFramesBefore = -1;
+    constructor() {
+        let controller;
+        super({
+            start: (controller_) => {
+                // WARN: can't use `this` here
+                controller = controller_;
+            },
+            transform: async (chunk) => {
+                if ("codec" in chunk) {
+                    this.#hardwareAcceleration.fire(chunk.hardwareAcceleration ?? "no-preference");
+                    // Validate the config by creating decoder right away
+                    switch (this.#state.type) {
+                        case "unconfigured":
+                        case "reclaimed":
+                            this.#state = {
+                                type: "configured",
+                                decoder: this.#createDecoder(chunk),
+                                config: chunk,
+                            };
+                            break;
+                        // @ts-expect-error: intentional case fallthrough
+                        case "decoding":
+                            await this.#state.decoder.flush();
+                        // fallthrough
+                        case "configured":
+                            // Reuse the existing decoder.
+                            // `decoder` is definitely in "configured" state,
+                            // otherwise the stream is in error state
+                            // and no `transform` callbacks will be called.
+                            this.#state.decoder.configure(chunk);
+                            this.#state = {
+                                type: "configured",
+                                decoder: this.#state.decoder,
+                                config: chunk,
+                            };
+                            break;
+                    }
+                    return;
+                }
+                this.#handleVideoChunk(chunk);
+            },
+            flush: async () => {
+                // only call `flush` when `decoder.state` is "configured".
+                if ("decoder" in this.#state &&
+                    this.#state.decoder.state === "configured") {
+                    // This also delays closing `readable` and `writable` streams
+                    // (if they are not in error state) until all frames are decoded.
+                    await this.#state.decoder.flush();
+                }
+                this.#dispose();
+            },
+            cancel: () => {
+                // Immediately close the decoder on stream cancel/error
+                this.#dispose();
+            },
+        });
+        this.#controller = controller;
+    }
+    #handleVideoChunk(chunk) {
+        switch (this.#state.type) {
+            case "unconfigured":
+                throw new Error("Decoder not configured");
+            case "configured": {
+                if (chunk.type === "delta") {
+                    throw new Error("Expect a keyframe but got a delta frame");
+                }
+                decodeFirstKeyFrame(this.#state.decoder, this.#state.config, chunk);
+                this.#state = {
+                    type: "decoding",
+                    decoder: this.#state.decoder,
+                    config: this.#state.config,
+                    frames: [chunk],
+                };
+                break;
+            }
+            case "decoding":
+                if (chunk.type === "key") {
+                    this.#state.frames.length = 0;
+                }
+                this.#state.frames.push(chunk);
+                decodeFrame(this.#state.decoder, chunk);
+                break;
+            case "reclaimed": {
+                const decoder = this.#createDecoder(this.#state.config);
+                if (chunk.type === "key") {
+                    decodeFirstKeyFrame(decoder, this.#state.config, chunk);
+                    this.#state = {
+                        type: "decoding",
+                        decoder: decoder,
+                        config: this.#state.config,
+                        frames: [chunk],
+                    };
+                    return;
+                }
+                if (!this.#state.frames) {
+                    // Reclaimed when configured but no frames have been decoded yet,
+                    // so a keyframe is required to start decoding.
+                    throw new Error("Expect a keyframe but got a delta frame");
+                }
+                decodeFirstKeyFrame(decoder, this.#state.config, this.#state.frames[0]);
+                // `chunk`s must have monotonic timestamps, so we can skip all frames before `chunk`.
+                this.#skipFramesBefore = chunk.timestamp;
+                this.#state.frames.push(chunk);
+                for (const frame of this.#state.frames.slice(1)) {
+                    decodeFrame(decoder, frame);
+                }
+                this.#state = {
+                    type: "decoding",
+                    decoder: decoder,
+                    config: this.#state.config,
+                    frames: this.#state.frames,
+                };
+                break;
+            }
+        }
+    }
+    #createDecoder(config) {
+        const decoder = new VideoDecoder({
+            output: (frame) => {
+                if (frame.timestamp < this.#skipFramesBefore) {
+                    frame.close();
+                    return;
+                }
+                this.#counter.increaseFramesDecoded();
+                try {
+                    this.#controller.enqueue(frame);
+                }
+                catch {
+                    // `controller` is closed
+                    frame.close();
+                }
+            },
+            error: (error) => {
+                if (error.name === "QuotaExceededError") {
+                    // Chrome reclaims inactive `VideoDecoder`'s after 90 seconds
+                    // (for example due to pausing decoding when the document is hidden).
+                    // https://source.chromium.org/chromium/chromium/src/+/main:third_party/blink/renderer/modules/webcodecs/reclaimable_codec.cc;l=181;drc=aba14550ed0b4620deb59e7c5f551cbde8970fb3
+                    // Reset states, it will be recreated in next `transform` call.
+                    switch (this.#state.type) {
+                        case "unconfigured":
+                        case "reclaimed":
+                            // Don't throw, report errors to stream
+                            this.#controller.error(new Error("Unexpected state when reclaiming decoder"));
+                            this.#dispose();
+                            return;
+                        case "configured":
+                            // Unlike `transform` callback,
+                            // this doesn't re-create the decoder immediately,
+                            // because the config has already been validated,
+                            // and since browser reclaims the decoder to save hardware resources,
+                            // we don't want to undo that by creating a new decoder immediately.
+                            this.#state = {
+                                type: "reclaimed",
+                                config,
+                                frames: undefined,
+                            };
+                            break;
+                        case "decoding":
+                            this.#state = {
+                                type: "reclaimed",
+                                config,
+                                frames: this.#state.frames,
+                            };
+                            break;
+                    }
+                    return;
+                }
+                // Maybe the decoder is hardware accelerated but the hardware has an issue,
+                // retry with software decoder.
+                if (error.name === "EncodingError" &&
+                    this.#state.type === "decoding" &&
+                    config.hardwareAcceleration !== "prefer-software") {
+                    try {
+                        config.hardwareAcceleration = "prefer-software";
+                        this.#hardwareAcceleration.fire("prefer-software");
+                        const decoder = this.#createDecoder(config);
+                        // Replay frames with rendering skipping
+                        this.#skipFramesBefore =
+                            this.#state.frames.at(-1).timestamp;
+                        decodeFirstKeyFrame(decoder, config, this.#state.frames[0]);
+                        for (const frame of this.#state.frames.slice(1)) {
+                            decodeFrame(decoder, frame);
+                        }
+                        // Update state
+                        this.#state.decoder = decoder;
+                        return;
+                    }
+                    catch {
+                        // ignore, report original error to stream
+                    }
+                }
+                // Propagate other decoder errors to stream.
+                this.#controller.error(error);
+                this.#dispose();
+            },
+        });
+        decoder.addEventListener("dequeue", () => this.#onDequeue.fire(undefined), { signal: this.#abortController.signal });
+        decoder.configure(config);
+        return decoder;
+    }
+    #dispose() {
+        this.#abortController.abort();
+        this.#onDequeue.dispose();
+        if ("decoder" in this.#state &&
+            this.#state.decoder.state !== "closed") {
+            this.#state.decoder.close();
+        }
+        this.#state = { type: "unconfigured" };
+    }
+}
+//# sourceMappingURL=video-decoder-stream.js.map

package/esm/video/utils/video-decoder-stream.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"video-decoder-stream.js","sourceRoot":"","sources":["../../../src/video/utils/video-decoder-stream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEpE,OAAO,EAAE,oCAAoC,EAAE,MAAM,kCAAkC,CAAC;AACxF,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AA0CzE,SAAS,mBAAmB,CACxB,OAAqB,EACrB,MAAmC,EACnC,KAAsC;IAEtC,OAAO,CAAC,MAAM,CACV,IAAI,iBAAiB,CAAC;QAClB,IAAI,EAAE,KAAK;QACX,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,2DAA2D;QAC3D,QAAQ,EAAE,KAAK,CAAC,QAAS;QACzB,IAAI,EAAE,MAAM,CAAC,GAAG;YACZ,CAAC,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACzC,CAAC,CAAC,KAAK,CAAC,IAAI;KACnB,CAAC,CACL,CAAC;AACN,CAAC;AAED,SAAS,WAAW,CAChB,OAAqB,EACrB,KAAsC;IAEtC,OAAO,CAAC,MAAM,CACV,IAAI,iBAAiB,CAAC;QAClB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,2DAA2D;QAC3D,QAAQ,EAAE,KAAK,CAAC,QAAS;QACzB,IAAI,EAAE,KAAK,CAAC,IAAI;KACnB,CAAC,CACL,CAAC;AACN,CAAC;AAED,MAAM,OAAO,kBACT,SAAQ,eAGP;IAGD,WAAW,CAAgD;IAE3D,MAAM,GAAU,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC;IAEzC,qBAAqB,GAAG,IAAI,kBAAkB,CAE5C;QACE,YAAY,EAAE,eAAe;QAC7B,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC;KAC5B,CAAC,CAAC;IACH,IAAI,oBAAoB;QACpB,OAAO,IAAI,CAAC,qBAAqB,CAAC,KAAK,CAAC;IAC5C,CAAC;IACD,IAAI,4BAA4B;QAC5B,OAAO,IAAI,CAAC,qBAAqB,CAAC,KAAK,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,IAAI,eAAe;QACf,IAAI,SAAS,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC;QAC/C,CAAC;QAED,OAAO,CAAC,CAAC;IACb,CAAC;IAED,UAAU,GAAG,IAAI,YAAY,EAAa,CAAC;IAC3C;;OAEG;IACH,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;IACjC,CAAC;IAED,QAAQ,GAAG,IAAI,oCAAoC,EAAE,CAAC;IACtD;;OAEG;IACH,IAAI,aAAa;QACb,OAAO,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC;IACvC,CAAC;IACD;;OAEG;IACH,IAAI,qBAAqB;QACrB,OAAO,IAAI,CAAC,QAAQ,CAAC,qBAAqB,CAAC;IAC/C,CAAC;IACD;;OAEG;IACH,IAAI,iBAAiB;QACjB,OAAO,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAC3C,CAAC;IAED,gBAAgB,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,iBAAiB,GAAG,CAAC,CAAC,CAAC;IAEvB;QACI,IAAI,UAAyD,CAAC;QAE9D,KAAK,CAAC;YACF,KAAK,EAAE,CAAC,WAAW,EAAE,EAAE;gBACnB,8BAA8B;gBAC9B,UAAU,GAAG,WAAW,CAAC;YAC7B,CAAC;YACD,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;gBACvB,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;oBACnB,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAC3B,KAAK,CAAC,oBAAoB,IAAI,eAAe,CAChD,CAAC;oBACF,qDAAqD;oBACrD,QAAQ,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;wBACvB,KAAK,cAAc,CAAC;wBACpB,KAAK,WAAW;4BACZ,IAAI,CAAC,MAAM,GAAG;gCACV,IAAI,EAAE,YAAY;gCAClB,OAAO,EAAE,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC;gCACnC,MAAM,EAAE,KAAK;6BAChB,CAAC;4BACF,MAAM;wBACV,iDAAiD;wBACjD,KAAK,UAAU;4BACX,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;wBACtC,cAAc;wBACd,KAAK,YAAY;4BACb,8BAA8B;4BAC9B,iDAAiD;4BACjD,yCAAyC;4BACzC,+CAA+C;4BAC/C,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;4BACrC,IAAI,CAAC,MAAM,GAAG;gCACV,IAAI,EAAE,YAAY;gCAClB,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;gCAC5B,MAAM,EAAE,KAAK;6BAChB,CAAC;4BACF,MAAM;oBACd,CAAC;oBACD,OAAO;gBACX,CAAC;gBAED,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;YAClC,CAAC;YACD,KAAK,EAAE,KAAK,IAAI,EAAE;gBACd,0DAA0D;gBAC1D,IACI,SAAS,IAAI,IAAI,CAAC,MAAM;oBACxB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,YAAY,EAC5C,CAAC;oBACC,6DAA6D;oBAC7D,iEAAiE;oBACjE,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACtC,CAAC;gBACD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpB,CAAC;YACD,MAAM,EAAE,GAAG,EAAE;gBACT,uDAAuD;gBACvD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpB,CAAC;SACJ,CAAC,CAAC;QAEH,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAClC,CAAC;IAED,iBAAiB,CAAC,KAAsC;QACpD,QAAQ,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACvB,KAAK,cAAc;gBACf,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;YAC9C,KAAK,YAAY,CAAC,CAAC,CAAC;gBAChB,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;oBACzB,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;gBAC/D,CAAC;gBAED,mBAAmB,CACf,IAAI,CAAC,MAAM,CAAC,OAAO,EACnB,IAAI,CAAC,MAAM,CAAC,MAAM,EAClB,KAAK,CACR,CAAC;gBAEF,IAAI,CAAC,MAAM,GAAG;oBACV,IAAI,EAAE,UAAU;oBAChB,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;oBAC5B,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;oBAC1B,MAAM,EAAE,CAAC,KAAK,CAAC;iBAClB,CAAC;gBACF,MAAM;YACV,CAAC;YACD,KAAK,UAAU;gBACX,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;oBACvB,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;gBAClC,CAAC;gBACD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAE/B,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBACxC,MAAM;YACV,KAAK,WAAW,CAAC,CAAC,CAAC;gBACf,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAExD,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;oBACvB,mBAAmB,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;oBAExD,IAAI,CAAC,MAAM,GAAG;wBACV,IAAI,EAAE,UAAU;wBAChB,OAAO,EAAE,OAAO;wBAChB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;wBAC1B,MAAM,EAAE,CAAC,KAAK,CAAC;qBAClB,CAAC;oBACF,OAAO;gBACX,CAAC;gBAED,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;oBACtB,iEAAiE;oBACjE,+CAA+C;oBAC/C,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;gBAC/D,CAAC;gBAED,mBAAmB,CACf,OAAO,EACP,IAAI,CAAC,MAAM,CAAC,MAAM,EAClB,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CACzB,CAAC;gBAEF,qFAAqF;gBACrF,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC,SAAS,CAAC;gBACzC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAE/B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC9C,WAAW,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBAChC,CAAC;gBAED,IAAI,CAAC,MAAM,GAAG;oBACV,IAAI,EAAE,UAAU;oBAChB,OAAO,EAAE,OAAO;oBAChB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;oBAC1B,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;iBAC7B,CAAC;gBAEF,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;IAED,cAAc,CAAC,MAAmC;QAC9C,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC;YAC7B,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;gBACd,IAAI,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;oBAC3C,KAAK,CAAC,KAAK,EAAE,CAAC;oBACd,OAAO;gBACX,CAAC;gBAED,IAAI,CAAC,QAAQ,CAAC,qBAAqB,EAAE,CAAC;gBACtC,IAAI,CAAC;oBACD,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBACpC,CAAC;gBAAC,MAAM,CAAC;oBACL,yBAAyB;oBACzB,KAAK,CAAC,KAAK,EAAE,CAAC;gBAClB,CAAC;YACL,CAAC;YACD,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE;gBACb,IAAI,KAAK,CAAC,IAAI,KAAK,oBAAoB,EAAE,CAAC;oBACtC,6DAA6D;oBAC7D,qEAAqE;oBACrE,gLAAgL;oBAChL,+DAA+D;oBAC/D,QAAQ,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;wBACvB,KAAK,cAAc,CAAC;wBACpB,KAAK,WAAW;4BACZ,uCAAuC;4BACvC,IAAI,CAAC,WAAW,CAAC,KAAK,CAClB,IAAI,KAAK,CACL,0CAA0C,CAC7C,CACJ,CAAC;4BACF,IAAI,CAAC,QAAQ,EAAE,CAAC;4BAChB,OAAO;wBACX,KAAK,YAAY;4BACb,+BAA+B;4BAC/B,kDAAkD;4BAClD,iDAAiD;4BACjD,qEAAqE;4BACrE,oEAAoE;4BACpE,IAAI,CAAC,MAAM,GAAG;gCACV,IAAI,EAAE,WAAW;gCACjB,MAAM;gCACN,MAAM,EAAE,SAAS;6BACpB,CAAC;4BACF,MAAM;wBACV,KAAK,UAAU;4BACX,IAAI,CAAC,MAAM,GAAG;gCACV,IAAI,EAAE,WAAW;gCACjB,MAAM;gCACN,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;6BAC7B,CAAC;4BACF,MAAM;oBACd,CAAC;oBACD,OAAO;gBACX,CAAC;gBAED,2EAA2E;gBAC3E,+BAA+B;gBAC/B,IACI,KAAK,CAAC,IAAI,KAAK,eAAe;oBAC9B,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,UAAU;oBAC/B,MAAM,CAAC,oBAAoB,KAAK,iBAAiB,EACnD,CAAC;oBACC,IAAI,CAAC;wBACD,MAAM,CAAC,oBAAoB,GAAG,iBAAiB,CAAC;wBAChD,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;wBACnD,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;wBAE5C,wCAAwC;wBACxC,IAAI,CAAC,iBAAiB;4BAClB,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,SAAS,CAAC;wBACzC,mBAAmB,CACf,OAAO,EACP,MAAM,EACN,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CACzB,CAAC;wBACF,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;4BAC9C,WAAW,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;wBAChC,CAAC;wBAED,eAAe;wBACf,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC;wBAC9B,OAAO;oBACX,CAAC;oBAAC,MAAM,CAAC;wBACL,0CAA0C;oBAC9C,CAAC;gBACL,CAAC;gBAED,4CAA4C;gBAC5C,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC9B,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpB,CAAC;SACJ,CAAC,CAAC;QAEH,OAAO,CAAC,gBAAgB,CACpB,SAAS,EACT,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,EACrC,EAAE,MAAM,EAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,CAC3C,CAAC;QAEF,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAE1B,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,QAAQ;QACJ,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;QAC9B,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QAE1B,IACI,SAAS,IAAI,IAAI,CAAC,MAAM;YACxB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,QAAQ,EACxC,CAAC;YACC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QAChC,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC;IAC3C,CAAC;CACJ"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yume-chan/scrcpy-decoder-webcodecs",
-  "version": "2.5.0",
+  "version": "3.0.0-beta.0",
   "description": "Raw H.264 stream decoder and renderer using WebCodecs API (requires modern browser).",
   "keywords": [
     "adb",
@@ -14,11 +14,11 @@
     "email": "cnsimonchan@live.com",
     "url": "https://chensi.moe/blog"
   },
-  "homepage": "https://github.com/yume-chan/ya-webadb/tree/main/packages/scrcpy-decoder-webcodecs#readme",
+  "homepage": "https://github.com/yume-chan/ya-webadb/tree/main/libraries/scrcpy-decoder-webcodecs#readme",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/yume-chan/ya-webadb.git",
-    "directory": "packages/scrcpy-decoder-webcodecs"
+    "directory": "libraries/scrcpy-decoder-webcodecs"
   },
   "bugs": {
     "url": "https://github.com/yume-chan/ya-webadb/issues"
@@ -29,20 +29,21 @@
   "sideEffects": false,
   "dependencies": {
     "@yume-chan/async": "^4.1.3",
-    "@yume-chan/event": "^2.0.0",
-    "@yume-chan/no-data-view": "^2.0.0",
-    "@yume-chan/scrcpy": "^2.3.0",
-    "@yume-chan/scrcpy-decoder-tinyh264": "^2.1.0",
-    "@yume-chan/stream-extra": "^2.1.0"
+    "@yume-chan/media-codec": "3.0.0-beta.0",
+    "@yume-chan/event": "3.0.0-beta.0",
+    "@yume-chan/no-data-view": "3.0.0-beta.0",
+    "@yume-chan/scrcpy": "3.0.0-beta.0",
+    "@yume-chan/scrcpy-decoder-shared": "3.0.0-beta.0",
+    "@yume-chan/stream-extra": "3.0.0-beta.0"
   },
   "devDependencies": {
-    "prettier": "^3.6.2",
-    "typescript": "^5.9.2",
+    "prettier": "^3.8.4",
+    "typescript": "^6.0.3",
     "@yume-chan/eslint-config": "^1.0.0",
     "@yume-chan/tsconfig": "^1.0.0"
   },
   "scripts": {
-    "build": "tsc -b tsconfig.build.json",
+    "build": "tsc -p tsconfig.build.json",
     "lint": "run-eslint && prettier src/**/*.ts --write --tab-width 4"
   }
 }

package/src/video/codec/av1.ts

@@ -1,106 +1,44 @@
-import type { ScrcpyMediaStreamPacket } from "@yume-chan/scrcpy";
-import { Av1 } from "@yume-chan/scrcpy";
-
-import type { CodecDecoder, CodecDecoderOptions } from "./type.js";
-import { decimalTwoDigits } from "./utils.js";
-
-export class Av1Codec implements CodecDecoder {
-    #decoder: VideoDecoder;
-    #updateSize: (width: number, height: number) => void;
-    #options: CodecDecoderOptions | undefined;
-
-    constructor(
-        decoder: VideoDecoder,
-        updateSize: (width: number, height: number) => void,
-        options?: CodecDecoderOptions,
-    ) {
-        this.#decoder = decoder;
-        this.#updateSize = updateSize;
-        this.#options = options;
-    }
-
-    #configure(data: Uint8Array) {
-        const parser = new Av1(data);
-        const sequenceHeader = parser.searchSequenceHeaderObu();
-
-        if (!sequenceHeader) {
-            return;
-        }
-
-        const {
-            seq_profile: seqProfile,
-            seq_level_idx: [seqLevelIdx = 0],
-            max_frame_width_minus_1,
-            max_frame_height_minus_1,
-            color_config: {
-                BitDepth,
-                mono_chrome: monoChrome,
-                subsampling_x: subsamplingX,
-                subsampling_y: subsamplingY,
-                chroma_sample_position: chromaSamplePosition,
-                color_description_present_flag,
+import { Av1 } from "@yume-chan/media-codec";
+import { TransformStream } from "@yume-chan/stream-extra";
+
+import type { CodecTransformStream } from "./type.js";
+
+export class Av1TransformStream
+    extends TransformStream<
+        CodecTransformStream.Input,
+        CodecTransformStream.Output
+    >
+    implements CodecTransformStream
+{
+    constructor() {
+        super({
+            transform: (packet, controller) => {
+                if (packet.type === "configuration") {
+                    // AV1 decoder doesn't need configuration packets
+                    return;
+                }
+
+                const parser = new Av1(packet.data);
+                const sequenceHeader = parser.searchSequenceHeaderObu();
+
+                if (sequenceHeader) {
+                    const width = sequenceHeader.max_frame_width_minus_1 + 1;
+                    const height = sequenceHeader.max_frame_height_minus_1 + 1;
+
+                    controller.enqueue({
+                        codec: Av1.toCodecString(sequenceHeader),
+                        codedWidth: width,
+                        codedHeight: height,
+                    });
+                }
+
+                controller.enqueue({
+                    // AV1 was added in Scrcpy 2.0 which must have `keyframe` property.
+                    type: packet.keyframe ? "key" : "delta",
+                    timestamp: packet.timestamp,
+                    data: packet.data,
+                });
             },
-        } = sequenceHeader;
-
-        let colorPrimaries: Av1.ColorPrimaries;
-        let transferCharacteristics: Av1.TransferCharacteristics;
-        let matrixCoefficients: Av1.MatrixCoefficients;
-        let colorRange: boolean;
-        if (color_description_present_flag) {
-            ({
-                color_primaries: colorPrimaries,
-                transfer_characteristics: transferCharacteristics,
-                matrix_coefficients: matrixCoefficients,
-                color_range: colorRange,
-            } = sequenceHeader.color_config);
-        } else {
-            colorPrimaries = Av1.ColorPrimaries.Bt709;
-            transferCharacteristics = Av1.TransferCharacteristics.Bt709;
-            matrixCoefficients = Av1.MatrixCoefficients.Bt709;
-            colorRange = false;
-        }
-
-        const width = max_frame_width_minus_1 + 1;
-        const height = max_frame_height_minus_1 + 1;
-
-        this.#updateSize(width, height);
-
-        const codec = [
-            "av01",
-            seqProfile.toString(16),
-            decimalTwoDigits(seqLevelIdx) +
-                (sequenceHeader.seq_tier[0] ? "H" : "M"),
-            decimalTwoDigits(BitDepth),
-            monoChrome ? "1" : "0",
-            (subsamplingX ? "1" : "0") +
-                (subsamplingY ? "1" : "0") +
-                chromaSamplePosition.toString(),
-            decimalTwoDigits(colorPrimaries),
-            decimalTwoDigits(transferCharacteristics),
-            decimalTwoDigits(matrixCoefficients),
-            colorRange ? "1" : "0",
-        ].join(".");
-        this.#decoder.configure({
-            codec,
-            hardwareAcceleration:
-                this.#options?.hardwareAcceleration ?? "no-preference",
-            optimizeForLatency: true,
         });
     }
-
-    decode(packet: ScrcpyMediaStreamPacket): void {
-        if (packet.type === "configuration") {
-            return;
-        }
-
-        this.#configure(packet.data);
-        this.#decoder.decode(
-            new EncodedVideoChunk({
-                // Treat `undefined` as `key`, otherwise it won't decode.
-                type: packet.keyframe === false ? "delta" : "key",
-                timestamp: 0,
-                data: packet.data,
-            }),
-        );
-    }
 }

package/src/video/codec/h264.ts

@@ -1,48 +1,24 @@
-import { h264ParseConfiguration } from "@yume-chan/scrcpy";
+import { H264 } from "@yume-chan/media-codec";
 
-import { H26xDecoder } from "./h26x.js";
-import type { CodecDecoderOptions } from "./type.js";
-import { hexTwoDigits } from "./utils.js";
+import { H26xTransformStream } from "./h26x.js";
 
-export class H264Decoder extends H26xDecoder {
-    #decoder: VideoDecoder;
-    #updateSize: (width: number, height: number) => void;
-    #options: CodecDecoderOptions | undefined;
-
-    constructor(
-        decoder: VideoDecoder,
-        updateSize: (width: number, height: number) => void,
-        options?: CodecDecoderOptions,
-    ) {
-        super(decoder);
-        this.#decoder = decoder;
-        this.#updateSize = updateSize;
-        this.#options = options;
+export class H264TransformStream extends H26xTransformStream {
+    override convertFrameType(
+        keyframe: boolean | undefined,
+        data: Uint8Array,
+    ): EncodedVideoChunkType {
+        // Older versions of Scrcpy doesn't have `keyframe` property,
+        // so detect it from the frame data.
+        return (keyframe ?? H264.containsKeyFrame(data)) ? "key" : "delta";
     }
 
-    override configure(data: Uint8Array): void {
-        const {
-            profileIndex,
-            constraintSet,
-            levelIndex,
-            croppedWidth,
-            croppedHeight,
-        } = h264ParseConfiguration(data);
-
-        this.#updateSize(croppedWidth, croppedHeight);
+    override configure(data: Uint8Array): H26xTransformStream.Config {
+        const configuration = H264.parseConfiguration(data);
 
-        // https://www.rfc-editor.org/rfc/rfc6381#section-3.3
-        // ISO Base Media File Format Name Space
-        const codec =
-            "avc1." +
-            hexTwoDigits(profileIndex) +
-            hexTwoDigits(constraintSet) +
-            hexTwoDigits(levelIndex);
-        this.#decoder.configure({
-            codec: codec,
-            hardwareAcceleration:
-                this.#options?.hardwareAcceleration ?? "no-preference",
-            optimizeForLatency: true,
-        });
+        return {
+            codec: H264.toCodecString(configuration),
+            codedHeight: configuration.croppedHeight,
+            codedWidth: configuration.croppedWidth,
+        };
     }
 }

package/src/video/codec/h265.ts

@@ -1,56 +1,25 @@
-import { getUint32LittleEndian } from "@yume-chan/no-data-view";
-import { h265ParseConfiguration } from "@yume-chan/scrcpy";
+import { H265 } from "@yume-chan/media-codec";
 
-import { H26xDecoder } from "./h26x.js";
-import type { CodecDecoderOptions } from "./type.js";
-import { hexDigits } from "./utils.js";
+import { H26xTransformStream } from "./h26x.js";
 
-export class H265Decoder extends H26xDecoder {
-    #decoder: VideoDecoder;
-    #updateSize: (width: number, height: number) => void;
-    #options: CodecDecoderOptions | undefined;
-
-    constructor(
-        decoder: VideoDecoder,
-        updateSize: (width: number, height: number) => void,
-        options?: CodecDecoderOptions,
-    ) {
-        super(decoder);
-        this.#decoder = decoder;
-        this.#updateSize = updateSize;
-        this.#options = options;
+export class H265TransformStream extends H26xTransformStream {
+    override convertFrameType(
+        keyframe: boolean | undefined,
+    ): EncodedVideoChunkType {
+        // H.265 was added in Scrcpy 2.0 which must have `keyframe` property.
+        return keyframe ? "key" : "delta";
     }
 
-    override configure(data: Uint8Array): void {
-        const {
-            generalProfileSpace,
-            generalProfileIndex,
-            generalProfileCompatibilitySet,
-            generalTierFlag,
-            generalLevelIndex,
-            generalConstraintSet,
-            croppedWidth,
-            croppedHeight,
-        } = h265ParseConfiguration(data);
-
-        this.#updateSize(croppedWidth, croppedHeight);
+    override configure(data: Uint8Array): H26xTransformStream.Config {
+        const configuration = H265.parseConfiguration(data);
 
-        const codec = [
-            "hev1",
-            ["", "A", "B", "C"][generalProfileSpace]! +
-                generalProfileIndex.toString(),
-            hexDigits(getUint32LittleEndian(generalProfileCompatibilitySet, 0)),
-            (generalTierFlag ? "H" : "L") + generalLevelIndex.toString(),
-            ...Array.from(generalConstraintSet, hexDigits),
-        ].join(".");
-        this.#decoder.configure({
-            codec,
-            // Microsoft Edge requires explicit size to work
-            codedWidth: croppedWidth,
-            codedHeight: croppedHeight,
-            hardwareAcceleration:
-                this.#options?.hardwareAcceleration ?? "no-preference",
-            optimizeForLatency: true,
-        });
+        return {
+            codec: H265.toCodecString(configuration),
+            // Microsoft Edge on Windows requires explicit size,
+            // otherwise it returns frames in incorrect size.
+            // And it needs cropped size, as opposed to the option name.
+            codedWidth: configuration.croppedWidth,
+            codedHeight: configuration.croppedHeight,
+        };
     }
 }