@yume-chan/scrcpy-decoder-webcodecs 2.5.3 → 3.0.0-beta.0

package/esm/video/utils/timestamp.js

@@ -0,0 +1,87 @@
+import { TransformStream } from "@yume-chan/stream-extra";
+const view = new DataView(new ArrayBuffer(8));
+/**
+ * Gets the next presentable double-precision floating point value greater than x
+ * @param x A number. The caller must ensure that x is not NaN or Infinity
+ * @returns The next presentable double-precision floating point value
+ */
+function nextUp(x) {
+    if (x === 0)
+        return Number.MIN_VALUE;
+    // Write the number as a float64
+    view.setFloat64(0, x, false);
+    let bits = view.getBigUint64(0, false);
+    // If x > 0, increment bits; if x < 0, decrement bits
+    bits += x > 0 ? 1n : -1n;
+    view.setBigUint64(0, bits, false);
+    return view.getFloat64(0, false);
+}
+let prevValue = 0;
+/**
+ * Gets current time.
+ *
+ * The returned value is guaranteed to be monotonically increasing.
+ */
+export function increasingNow() {
+    let now = performance.now();
+    if (now <= prevValue) {
+        now = nextUp(prevValue);
+    }
+    prevValue = now;
+    return now;
+}
+export class TimestampTransforms {
+    /**
+     * Timestamp of the last frame to be skipped by pause controller.
+     */
+    #skipFramesUntil = -1;
+    #addTimestamp = new TransformStream({
+        transform: (packet, controller) => {
+            if (packet.type === "configuration") {
+                controller.enqueue(packet);
+                return;
+            }
+            // Use `timestamp` to convey `skipRendering` to later step
+            // and track total decoding time
+            const timestamp = increasingNow();
+            if (packet.skipRendering) {
+                this.#skipFramesUntil = timestamp;
+            }
+            controller.enqueue({
+                ...packet,
+                timestamp,
+            });
+        },
+    });
+    get addTimestamp() {
+        return this.#addTimestamp;
+    }
+    // This is not in `VideoDecoderStream` because
+    // this time includes all pre-processing time,
+    // and requires `EncodedVideoChunk.timestamp` to contain
+    // local time of when the frame is received,
+    // which is set by this class.
+    #totalDecodeTime = 0;
+    /**
+     * Gets the total time spent processing and decoding frames in milliseconds.
+     */
+    get totalDecodeTime() {
+        return this.#totalDecodeTime;
+    }
+    #consumeTimestamp = new TransformStream({
+        transform: (frame, controller) => {
+            // `frame.timestamp` is the same `EncodedVideoChunk.timestamp` set above
+            this.#totalDecodeTime += performance.now() - frame.timestamp;
+            // Don't count these frames as skipped rendering
+            if (frame.timestamp <= this.#skipFramesUntil) {
+                frame.close();
+                return;
+            }
+            controller.enqueue(frame);
+        },
+    });
+    get consumeTimestamp() {
+        return this.#consumeTimestamp;
+    }
+}
+//# sourceMappingURL=timestamp.js.map

package/esm/video/utils/timestamp.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"timestamp.js","sourceRoot":"","sources":["../../../src/video/utils/timestamp.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAI1D,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;AAE9C;;;;GAIG;AACH,SAAS,MAAM,CAAC,CAAS;IACrB,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC,SAAS,CAAC;IAErC,gCAAgC;IAChC,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;IAE7B,IAAI,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAEvC,qDAAqD;IACrD,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAEzB,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;IAClC,OAAO,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AACrC,CAAC;AAED,IAAI,SAAS,GAAG,CAAC,CAAC;AAElB;;;;GAIG;AACH,MAAM,UAAU,aAAa;IACzB,IAAI,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAC5B,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;QACnB,GAAG,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;IAC5B,CAAC;IACD,SAAS,GAAG,GAAG,CAAC;IAChB,OAAO,GAAG,CAAC;AACf,CAAC;AAED,MAAM,OAAO,mBAAmB;IAC5B;;OAEG;IACH,gBAAgB,GAAG,CAAC,CAAC,CAAC;IAEtB,aAAa,GAAG,IAAI,eAAe,CAGjC;QACE,SAAS,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE;YAC9B,IAAI,MAAM,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;gBAClC,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC3B,OAAO;YACX,CAAC;YAED,0DAA0D;YAC1D,gCAAgC;YAChC,MAAM,SAAS,GAAG,aAAa,EAAE,CAAC;YAElC,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;gBACvB,IAAI,CAAC,gBAAgB,GAAG,SAAS,CAAC;YACtC,CAAC;YAED,UAAU,CAAC,OAAO,CAAC;gBACf,GAAG,MAAM;gBACT,SAAS;aACZ,CAAC,CAAC;QACP,CAAC;KACJ,CAAC,CAAC;IACH,IAAI,YAAY;QACZ,OAAO,IAAI,CAAC,aAAa,CAAC;IAC9B,CAAC;IAED,8CAA8C;IAC9C,8CAA8C;IAC9C,wDAAwD;IACxD,4CAA4C;IAC5C,8BAA8B;IAC9B,gBAAgB,GAAG,CAAC,CAAC;IACrB;;OAEG;IACH,IAAI,eAAe;QACf,OAAO,IAAI,CAAC,gBAAgB,CAAC;IACjC,CAAC;IAED,iBAAiB,GAAG,IAAI,eAAe,CAAyB;QAC5D,SAAS,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE;YAC7B,wEAAwE;YACxE,IAAI,CAAC,gBAAgB,IAAI,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS,CAAC;YAE7D,gDAAgD;YAChD,IAAI,KAAK,CAAC,SAAS,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC3C,KAAK,CAAC,KAAK,EAAE,CAAC;gBACd,OAAO;YACX,CAAC;YAED,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;KACJ,CAAC,CAAC;IACH,IAAI,gBAAgB;QAChB,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAClC,CAAC;CACJ"}

package/esm/video/utils/video-decoder-stream.d.ts

@@ -0,0 +1,30 @@
+import type { ScrcpyVideoDecoderPerformanceCounterInterface } from "@yume-chan/scrcpy-decoder-shared";
+import { TransformStream } from "@yume-chan/stream-extra";
+import type { CodecTransformStream } from "../codec/type.js";
+export declare class VideoDecoderStream extends TransformStream<CodecTransformStream.Config | CodecTransformStream.VideoChunk, VideoFrame> implements ScrcpyVideoDecoderPerformanceCounterInterface {
+    #private;
+    get hardwareAcceleration(): HardwareAcceleration;
+    get onHardwareAccelerationChange(): import("@yume-chan/event").Event<HardwareAcceleration>;
+    /**
+     * Gets the number of frames waiting to be decoded.
+     */
+    get decodeQueueSize(): number;
+    /**
+     * Gets an event when a frame is dequeued (either decoded or discarded).
+     */
+    get onDequeue(): import("@yume-chan/event").Event<undefined>;
+    /**
+     * Gets the number of frames decoded by the decoder.
+     */
+    get framesDecoded(): number;
+    /**
+     * Gets the number of frames skipped by the decoder.
+     */
+    get framesSkippedDecoding(): number;
+    /**
+     * Gets the number of times the decoder has been reset to catch up new keyframes.
+     */
+    get decoderResetCount(): number;
+    constructor();
+}
+//# sourceMappingURL=video-decoder-stream.d.ts.map

package/esm/video/utils/video-decoder-stream.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"video-decoder-stream.d.ts","sourceRoot":"","sources":["../../../src/video/utils/video-decoder-stream.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,6CAA6C,EAAE,MAAM,kCAAkC,CAAC;AAEtG,OAAO,EAAiB,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAEzE,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAyE7D,qBAAa,kBACT,SAAQ,eAAe,CACnB,oBAAoB,CAAC,MAAM,GAAG,oBAAoB,CAAC,UAAU,EAC7D,UAAU,CAEd,YAAW,6CAA6C;;IAYxD,IAAI,oBAAoB,yBAEvB;IACD,IAAI,4BAA4B,2DAE/B;IAED;;OAEG;IACH,IAAI,eAAe,WAMlB;IAGD;;OAEG;IACH,IAAI,SAAS,gDAEZ;IAGD;;OAEG;IACH,IAAI,aAAa,WAEhB;IACD;;OAEG;IACH,IAAI,qBAAqB,WAExB;IACD;;OAEG;IACH,IAAI,iBAAiB,WAEpB;;CA0QJ"}

package/esm/video/utils/video-decoder-stream.js

@@ -0,0 +1,284 @@
+import { EventEmitter, StickyEventEmitter } from "@yume-chan/event";
+import { ScrcpyVideoDecoderPerformanceCounter } from "@yume-chan/scrcpy-decoder-shared";
+import { concatBuffers, TransformStream } from "@yume-chan/stream-extra";
+function decodeFirstKeyFrame(decoder, config, chunk) {
+    decoder.decode(new EncodedVideoChunk({
+        type: "key",
+        timestamp: chunk.timestamp,
+        // `lib.dom.d.ts` doesn't allow `duration` to be undefined.
+        duration: chunk.duration,
+        data: config.raw
+            ? concatBuffers([config.raw, chunk.data])
+            : chunk.data,
+    }));
+}
+function decodeFrame(decoder, chunk) {
+    decoder.decode(new EncodedVideoChunk({
+        type: chunk.type,
+        timestamp: chunk.timestamp,
+        // `lib.dom.d.ts` doesn't allow `duration` to be undefined.
+        duration: chunk.duration,
+        data: chunk.data,
+    }));
+}
+export class VideoDecoderStream extends TransformStream {
+    #controller;
+    #state = { type: "unconfigured" };
+    #hardwareAcceleration = new StickyEventEmitter({
+        initialValue: "no-preference",
+        equals: (a, b) => a === b,
+    });
+    get hardwareAcceleration() {
+        return this.#hardwareAcceleration.value;
+    }
+    get onHardwareAccelerationChange() {
+        return this.#hardwareAcceleration.event;
+    }
+    /**
+     * Gets the number of frames waiting to be decoded.
+     */
+    get decodeQueueSize() {
+        if ("decoder" in this.#state) {
+            return this.#state.decoder.decodeQueueSize;
+        }
+        return 0;
+    }
+    #onDequeue = new EventEmitter();
+    /**
+     * Gets an event when a frame is dequeued (either decoded or discarded).
+     */
+    get onDequeue() {
+        return this.#onDequeue.event;
+    }
+    #counter = new ScrcpyVideoDecoderPerformanceCounter();
+    /**
+     * Gets the number of frames decoded by the decoder.
+     */
+    get framesDecoded() {
+        return this.#counter.framesDecoded;
+    }
+    /**
+     * Gets the number of frames skipped by the decoder.
+     */
+    get framesSkippedDecoding() {
+        return this.#counter.framesSkippedDecoding;
+    }
+    /**
+     * Gets the number of times the decoder has been reset to catch up new keyframes.
+     */
+    get decoderResetCount() {
+        return this.#counter.decoderResetCount;
+    }
+    #abortController = new AbortController();
+    #skipFramesBefore = -1;
+    constructor() {
+        let controller;
+        super({
+            start: (controller_) => {
+                // WARN: can't use `this` here
+                controller = controller_;
+            },
+            transform: async (chunk) => {
+                if ("codec" in chunk) {
+                    this.#hardwareAcceleration.fire(chunk.hardwareAcceleration ?? "no-preference");
+                    // Validate the config by creating decoder right away
+                    switch (this.#state.type) {
+                        case "unconfigured":
+                        case "reclaimed":
+                            this.#state = {
+                                type: "configured",
+                                decoder: this.#createDecoder(chunk),
+                                config: chunk,
+                            };
+                            break;
+                        // @ts-expect-error: intentional case fallthrough
+                        case "decoding":
+                            await this.#state.decoder.flush();
+                        // fallthrough
+                        case "configured":
+                            // Reuse the existing decoder.
+                            // `decoder` is definitely in "configured" state,
+                            // otherwise the stream is in error state
+                            // and no `transform` callbacks will be called.
+                            this.#state.decoder.configure(chunk);
+                            this.#state = {
+                                type: "configured",
+                                decoder: this.#state.decoder,
+                                config: chunk,
+                            };
+                            break;
+                    }
+                    return;
+                }
+                this.#handleVideoChunk(chunk);
+            },
+            flush: async () => {
+                // only call `flush` when `decoder.state` is "configured".
+                if ("decoder" in this.#state &&
+                    this.#state.decoder.state === "configured") {
+                    // This also delays closing `readable` and `writable` streams
+                    // (if they are not in error state) until all frames are decoded.
+                    await this.#state.decoder.flush();
+                }
+                this.#dispose();
+            },
+            cancel: () => {
+                // Immediately close the decoder on stream cancel/error
+                this.#dispose();
+            },
+        });
+        this.#controller = controller;
+    }
+    #handleVideoChunk(chunk) {
+        switch (this.#state.type) {
+            case "unconfigured":
+                throw new Error("Decoder not configured");
+            case "configured": {
+                if (chunk.type === "delta") {
+                    throw new Error("Expect a keyframe but got a delta frame");
+                }
+                decodeFirstKeyFrame(this.#state.decoder, this.#state.config, chunk);
+                this.#state = {
+                    type: "decoding",
+                    decoder: this.#state.decoder,
+                    config: this.#state.config,
+                    frames: [chunk],
+                };
+                break;
+            }
+            case "decoding":
+                if (chunk.type === "key") {
+                    this.#state.frames.length = 0;
+                }
+                this.#state.frames.push(chunk);
+                decodeFrame(this.#state.decoder, chunk);
+                break;
+            case "reclaimed": {
+                const decoder = this.#createDecoder(this.#state.config);
+                if (chunk.type === "key") {
+                    decodeFirstKeyFrame(decoder, this.#state.config, chunk);
+                    this.#state = {
+                        type: "decoding",
+                        decoder: decoder,
+                        config: this.#state.config,
+                        frames: [chunk],
+                    };
+                    return;
+                }
+                if (!this.#state.frames) {
+                    // Reclaimed when configured but no frames have been decoded yet,
+                    // so a keyframe is required to start decoding.
+                    throw new Error("Expect a keyframe but got a delta frame");
+                }
+                decodeFirstKeyFrame(decoder, this.#state.config, this.#state.frames[0]);
+                // `chunk`s must have monotonic timestamps, so we can skip all frames before `chunk`.
+                this.#skipFramesBefore = chunk.timestamp;
+                this.#state.frames.push(chunk);
+                for (const frame of this.#state.frames.slice(1)) {
+                    decodeFrame(decoder, frame);
+                }
+                this.#state = {
+                    type: "decoding",
+                    decoder: decoder,
+                    config: this.#state.config,
+                    frames: this.#state.frames,
+                };
+                break;
+            }
+        }
+    }
+    #createDecoder(config) {
+        const decoder = new VideoDecoder({
+            output: (frame) => {
+                if (frame.timestamp < this.#skipFramesBefore) {
+                    frame.close();
+                    return;
+                }
+                this.#counter.increaseFramesDecoded();
+                try {
+                    this.#controller.enqueue(frame);
+                }
+                catch {
+                    // `controller` is closed
+                    frame.close();
+                }
+            },
+            error: (error) => {
+                if (error.name === "QuotaExceededError") {
+                    // Chrome reclaims inactive `VideoDecoder`'s after 90 seconds
+                    // (for example due to pausing decoding when the document is hidden).
+                    // https://source.chromium.org/chromium/chromium/src/+/main:third_party/blink/renderer/modules/webcodecs/reclaimable_codec.cc;l=181;drc=aba14550ed0b4620deb59e7c5f551cbde8970fb3
+                    // Reset states, it will be recreated in next `transform` call.
+                    switch (this.#state.type) {
+                        case "unconfigured":
+                        case "reclaimed":
+                            // Don't throw, report errors to stream
+                            this.#controller.error(new Error("Unexpected state when reclaiming decoder"));
+                            this.#dispose();
+                            return;
+                        case "configured":
+                            // Unlike `transform` callback,
+                            // this doesn't re-create the decoder immediately,
+                            // because the config has already been validated,
+                            // and since browser reclaims the decoder to save hardware resources,
+                            // we don't want to undo that by creating a new decoder immediately.
+                            this.#state = {
+                                type: "reclaimed",
+                                config,
+                                frames: undefined,
+                            };
+                            break;
+                        case "decoding":
+                            this.#state = {
+                                type: "reclaimed",
+                                config,
+                                frames: this.#state.frames,
+                            };
+                            break;
+                    }
+                    return;
+                }
+                // Maybe the decoder is hardware accelerated but the hardware has an issue,
+                // retry with software decoder.
+                if (error.name === "EncodingError" &&
+                    this.#state.type === "decoding" &&
+                    config.hardwareAcceleration !== "prefer-software") {
+                    try {
+                        config.hardwareAcceleration = "prefer-software";
+                        this.#hardwareAcceleration.fire("prefer-software");
+                        const decoder = this.#createDecoder(config);
+                        // Replay frames with rendering skipping
+                        this.#skipFramesBefore =
+                            this.#state.frames.at(-1).timestamp;
+                        decodeFirstKeyFrame(decoder, config, this.#state.frames[0]);
+                        for (const frame of this.#state.frames.slice(1)) {
+                            decodeFrame(decoder, frame);
+                        }
+                        // Update state
+                        this.#state.decoder = decoder;
+                        return;
+                    }
+                    catch {
+                        // ignore, report original error to stream
+                    }
+                }
+                // Propagate other decoder errors to stream.
+                this.#controller.error(error);
+                this.#dispose();
+            },
+        });
+        decoder.addEventListener("dequeue", () => this.#onDequeue.fire(undefined), { signal: this.#abortController.signal });
+        decoder.configure(config);
+        return decoder;
+    }
+    #dispose() {
+        this.#abortController.abort();
+        this.#onDequeue.dispose();
+        if ("decoder" in this.#state &&
+            this.#state.decoder.state !== "closed") {
+            this.#state.decoder.close();
+        }
+        this.#state = { type: "unconfigured" };
+    }
+}
+//# sourceMappingURL=video-decoder-stream.js.map

package/esm/video/utils/video-decoder-stream.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"video-decoder-stream.js","sourceRoot":"","sources":["../../../src/video/utils/video-decoder-stream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEpE,OAAO,EAAE,oCAAoC,EAAE,MAAM,kCAAkC,CAAC;AACxF,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AA0CzE,SAAS,mBAAmB,CACxB,OAAqB,EACrB,MAAmC,EACnC,KAAsC;IAEtC,OAAO,CAAC,MAAM,CACV,IAAI,iBAAiB,CAAC;QAClB,IAAI,EAAE,KAAK;QACX,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,2DAA2D;QAC3D,QAAQ,EAAE,KAAK,CAAC,QAAS;QACzB,IAAI,EAAE,MAAM,CAAC,GAAG;YACZ,CAAC,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACzC,CAAC,CAAC,KAAK,CAAC,IAAI;KACnB,CAAC,CACL,CAAC;AACN,CAAC;AAED,SAAS,WAAW,CAChB,OAAqB,EACrB,KAAsC;IAEtC,OAAO,CAAC,MAAM,CACV,IAAI,iBAAiB,CAAC;QAClB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,2DAA2D;QAC3D,QAAQ,EAAE,KAAK,CAAC,QAAS;QACzB,IAAI,EAAE,KAAK,CAAC,IAAI;KACnB,CAAC,CACL,CAAC;AACN,CAAC;AAED,MAAM,OAAO,kBACT,SAAQ,eAGP;IAGD,WAAW,CAAgD;IAE3D,MAAM,GAAU,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC;IAEzC,qBAAqB,GAAG,IAAI,kBAAkB,CAE5C;QACE,YAAY,EAAE,eAAe;QAC7B,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC;KAC5B,CAAC,CAAC;IACH,IAAI,oBAAoB;QACpB,OAAO,IAAI,CAAC,qBAAqB,CAAC,KAAK,CAAC;IAC5C,CAAC;IACD,IAAI,4BAA4B;QAC5B,OAAO,IAAI,CAAC,qBAAqB,CAAC,KAAK,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,IAAI,eAAe;QACf,IAAI,SAAS,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC;QAC/C,CAAC;QAED,OAAO,CAAC,CAAC;IACb,CAAC;IAED,UAAU,GAAG,IAAI,YAAY,EAAa,CAAC;IAC3C;;OAEG;IACH,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;IACjC,CAAC;IAED,QAAQ,GAAG,IAAI,oCAAoC,EAAE,CAAC;IACtD;;OAEG;IACH,IAAI,aAAa;QACb,OAAO,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC;IACvC,CAAC;IACD;;OAEG;IACH,IAAI,qBAAqB;QACrB,OAAO,IAAI,CAAC,QAAQ,CAAC,qBAAqB,CAAC;IAC/C,CAAC;IACD;;OAEG;IACH,IAAI,iBAAiB;QACjB,OAAO,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAC3C,CAAC;IAED,gBAAgB,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,iBAAiB,GAAG,CAAC,CAAC,CAAC;IAEvB;QACI,IAAI,UAAyD,CAAC;QAE9D,KAAK,CAAC;YACF,KAAK,EAAE,CAAC,WAAW,EAAE,EAAE;gBACnB,8BAA8B;gBAC9B,UAAU,GAAG,WAAW,CAAC;YAC7B,CAAC;YACD,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;gBACvB,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;oBACnB,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAC3B,KAAK,CAAC,oBAAoB,IAAI,eAAe,CAChD,CAAC;oBACF,qDAAqD;oBACrD,QAAQ,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;wBACvB,KAAK,cAAc,CAAC;wBACpB,KAAK,WAAW;4BACZ,IAAI,CAAC,MAAM,GAAG;gCACV,IAAI,EAAE,YAAY;gCAClB,OAAO,EAAE,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC;gCACnC,MAAM,EAAE,KAAK;6BAChB,CAAC;4BACF,MAAM;wBACV,iDAAiD;wBACjD,KAAK,UAAU;4BACX,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;wBACtC,cAAc;wBACd,KAAK,YAAY;4BACb,8BAA8B;4BAC9B,iDAAiD;4BACjD,yCAAyC;4BACzC,+CAA+C;4BAC/C,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;4BACrC,IAAI,CAAC,MAAM,GAAG;gCACV,IAAI,EAAE,YAAY;gCAClB,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;gCAC5B,MAAM,EAAE,KAAK;6BAChB,CAAC;4BACF,MAAM;oBACd,CAAC;oBACD,OAAO;gBACX,CAAC;gBAED,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;YAClC,CAAC;YACD,KAAK,EAAE,KAAK,IAAI,EAAE;gBACd,0DAA0D;gBAC1D,IACI,SAAS,IAAI,IAAI,CAAC,MAAM;oBACxB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,YAAY,EAC5C,CAAC;oBACC,6DAA6D;oBAC7D,iEAAiE;oBACjE,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACtC,CAAC;gBACD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpB,CAAC;YACD,MAAM,EAAE,GAAG,EAAE;gBACT,uDAAuD;gBACvD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpB,CAAC;SACJ,CAAC,CAAC;QAEH,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAClC,CAAC;IAED,iBAAiB,CAAC,KAAsC;QACpD,QAAQ,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACvB,KAAK,cAAc;gBACf,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;YAC9C,KAAK,YAAY,CAAC,CAAC,CAAC;gBAChB,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;oBACzB,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;gBAC/D,CAAC;gBAED,mBAAmB,CACf,IAAI,CAAC,MAAM,CAAC,OAAO,EACnB,IAAI,CAAC,MAAM,CAAC,MAAM,EAClB,KAAK,CACR,CAAC;gBAEF,IAAI,CAAC,MAAM,GAAG;oBACV,IAAI,EAAE,UAAU;oBAChB,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;oBAC5B,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;oBAC1B,MAAM,EAAE,CAAC,KAAK,CAAC;iBAClB,CAAC;gBACF,MAAM;YACV,CAAC;YACD,KAAK,UAAU;gBACX,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;oBACvB,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;gBAClC,CAAC;gBACD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAE/B,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBACxC,MAAM;YACV,KAAK,WAAW,CAAC,CAAC,CAAC;gBACf,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAExD,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;oBACvB,mBAAmB,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;oBAExD,IAAI,CAAC,MAAM,GAAG;wBACV,IAAI,EAAE,UAAU;wBAChB,OAAO,EAAE,OAAO;wBAChB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;wBAC1B,MAAM,EAAE,CAAC,KAAK,CAAC;qBAClB,CAAC;oBACF,OAAO;gBACX,CAAC;gBAED,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;oBACtB,iEAAiE;oBACjE,+CAA+C;oBAC/C,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;gBAC/D,CAAC;gBAED,mBAAmB,CACf,OAAO,EACP,IAAI,CAAC,MAAM,CAAC,MAAM,EAClB,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CACzB,CAAC;gBAEF,qFAAqF;gBACrF,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC,SAAS,CAAC;gBACzC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAE/B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC9C,WAAW,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBAChC,CAAC;gBAED,IAAI,CAAC,MAAM,GAAG;oBACV,IAAI,EAAE,UAAU;oBAChB,OAAO,EAAE,OAAO;oBAChB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;oBAC1B,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;iBAC7B,CAAC;gBAEF,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;IAED,cAAc,CAAC,MAAmC;QAC9C,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC;YAC7B,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;gBACd,IAAI,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;oBAC3C,KAAK,CAAC,KAAK,EAAE,CAAC;oBACd,OAAO;gBACX,CAAC;gBAED,IAAI,CAAC,QAAQ,CAAC,qBAAqB,EAAE,CAAC;gBACtC,IAAI,CAAC;oBACD,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBACpC,CAAC;gBAAC,MAAM,CAAC;oBACL,yBAAyB;oBACzB,KAAK,CAAC,KAAK,EAAE,CAAC;gBAClB,CAAC;YACL,CAAC;YACD,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE;gBACb,IAAI,KAAK,CAAC,IAAI,KAAK,oBAAoB,EAAE,CAAC;oBACtC,6DAA6D;oBAC7D,qEAAqE;oBACrE,gLAAgL;oBAChL,+DAA+D;oBAC/D,QAAQ,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;wBACvB,KAAK,cAAc,CAAC;wBACpB,KAAK,WAAW;4BACZ,uCAAuC;4BACvC,IAAI,CAAC,WAAW,CAAC,KAAK,CAClB,IAAI,KAAK,CACL,0CAA0C,CAC7C,CACJ,CAAC;4BACF,IAAI,CAAC,QAAQ,EAAE,CAAC;4BAChB,OAAO;wBACX,KAAK,YAAY;4BACb,+BAA+B;4BAC/B,kDAAkD;4BAClD,iDAAiD;4BACjD,qEAAqE;4BACrE,oEAAoE;4BACpE,IAAI,CAAC,MAAM,GAAG;gCACV,IAAI,EAAE,WAAW;gCACjB,MAAM;gCACN,MAAM,EAAE,SAAS;6BACpB,CAAC;4BACF,MAAM;wBACV,KAAK,UAAU;4BACX,IAAI,CAAC,MAAM,GAAG;gCACV,IAAI,EAAE,WAAW;gCACjB,MAAM;gCACN,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;6BAC7B,CAAC;4BACF,MAAM;oBACd,CAAC;oBACD,OAAO;gBACX,CAAC;gBAED,2EAA2E;gBAC3E,+BAA+B;gBAC/B,IACI,KAAK,CAAC,IAAI,KAAK,eAAe;oBAC9B,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,UAAU;oBAC/B,MAAM,CAAC,oBAAoB,KAAK,iBAAiB,EACnD,CAAC;oBACC,IAAI,CAAC;wBACD,MAAM,CAAC,oBAAoB,GAAG,iBAAiB,CAAC;wBAChD,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;wBACnD,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;wBAE5C,wCAAwC;wBACxC,IAAI,CAAC,iBAAiB;4BAClB,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,SAAS,CAAC;wBACzC,mBAAmB,CACf,OAAO,EACP,MAAM,EACN,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CACzB,CAAC;wBACF,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;4BAC9C,WAAW,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;wBAChC,CAAC;wBAED,eAAe;wBACf,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC;wBAC9B,OAAO;oBACX,CAAC;oBAAC,MAAM,CAAC;wBACL,0CAA0C;oBAC9C,CAAC;gBACL,CAAC;gBAED,4CAA4C;gBAC5C,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC9B,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpB,CAAC;SACJ,CAAC,CAAC;QAEH,OAAO,CAAC,gBAAgB,CACpB,SAAS,EACT,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,EACrC,EAAE,MAAM,EAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,CAC3C,CAAC;QAEF,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAE1B,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,QAAQ;QACJ,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;QAC9B,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QAE1B,IACI,SAAS,IAAI,IAAI,CAAC,MAAM;YACxB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,QAAQ,EACxC,CAAC;YACC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QAChC,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC;IAC3C,CAAC;CACJ"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yume-chan/scrcpy-decoder-webcodecs",
-  "version": "2.5.3",
+  "version": "3.0.0-beta.0",
   "description": "Raw H.264 stream decoder and renderer using WebCodecs API (requires modern browser).",
   "keywords": [
     "adb",
@@ -14,11 +14,11 @@
     "email": "cnsimonchan@live.com",
     "url": "https://chensi.moe/blog"
   },
-  "homepage": "https://github.com/yume-chan/ya-webadb/tree/main/packages/scrcpy-decoder-webcodecs#readme",
+  "homepage": "https://github.com/yume-chan/ya-webadb/tree/main/libraries/scrcpy-decoder-webcodecs#readme",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/yume-chan/ya-webadb.git",
-    "directory": "packages/scrcpy-decoder-webcodecs"
+    "directory": "libraries/scrcpy-decoder-webcodecs"
   },
   "bugs": {
     "url": "https://github.com/yume-chan/ya-webadb/issues"
@@ -29,20 +29,21 @@
   "sideEffects": false,
   "dependencies": {
     "@yume-chan/async": "^4.1.3",
-    "@yume-chan/event": "^2.0.0",
-    "@yume-chan/scrcpy": "^2.3.0",
-    "@yume-chan/no-data-view": "^2.0.0",
-    "@yume-chan/scrcpy-decoder-tinyh264": "^2.1.0",
-    "@yume-chan/stream-extra": "^2.5.3"
+    "@yume-chan/media-codec": "3.0.0-beta.0",
+    "@yume-chan/event": "3.0.0-beta.0",
+    "@yume-chan/no-data-view": "3.0.0-beta.0",
+    "@yume-chan/scrcpy": "3.0.0-beta.0",
+    "@yume-chan/scrcpy-decoder-shared": "3.0.0-beta.0",
+    "@yume-chan/stream-extra": "3.0.0-beta.0"
   },
   "devDependencies": {
-    "prettier": "^3.6.2",
-    "typescript": "^5.9.2",
+    "prettier": "^3.8.4",
+    "typescript": "^6.0.3",
     "@yume-chan/eslint-config": "^1.0.0",
     "@yume-chan/tsconfig": "^1.0.0"
   },
   "scripts": {
-    "build": "tsc -b tsconfig.build.json",
+    "build": "tsc -p tsconfig.build.json",
     "lint": "run-eslint && prettier src/**/*.ts --write --tab-width 4"
   }
 }

package/src/video/codec/av1.ts

@@ -1,134 +1,44 @@
-import type { ScrcpyMediaStreamPacket } from "@yume-chan/scrcpy";
-import { Av1 } from "@yume-chan/scrcpy";
-
-import type { CodecDecoder, CodecDecoderOptions } from "./type.js";
-import { decimalTwoDigits } from "./utils.js";
-
-export class Av1Codec implements CodecDecoder {
-    #decoder: VideoDecoder;
-    #updateSize: (width: number, height: number) => void;
-    #options: CodecDecoderOptions | undefined;
-
-    #config: VideoDecoderConfig | undefined;
-    #configured = false;
-
-    constructor(
-        decoder: VideoDecoder,
-        updateSize: (width: number, height: number) => void,
-        options?: CodecDecoderOptions,
-    ) {
-        this.#decoder = decoder;
-        this.#updateSize = updateSize;
-        this.#options = options;
-    }
-
-    #parseConfig(data: Uint8Array) {
-        const parser = new Av1(data);
-        const sequenceHeader = parser.searchSequenceHeaderObu();
-
-        if (!sequenceHeader) {
-            return;
-        }
-
-        const {
-            seq_profile: seqProfile,
-            seq_level_idx: [seqLevelIdx = 0],
-            max_frame_width_minus_1,
-            max_frame_height_minus_1,
-            color_config: {
-                BitDepth,
-                mono_chrome: monoChrome,
-                subsampling_x: subsamplingX,
-                subsampling_y: subsamplingY,
-                chroma_sample_position: chromaSamplePosition,
-                color_description_present_flag,
+import { Av1 } from "@yume-chan/media-codec";
+import { TransformStream } from "@yume-chan/stream-extra";
+
+import type { CodecTransformStream } from "./type.js";
+
+export class Av1TransformStream
+    extends TransformStream<
+        CodecTransformStream.Input,
+        CodecTransformStream.Output
+    >
+    implements CodecTransformStream
+{
+    constructor() {
+        super({
+            transform: (packet, controller) => {
+                if (packet.type === "configuration") {
+                    // AV1 decoder doesn't need configuration packets
+                    return;
+                }
+
+                const parser = new Av1(packet.data);
+                const sequenceHeader = parser.searchSequenceHeaderObu();
+
+                if (sequenceHeader) {
+                    const width = sequenceHeader.max_frame_width_minus_1 + 1;
+                    const height = sequenceHeader.max_frame_height_minus_1 + 1;
+
+                    controller.enqueue({
+                        codec: Av1.toCodecString(sequenceHeader),
+                        codedWidth: width,
+                        codedHeight: height,
+                    });
+                }
+
+                controller.enqueue({
+                    // AV1 was added in Scrcpy 2.0 which must have `keyframe` property.
+                    type: packet.keyframe ? "key" : "delta",
+                    timestamp: packet.timestamp,
+                    data: packet.data,
+                });
             },
-        } = sequenceHeader;
-
-        let colorPrimaries: Av1.ColorPrimaries;
-        let transferCharacteristics: Av1.TransferCharacteristics;
-        let matrixCoefficients: Av1.MatrixCoefficients;
-        let colorRange: boolean;
-        if (color_description_present_flag) {
-            ({
-                color_primaries: colorPrimaries,
-                transfer_characteristics: transferCharacteristics,
-                matrix_coefficients: matrixCoefficients,
-                color_range: colorRange,
-            } = sequenceHeader.color_config);
-        } else {
-            colorPrimaries = Av1.ColorPrimaries.Bt709;
-            transferCharacteristics = Av1.TransferCharacteristics.Bt709;
-            matrixCoefficients = Av1.MatrixCoefficients.Bt709;
-            colorRange = false;
-        }
-
-        const width = max_frame_width_minus_1 + 1;
-        const height = max_frame_height_minus_1 + 1;
-
-        this.#updateSize(width, height);
-
-        const codec = [
-            "av01",
-            seqProfile.toString(16),
-            decimalTwoDigits(seqLevelIdx) +
-                (sequenceHeader.seq_tier[0] ? "H" : "M"),
-            decimalTwoDigits(BitDepth),
-            monoChrome ? "1" : "0",
-            (subsamplingX ? "1" : "0") +
-                (subsamplingY ? "1" : "0") +
-                chromaSamplePosition.toString(),
-            decimalTwoDigits(colorPrimaries),
-            decimalTwoDigits(transferCharacteristics),
-            decimalTwoDigits(matrixCoefficients),
-            colorRange ? "1" : "0",
-        ].join(".");
-        this.#config = {
-            codec,
-            hardwareAcceleration:
-                this.#options?.hardwareAcceleration ?? "no-preference",
-            optimizeForLatency: true,
-        };
-        this.#configured = false;
-    }
-
-    decode(packet: ScrcpyMediaStreamPacket): void {
-        if (packet.type === "configuration") {
-            return;
-        }
-
-        this.#parseConfig(packet.data);
-
-        if (!this.#config) {
-            throw new Error("Decoder not configured");
-        }
-
-        if (packet.keyframe) {
-            if (this.#decoder.decodeQueueSize) {
-                // If the device is too slow to decode all frames,
-                // discard queued frames when next keyframe arrives.
-                // (can only do this for keyframes because decoding must start from a keyframe)
-                // This limits the maximum latency to 1 keyframe interval
-                // (60 frames by default).
-                this.#decoder.reset();
-
-                // `reset` also resets the decoder configuration
-                // so we need to re-configure it again.
-                this.#decoder.configure(this.#config);
-                this.#configured = true;
-            } else if (!this.#configured) {
-                this.#decoder.configure(this.#config);
-                this.#configured = true;
-            }
-        }
-
-        this.#decoder.decode(
-            new EncodedVideoChunk({
-                // AV1 requires Scrcpy 2.0 where `keyframe` flag must be set
-                type: packet.keyframe! ? "key" : "delta",
-                timestamp: 0,
-                data: packet.data,
-            }),
-        );
+        });
     }
 }