@yume-chan/scrcpy 0.0.11

package/src/connection.ts

@@ -0,0 +1,123 @@
+import { Adb, AdbBufferedStream, AdbSocket } from "@yume-chan/adb";
+import type { Disposable } from "@yume-chan/event";
+import type { ValueOrPromise } from "@yume-chan/struct";
+import { delay } from "./utils.js";
+
+export interface ScrcpyClientConnectionOptions {
+    control: boolean;
+
+    /**
+     * Write a byte on start to detect connection issues
+     */
+    sendDummyByte: boolean;
+
+    /**
+     * Send device name and size
+     */
+    sendDeviceMeta: boolean;
+}
+
+export abstract class ScrcpyClientConnection implements Disposable {
+    protected device: Adb;
+
+    protected options: ScrcpyClientConnectionOptions;
+
+    public constructor(device: Adb, options: ScrcpyClientConnectionOptions) {
+        this.device = device;
+        this.options = options;
+    }
+
+    public initialize(): ValueOrPromise<void> { }
+
+    public abstract getStreams(): ValueOrPromise<[videoSteam: AdbBufferedStream, controlStream: AdbSocket | undefined]>;
+
+    public dispose(): void { }
+}
+
+export class ScrcpyClientForwardConnection extends ScrcpyClientConnection {
+    private async connect(): Promise<AdbSocket> {
+        return await this.device.createSocket('localabstract:scrcpy');
+    }
+
+    private async connectAndRetry(): Promise<AdbSocket> {
+        for (let i = 0; i < 100; i++) {
+            try {
+                return await this.connect();
+            } catch (e) {
+                await delay(100);
+            }
+        }
+        throw new Error(`Can't connect to server after 100 retries`);
+    }
+
+    private async connectVideoStream(): Promise<AdbBufferedStream> {
+        const socket = await this.connectAndRetry();
+        const stream = new AdbBufferedStream(socket);
+        if (this.options.sendDummyByte) {
+            // server will write a `0` to signal connection success
+            await stream.read(1);
+        }
+        return stream;
+    }
+
+    public async getStreams(): Promise<[videoSteam: AdbBufferedStream, controlStream: AdbSocket | undefined]> {
+        const videoStream = await this.connectVideoStream();
+        let controlStream: AdbSocket | undefined;
+        if (this.options.control) {
+            controlStream = await this.connectAndRetry();
+        }
+        if (this.options.sendDeviceMeta) {
+            // 64 bytes device name + 2 bytes video width + 2 bytes video height
+            await videoStream.read(64 + 2 + 2);
+        }
+        return [videoStream, controlStream];
+    }
+}
+
+export class ScrcpyClientReverseConnection extends ScrcpyClientConnection {
+    private streams!: ReadableStreamDefaultReader<AdbSocket>;
+
+    private address!: string;
+
+    public override async initialize(): Promise<void> {
+        try {
+            // try to unbind first
+            await this.device.reverse.remove('localabstract:scrcpy');
+        } catch {
+            // ignore error
+        }
+
+        const queue = new TransformStream<AdbSocket>();
+        this.streams = queue.readable.getReader();
+        const writer = queue.writable.getWriter();
+        this.address = await this.device.reverse.add('localabstract:scrcpy', 27183, {
+            onSocket: (packet, stream) => {
+                writer.write(stream);
+            },
+        });
+    }
+
+    private async accept(): Promise<AdbSocket> {
+        return (await this.streams.read()).value!;
+    }
+
+    public async getStreams(): Promise<[videoSteam: AdbBufferedStream, controlStream: AdbSocket | undefined]> {
+        const videoStream = new AdbBufferedStream(await this.accept());
+        let controlStream: AdbSocket | undefined;
+        if (this.options.control) {
+            controlStream = await this.accept();
+        }
+        if (this.options.sendDeviceMeta) {
+            // 64 bytes device name + 2 bytes video width + 2 bytes video height
+            await videoStream.read(64 + 2 + 2);
+        }
+        return [videoStream, controlStream];
+    }
+
+    public override dispose() {
+        // Don't await this!
+        // `reverse.remove`'s response will never arrive
+        // before we read all pending data from `videoStream`
+        this.device.reverse.remove(this.address);
+    }
+}

package/src/decoder/index.ts

@@ -0,0 +1,3 @@
+export * from './types.js';
+export * from './tinyh264/index.js';
+export * from './web-codecs/index.js';

package/src/decoder/tinyh264/index.ts

@@ -0,0 +1,112 @@
+import { WritableStream } from "@yume-chan/adb";
+import { PromiseResolver } from "@yume-chan/async";
+import { AndroidCodecLevel, AndroidCodecProfile } from "../../codec.js";
+import type { VideoStreamPacket } from "../../options/index.js";
+import type { H264Configuration, H264Decoder } from "../types.js";
+import { createTinyH264Wrapper, type TinyH264Wrapper } from "./wrapper.js";
+
+let cachedInitializePromise: Promise<{ YuvBuffer: typeof import('yuv-buffer'), YuvCanvas: typeof import('yuv-canvas').default; }> | undefined;
+function initialize() {
+    if (!cachedInitializePromise) {
+        cachedInitializePromise = Promise.all(
+            [import('yuv-buffer'), import('yuv-canvas')]
+        ).then(([YuvBuffer, { default: YuvCanvas }]) => ({
+            YuvBuffer, YuvCanvas
+        }));
+    }
+
+    return cachedInitializePromise;
+}
+
+export class TinyH264Decoder implements H264Decoder {
+    public readonly maxProfile = AndroidCodecProfile.Baseline;
+
+    public readonly maxLevel = AndroidCodecLevel.Level4;
+
+    private _renderer: HTMLCanvasElement;
+    public get renderer() { return this._renderer; }
+
+    private _frameRendered = 0;
+    public get frameRendered() { return this._frameRendered; }
+
+    private _writable: WritableStream<VideoStreamPacket>;
+    public get writable() { return this._writable; }
+
+    private _yuvCanvas: import('yuv-canvas').default | undefined;
+    private _initializer: PromiseResolver<TinyH264Wrapper> | undefined;
+
+    public constructor() {
+        initialize();
+
+        this._renderer = document.createElement('canvas');
+
+        this._writable = new WritableStream<VideoStreamPacket>({
+            write: async (packet) => {
+                switch (packet.type) {
+                    case 'configuration':
+                        this.configure(packet.data);
+                        break;
+                    case 'frame':
+                        if (!this._initializer) {
+                            throw new Error('Decoder not initialized');
+                        }
+
+                        const wrapper = await this._initializer.promise;
+                        wrapper.feed(packet.data.slice().buffer);
+                        break;
+                }
+            }
+        });
+    }
+
+    private async configure(config: H264Configuration) {
+        this.dispose();
+
+        this._initializer = new PromiseResolver<TinyH264Wrapper>();
+        const { YuvBuffer, YuvCanvas } = await initialize();
+
+        if (!this._yuvCanvas) {
+            this._yuvCanvas = YuvCanvas.attach(this._renderer);;
+        }
+
+        const { encodedWidth, encodedHeight } = config;
+        const chromaWidth = encodedWidth / 2;
+        const chromaHeight = encodedHeight / 2;
+
+        this._renderer.width = config.croppedWidth;
+        this._renderer.height = config.croppedHeight;
+        const format = YuvBuffer.format({
+            width: encodedWidth,
+            height: encodedHeight,
+            chromaWidth,
+            chromaHeight,
+            cropLeft: config.cropLeft,
+            cropTop: config.cropTop,
+            cropWidth: config.croppedWidth,
+            cropHeight: config.croppedHeight,
+            displayWidth: config.croppedWidth,
+            displayHeight: config.croppedHeight,
+        });
+
+        const wrapper = await createTinyH264Wrapper();
+        this._initializer.resolve(wrapper);
+
+        const uPlaneOffset = encodedWidth * encodedHeight;
+        const vPlaneOffset = uPlaneOffset + chromaWidth * chromaHeight;
+        wrapper.onPictureReady(({ data }) => {
+            this._frameRendered += 1;
+            const array = new Uint8Array(data);
+            const frame = YuvBuffer.frame(format,
+                YuvBuffer.lumaPlane(format, array, encodedWidth, 0),
+                YuvBuffer.chromaPlane(format, array, chromaWidth, uPlaneOffset),
+                YuvBuffer.chromaPlane(format, array, chromaWidth, vPlaneOffset)
+            );
+            this._yuvCanvas!.drawFrame(frame);
+        });
+    }
+
+    public dispose(): void {
+        this._initializer?.promise.then(wrapper => wrapper.dispose());
+        this._initializer = undefined;
+    }
+}

package/src/decoder/tinyh264/types.d.ts

@@ -0,0 +1,137 @@
+declare module 'tinyh264' {
+    export function init(): void;
+}
+
+declare module 'yuv-buffer' {
+    /**
+     * Validate and fill out a YUVFormat object structure.
+     *
+     * At least width and height fields are required; other fields will be
+     * derived if left missing or empty:
+     * - chromaWidth and chromaHeight will be copied from width and height as for a 4:4:4 layout
+     * - cropLeft and cropTop will be 0
+     * - cropWidth and cropHeight will be set to whatever of the frame is visible after cropTop and cropLeft are applied
+     * - displayWidth and displayHeight will be set to cropWidth and cropHeight.
+     *
+     * @param {YUVFormat} fields - input fields, must include width and height.
+     * @returns {YUVFormat} - validated structure, with all derivable fields filled out.
+     * @throws exception on invalid fields or missing width/height
+     */
+    export function format(fields: YUVFormat): YUVFormat;
+
+    /**
+     * Allocate a new YUVPlane object big enough for a luma plane in the given format
+     * @param {YUVFormat} format - target frame format
+     * @param {Uint8Array} source - input byte array; optional (will create empty buffer if missing)
+     * @param {number} stride - row length in bytes; optional (will create a default if missing)
+     * @param {number} offset - offset into source array to extract; optional (will start at 0 if missing)
+     * @returns {YUVPlane} - freshly allocated planar buffer
+     */
+    export function lumaPlane(format: YUVFormat, source: Uint8Array, stride: number, offset: number): YUVPlane;
+
+    /**
+     * Allocate a new YUVPlane object big enough for a chroma plane in the given format,
+     * optionally copying data from an existing buffer.
+     *
+     * @param {YUVFormat} format - target frame format
+     * @param {Uint8Array} source - input byte array; optional (will create empty buffer if missing)
+     * @param {number} stride - row length in bytes; optional (will create a default if missing)
+     * @param {number} offset - offset into source array to extract; optional (will start at 0 if missing)
+     * @returns {YUVPlane} - freshly allocated planar buffer
+     */
+    export function chromaPlane(format: YUVFormat, source: Uint8Array, stride: number, offset: number): YUVPlane;
+
+    /**
+     * Allocate a new YUVFrame object big enough for the given format
+     * @param {YUVFormat} format - target frame format
+     * @param {YUVPlane} y - optional Y plane; if missing, fresh one will be allocated
+     * @param {YUVPlane} u - optional U plane; if missing, fresh one will be allocated
+     * @param {YUVPlane} v - optional V plane; if missing, fresh one will be allocated
+     * @returns {YUVFrame} - freshly allocated frame buffer
+     */
+    export function frame(format: YUVFormat, y: YUVPlane, u: YUVPlane, v: YUVPlane): YUVFrame;
+
+    /**
+     * Duplicate a frame using new buffer memory.
+     * @param {YUVFrame} frame - input frame to copyFrame
+     * @returns {YUVFrame} - freshly allocated and filled frame buffer
+     */
+    export function copyFrame(frame: YUVFrame): YUVFrame;
+
+    /**
+     * List the backing buffers for the frame's planes for transfer between
+     * threads via Worker.postMessage.
+     * @param {YUVFrame} frame - input frame
+     * @returns {Array} - list of transferable objects
+     */
+    export function transferables(frame: YUVFrame): (ArrayBuffer | SharedArrayBuffer)[];
+
+
+    /**
+     * Represents metadata about a YUV frame format.
+     * @typedef {Object} YUVFormat
+     * @property {number} width - width of encoded frame in luma pixels
+     * @property {number} height - height of encoded frame in luma pixels
+     * @property {number} chromaWidth - width of encoded frame in chroma pixels
+     * @property {number} chromaHeight - height of encoded frame in chroma pixels
+     * @property {number} cropLeft - upper-left X coordinate of visible crop region, in luma pixels
+     * @property {number} cropTop - upper-left Y coordinate of visible crop region, in luma pixels
+     * @property {number} cropWidth - width of visible crop region, in luma pixels
+     * @property {number} cropHeight - height of visible crop region, in luma pixels
+     * @property {number} displayWidth - final display width of visible region, in luma pixels
+     * @property {number} displayHeight - final display height of visible region, in luma pixels
+     */
+    export interface YUVFormat {
+        width: number;
+        height: number;
+        chromaWidth: number;
+        chromaHeight: number;
+        cropLeft: number;
+        cropTop: number;
+        cropWidth: number;
+        cropHeight: number;
+        displayWidth: number;
+        displayHeight: number;
+    }
+
+    /**
+     * Represents underlying image data for a single luma or chroma plane.
+     * Cannot be interpreted without the format data from a frame buffer.
+     * @typedef {Object} YUVPlane
+     * @property {Uint8Array} bytes - typed array containing image data bytes
+     * @property {number} stride - byte distance between rows in data
+     */
+    export interface YUVPlane {
+        bytes: Uint8Array;
+        stride: number;
+    }
+
+    /**
+     * Represents a YUV image frame buffer, with enough format information
+     * to interpret the data usefully. Buffer objects use generic objects
+     * under the hood and can be transferred between worker threads using
+     * the structured clone algorithm.
+     *
+     * @typedef {Object} YUVFrame
+     * @property {YUVFormat} format
+     * @property {YUVPlane} y
+     * @property {YUVPlane} u
+     * @property {YUVPlane} v
+     */
+    export interface YUVFrame {
+        format: YUVFormat;
+        y: YUVPlane;
+        u: YUVPlane;
+        v: YUVPlane;
+    }
+}
+
+declare module 'yuv-canvas' {
+    import { YUVFrame } from 'yuv-buffer';
+
+    export default class YUVCanvas {
+        public static attach(canvas: HTMLCanvasElement): YUVCanvas;
+
+        public drawFrame(data: YUVFrame): void;
+    }
+}

package/src/decoder/tinyh264/worker.ts

@@ -0,0 +1,2 @@
+import { init } from 'tinyh264';
+init();

package/src/decoder/tinyh264/wrapper.ts

@@ -0,0 +1,89 @@
+import { PromiseResolver } from '@yume-chan/async';
+import { AutoDisposable, EventEmitter } from '@yume-chan/event';
+
+let worker: Worker | undefined;
+let workerReady = false;
+const pendingResolvers: PromiseResolver<TinyH264Wrapper>[] = [];
+let streamId = 0;
+
+export interface PictureReadyEventArgs {
+    renderStateId: number;
+
+    width: number;
+
+    height: number;
+
+    data: ArrayBuffer;
+}
+
+const pictureReadyEvent = new EventEmitter<PictureReadyEventArgs>();
+
+export class TinyH264Wrapper extends AutoDisposable {
+    public readonly streamId: number;
+
+    private readonly pictureReadyEvent = new EventEmitter<PictureReadyEventArgs>();
+    public get onPictureReady() { return this.pictureReadyEvent.event; }
+
+    public constructor(streamId: number) {
+        super();
+
+        this.streamId = streamId;
+        this.addDisposable(pictureReadyEvent.event(this.handlePictureReady, this));
+    }
+
+    private handlePictureReady(e: PictureReadyEventArgs) {
+        if (e.renderStateId === this.streamId) {
+            this.pictureReadyEvent.fire(e);
+        }
+    }
+
+    public feed(data: ArrayBuffer) {
+        worker!.postMessage({
+            type: 'decode',
+            data: data,
+            offset: 0,
+            length: data.byteLength,
+            renderStateId: this.streamId,
+        }, [data]);
+    }
+
+    public override dispose() {
+        super.dispose();
+        worker!.postMessage({
+            type: 'release',
+            renderStateId: this.streamId,
+        });
+    }
+}
+
+export function createTinyH264Wrapper(): Promise<TinyH264Wrapper> {
+    if (!worker) {
+        worker = new Worker(new URL('./worker.js', import.meta.url));
+        worker.addEventListener('message', (e) => {
+            const { data } = e;
+            switch (data.type) {
+                case 'decoderReady':
+                    workerReady = true;
+                    for (const resolver of pendingResolvers) {
+                        resolver.resolve(new TinyH264Wrapper(streamId));
+                        streamId += 1;
+                    }
+                    pendingResolvers.length = 0;
+                    break;
+                case 'pictureReady':
+                    pictureReadyEvent.fire(data);
+                    break;
+            }
+        });
+    }
+
+    if (!workerReady) {
+        const resolver = new PromiseResolver<TinyH264Wrapper>();
+        pendingResolvers.push(resolver);
+        return resolver.promise;
+    }
+
+    const decoder = new TinyH264Wrapper(streamId);
+    streamId += 1;
+    return Promise.resolve(decoder);
+}

package/src/decoder/types.ts

@@ -0,0 +1,35 @@
+import type { WritableStream } from '@yume-chan/adb';
+import type { Disposable } from "@yume-chan/event";
+import type { AndroidCodecLevel, AndroidCodecProfile } from "../codec.js";
+import type { VideoStreamPacket } from "../options/index.js";
+
+export interface H264Configuration {
+    profileIndex: number;
+    constraintSet: number;
+    levelIndex: number;
+
+    encodedWidth: number;
+    encodedHeight: number;
+
+    cropLeft: number;
+    cropRight: number;
+
+    cropTop: number;
+    cropBottom: number;
+
+    croppedWidth: number;
+    croppedHeight: number;
+}
+
+export interface H264Decoder extends Disposable {
+    readonly maxProfile: AndroidCodecProfile | undefined;
+    readonly maxLevel: AndroidCodecLevel | undefined;
+
+    readonly renderer: HTMLElement;
+    readonly frameRendered: number;
+    readonly writable: WritableStream<VideoStreamPacket>;
+}
+
+export interface H264DecoderConstructor {
+    new(): H264Decoder;
+}

package/src/decoder/web-codecs/index.ts

@@ -0,0 +1,98 @@
+import { WritableStream } from '@yume-chan/adb';
+import type { VideoStreamPacket } from "../../options/index.js";
+import type { H264Configuration, H264Decoder } from "../types.js";
+
+function toHex(value: number) {
+    return value.toString(16).padStart(2, '0').toUpperCase();
+}
+
+export class WebCodecsDecoder implements H264Decoder {
+    // Usually, browsers can decode most configurations,
+    // So let device choose best profile and level for itself.
+    public readonly maxProfile = undefined;
+    public readonly maxLevel = undefined;
+
+    private _writable: WritableStream<VideoStreamPacket>;
+    public get writable() { return this._writable; }
+
+    private _renderer: HTMLCanvasElement;
+    public get renderer() { return this._renderer; }
+
+    private _frameRendered = 0;
+    public get frameRendered() { return this._frameRendered; }
+
+    private context: CanvasRenderingContext2D;
+    private decoder: VideoDecoder;
+
+    // Limit FPS to system refresh rate
+    private lastFrame: VideoFrame | undefined;
+    private animationFrame: number = 0;
+
+    public constructor() {
+        this._renderer = document.createElement('canvas');
+
+        this.context = this._renderer.getContext('2d')!;
+        this.decoder = new VideoDecoder({
+            output: (frame) => {
+                if (this.lastFrame) {
+                    this.lastFrame.close();
+                }
+                this.lastFrame = frame;
+
+                if (!this.animationFrame) {
+                    // Start render loop on first frame
+                    this.render();
+                }
+            },
+            error() { },
+        });
+
+        this._writable = new WritableStream<VideoStreamPacket>({
+            write: async (packet) => {
+                switch (packet.type) {
+                    case 'configuration':
+                        this.configure(packet.data);
+                        break;
+                    case 'frame':
+                        this.decoder.decode(new EncodedVideoChunk({
+                            type: 'key',
+                            timestamp: 0,
+                            data: packet.data,
+                        }));
+                        break;
+                }
+            }
+        });
+    }
+
+    private render = () => {
+        if (this.lastFrame) {
+            this._frameRendered += 1;
+            this.context.drawImage(this.lastFrame, 0, 0);
+            this.lastFrame.close();
+            this.lastFrame = undefined;
+        }
+
+        this.animationFrame = requestAnimationFrame(this.render);
+    };
+
+    private configure(config: H264Configuration) {
+        const { profileIndex, constraintSet, levelIndex } = config;
+
+        this._renderer.width = config.croppedWidth;
+        this._renderer.height = config.croppedHeight;
+
+        // https://www.rfc-editor.org/rfc/rfc6381#section-3.3
+        // ISO Base Media File Format Name Space
+        const codec = `avc1.${[profileIndex, constraintSet, levelIndex].map(toHex).join('')}`;
+        this.decoder.configure({
+            codec: codec,
+            optimizeForLatency: true,
+        });
+    }
+
+    public dispose() {
+        cancelAnimationFrame(this.animationFrame);
+        this.decoder.close();
+    }
+}

package/src/index.ts

@@ -0,0 +1,8 @@
+export * from './client.js';
+export * from './codec.js';
+export * from './connection.js';
+export * from './decoder/index.js';
+export * from './message.js';
+export * from './options/index.js';
+export * from './push-server.js';
+export * from './utils.js';