@yume-chan/adb 2.6.2 → 3.0.0-beta.0

package/src/service/sync/request/pull.ts

@@ -0,0 +1,49 @@
+import { ReadableStream } from "@yume-chan/stream-extra";
+import type { StructValue } from "@yume-chan/struct";
+import { buffer, struct, u32 } from "@yume-chan/struct";
+
+import { RequestId, ResponseId } from "../id/index.js";
+import type { SocketPool } from "../socket-pool.js";
+import { Error as AdbSyncError } from "../socket.js";
+
+export const DataResponse = struct(
+    { data: buffer(u32) },
+    { littleEndian: true },
+);
+
+export type DataResponse = StructValue<typeof DataResponse>;
+
+export async function* generator(
+    pool: SocketPool,
+    path: string,
+): AsyncGenerator<Uint8Array, void, void> {
+    const socket = await pool.acquire();
+    let completed = false;
+    let error: unknown;
+
+    try {
+        await socket.writeRequest(RequestId.Receive, path);
+        for await (const packet of socket.readResponses(
+            ResponseId.Data,
+            DataResponse,
+        )) {
+            yield packet.data;
+        }
+        completed = true;
+    } catch (e) {
+        error = e;
+        throw e;
+    } finally {
+        await pool.release(
+            socket,
+            !(completed || error instanceof AdbSyncError),
+        );
+    }
+}
+
+export function stream(
+    pool: SocketPool,
+    path: string,
+): ReadableStream<Uint8Array> {
+    return ReadableStream.from(generator(pool, path));
+}

package/src/service/sync/request/push.ts

@@ -0,0 +1,184 @@
+import type { ReadableStream } from "@yume-chan/stream-extra";
+import {
+    AbortController,
+    DistributionStream,
+    MaybeConsumable,
+} from "@yume-chan/stream-extra";
+import { struct, u32 } from "@yume-chan/struct";
+
+import { NOOP } from "../../../utils/index.js";
+import { LinuxFileType } from "../android.js";
+import { Compression } from "../compression/index.js";
+import { RequestId, ResponseId } from "../id/index.js";
+import type { SocketPool } from "../socket-pool.js";
+import type { Socket } from "../socket.js";
+import { Error as AdbSyncError } from "../socket.js";
+
+export const MaxPacketSize = 64 * 1024;
+
+export const OkResponse = struct({ unused: u32 }, { littleEndian: true });
+
+// eslint-disable-next-line @typescript-eslint/max-params
+async function pipeFileData(
+    socket: Socket,
+    file: ReadableStream<MaybeConsumable<Uint8Array>>,
+    packetSize: number,
+    mtime: number,
+    compression: Compression.Format,
+): Promise<void> {
+    if (compression !== Compression.Format.None) {
+        const stream = Compression.createCompressionStream(compression);
+        void file
+            .pipeTo(new MaybeConsumable.WrapWritableStream(stream.writable))
+            .catch(NOOP);
+        file = stream.readable;
+    }
+
+    // Read and write in parallel,
+    // allow error response to abort the write.
+    const abortController = new AbortController();
+    file.pipeThrough(new DistributionStream(packetSize, true))
+        .pipeTo(
+            new MaybeConsumable.WritableStream({
+                write(chunk) {
+                    return socket.writeRequest(RequestId.Data, chunk);
+                },
+            }),
+            { signal: abortController.signal },
+        )
+        .then(async () => {
+            await socket.writeRequest(RequestId.Done, mtime);
+            await socket.flush();
+        }, NOOP);
+
+    try {
+        await socket.readResponse(ResponseId.Ok, OkResponse);
+    } catch (e) {
+        abortController.abort(e);
+        throw e;
+    }
+}
+
+export interface SendV1Options {
+    pool: SocketPool;
+    filename: string;
+    file: ReadableStream<MaybeConsumable<Uint8Array>>;
+    type?: LinuxFileType | undefined;
+    permission?: number | undefined;
+    mtime?: number | undefined;
+    packetSize?: number | undefined;
+}
+
+export function sendV1({
+    pool,
+    filename,
+    file,
+    type = LinuxFileType.File,
+    permission = 0o666,
+    mtime = (Date.now() / 1000) | 0,
+    packetSize = MaxPacketSize,
+}: SendV1Options) {
+    return pool.withSocket(async (socket) => {
+        const mode = (type << 12) | permission;
+        const pathAndMode = `${filename},${mode.toString()}`;
+        await socket.writeRequest(RequestId.Send, pathAndMode);
+        await pipeFileData(
+            socket,
+            file,
+            packetSize,
+            mtime,
+            Compression.Format.None,
+        );
+    });
+}
+
+export const SendV2Flags = {
+    None: 0,
+    Brotli: 1,
+    Lz4: 2,
+    Zstd: 4,
+    DryRun: 0x80000000,
+} as const;
+
+export type SendV2Flags = (typeof SendV2Flags)[keyof typeof SendV2Flags];
+
+export const SendV2Request = struct(
+    { id: u32, mode: u32, flags: u32 },
+    { littleEndian: true },
+);
+
+export interface SendV2Options extends SendV1Options {
+    compression?: Compression.Format | undefined;
+
+    /**
+     * Don't write the file to disk. Requires the `sendrecv_v2` feature.
+     *
+     * It was used during ADB development to benchmark the performance of
+     * compression algorithms.
+     */
+    dryRun?: boolean | undefined;
+}
+
+export function sendV2({
+    pool,
+    filename,
+    file,
+    type = LinuxFileType.File,
+    permission = 0o666,
+    mtime = (Date.now() / 1000) | 0,
+    packetSize = MaxPacketSize,
+    compression = Compression.Format.None,
+    dryRun = false,
+}: SendV2Options) {
+    return pool.withSocket(async (socket) => {
+        let flags: SendV2Flags = SendV2Flags.None;
+        switch (compression) {
+            case Compression.Format.Brotli:
+                flags |= SendV2Flags.Brotli;
+                break;
+            case Compression.Format.Lz4:
+                flags |= SendV2Flags.Lz4;
+                break;
+            case Compression.Format.Zstd:
+                flags |= SendV2Flags.Zstd;
+                break;
+        }
+        if (dryRun) {
+            flags |= SendV2Flags.DryRun;
+        }
+
+        await socket.writeRequest(RequestId.SendV2, filename);
+        await socket.write(
+            SendV2Request.serialize({
+                id: RequestId.SendV2,
+                mode: (type << 12) | permission,
+                flags,
+            }),
+        );
+
+        await pipeFileData(socket, file, packetSize, mtime, compression);
+    });
+}
+
+export interface SendOptions extends SendV2Options {
+    version: 1 | 2;
+}
+
+export function send(options: SendOptions) {
+    if (options.version === 2) {
+        return sendV2(options);
+    }
+
+    if (options.dryRun) {
+        throw new AdbSyncError("dryRun is not supported in v1");
+    }
+
+    if (
+        options.compression !== undefined &&
+        options.compression !== Compression.Format.None
+    ) {
+        throw new AdbSyncError("compression is not supported in v1");
+    }
+
+    return sendV1(options);
+}

package/src/service/sync/request/stat.ts

@@ -0,0 +1,127 @@
+import type { StructValue } from "@yume-chan/struct";
+import { struct, u32, u64 } from "@yume-chan/struct";
+
+import type { AndroidSyscallErrorCode, LinuxFileType } from "../android.js";
+import { AndroidSyscallErrorNameMap } from "../android.js";
+import { RequestId, ResponseId } from "../id/index.js";
+import type { SocketPool } from "../socket-pool.js";
+import { Error as AdbSyncError } from "../socket.js";
+
+export const LstatResponse = struct(
+    { mode: u32, size: u32, mtime: u32 },
+    {
+        littleEndian: true,
+        extra: {
+            get type(): LinuxFileType {
+                return (this.mode >> 12) as LinuxFileType;
+            },
+            get permission(): number {
+                return this.mode & 0b00001111_11111111;
+            },
+        },
+        postDeserialize(value) {
+            if (value.mode === 0 && value.size === 0 && value.mtime === 0) {
+                throw new AdbSyncError("lstat error");
+            }
+            return value;
+        },
+    },
+);
+
+export type LstatResponse = StructValue<typeof LstatResponse>;
+
+export function lstatV1(
+    pool: SocketPool,
+    path: string,
+): Promise<LstatResponse> {
+    return pool.withSocket(async (socket) => {
+        await socket.writeRequest(RequestId.Lstat, path);
+        return await socket.readResponse(ResponseId.Lstat, LstatResponse);
+    });
+}
+
+export const StatResponse = struct(
+    {
+        error: u32<AndroidSyscallErrorCode>(),
+        dev: u64,
+        ino: u64,
+        mode: u32,
+        nlink: u32,
+        uid: u32,
+        gid: u32,
+        size: u64,
+        atime: u64,
+        mtime: u64,
+        ctime: u64,
+    },
+    {
+        littleEndian: true,
+        extra: {
+            get type(): LinuxFileType {
+                return (this.mode >> 12) as LinuxFileType;
+            },
+            get permission(): number {
+                return this.mode & 0b00001111_11111111;
+            },
+        },
+        postDeserialize(value) {
+            if (value.error) {
+                throw new AdbSyncError(AndroidSyscallErrorNameMap[value.error]);
+            }
+            return value;
+        },
+    },
+);
+
+export type StatResponse = StructValue<typeof StatResponse>;
+
+export function lstatV2(pool: SocketPool, path: string) {
+    return pool.withSocket(async (socket) => {
+        await socket.writeRequest(RequestId.LstatV2, path);
+        return await socket.readResponse(ResponseId.LstatV2, StatResponse);
+    });
+}
+
+export async function lstat(
+    pool: SocketPool,
+    path: string,
+    { version }: { version: 1 | 2 },
+): Promise<Stat> {
+    if (version === 2) {
+        return await lstatV2(pool, path);
+    } else {
+        const response = await lstatV1(pool, path);
+        return {
+            mode: response.mode,
+            // Convert to `BigInt` to make it compatible with `StatResponse`
+            size: BigInt(response.size),
+            mtime: BigInt(response.mtime),
+            get type() {
+                return response.type;
+            },
+            get permission() {
+                return response.permission;
+            },
+        };
+    }
+}
+
+export interface Stat {
+    mode: number;
+    size: bigint;
+    mtime: bigint;
+    get type(): LinuxFileType;
+    get permission(): number;
+
+    uid?: number;
+    gid?: number;
+    atime?: bigint;
+    ctime?: bigint;
+}
+
+export function stat(pool: SocketPool, path: string): Promise<StatResponse> {
+    return pool.withSocket(async (socket) => {
+        await socket.writeRequest(RequestId.Stat, path);
+        return await socket.readResponse(ResponseId.Stat, StatResponse);
+    });
+}

package/src/service/sync/service.ts

@@ -0,0 +1,255 @@
+import type {
+    AbortSignal,
+    MaybeConsumable,
+    ReadableStream,
+} from "@yume-chan/stream-extra";
+
+import type { Adb } from "../../adb.js";
+import { AdbFeature } from "../../features.js";
+import { escapeArg } from "../subprocess/index.js";
+
+import type { LinuxFileType } from "./android.js";
+import { Compression } from "./compression/index.js";
+import { OpenDir, Receive, Send, Stat } from "./request/index.js";
+import { SocketPool } from "./socket-pool.js";
+import type { Socket } from "./socket.js";
+import { Error as AdbSyncError } from "./socket.js";
+
+export interface ServiceOptions {
+    /**
+     * Maximum number of sockets to keep in the pool for reuse.
+     * Extra sockets beyond this limit will be closed when released.
+     * @default 4
+     */
+    maxPoolSize?: number | undefined;
+
+    /**
+     * Time in milliseconds to keep idle sockets in the pool before closing them.
+     * Set to 0 to disable idle timeout (sockets will remain until pool is disposed).
+     * @default 60000 (1 minute)
+     */
+    idleTimeout?: number | undefined;
+
+    /**
+     * Abort signal to dispose the socket pool when aborted.
+     */
+    signal?: AbortSignal | undefined;
+}
+
+/**
+ * A simplified `dirname` function that only handles absolute unix paths.
+ * @param path an absolute unix path
+ * @returns the directory name of the input path
+ */
+export function dirname(path: string): string {
+    const end = path.lastIndexOf("/");
+    if (end === -1) {
+        throw new Error(`Invalid absolute unix path: ${path}`);
+    }
+    if (end === 0) {
+        return "/";
+    }
+    return path.substring(0, end);
+}
+
+export interface WriteOptions {
+    filename: string;
+    file: ReadableStream<MaybeConsumable<Uint8Array>>;
+    type?: LinuxFileType | undefined;
+    permission?: number | undefined;
+    mtime?: number | undefined;
+    /**
+     * The format to compress the file stream for sending.
+     *
+     * If the device doesn't support compressed sending, it will be ignored.
+     *
+     * If the device or current runtime doesn't support the specified format,
+     * an Error will be thrown.
+     * {@link Compression.canUseFormat} can be used to check if
+     * the device and current runtime both supports the format.
+     */
+    compression?: Compression.Format | undefined;
+    dryRun?: boolean | undefined;
+}
+
+export class Service {
+    protected _adb: Adb;
+    readonly #socketPool: SocketPool;
+
+    readonly #supportsStat2: boolean;
+    readonly #supportsLs2: boolean;
+    readonly #fixedPushMkdir: boolean;
+    readonly #supportsSendReceive2: boolean;
+    readonly #needPushMkdirWorkaround: boolean;
+
+    get supportsStat2(): boolean {
+        return this.#supportsStat2;
+    }
+
+    get supportsLs2(): boolean {
+        return this.#supportsLs2;
+    }
+
+    get fixedPushMkdir(): boolean {
+        return this.#fixedPushMkdir;
+    }
+
+    get supportsSendReceive2(): boolean {
+        return this.#supportsSendReceive2;
+    }
+
+    get needPushMkdirWorkaround(): boolean {
+        return this.#needPushMkdirWorkaround;
+    }
+
+    constructor(adb: Adb, options?: ServiceOptions) {
+        this._adb = adb;
+        this.#socketPool = new SocketPool(
+            adb,
+            options?.maxPoolSize,
+            options?.idleTimeout,
+        );
+
+        // Dispose the socket pool when the abort signal is triggered
+        options?.signal?.addEventListener("abort", () => {
+            void this.#socketPool.dispose();
+        });
+
+        this.#supportsStat2 = adb.canUseFeature(AdbFeature.Stat2);
+        this.#supportsLs2 = adb.canUseFeature(AdbFeature.Ls2);
+        this.#fixedPushMkdir = adb.canUseFeature(AdbFeature.FixedPushMkdir);
+        this.#supportsSendReceive2 = adb.canUseFeature(AdbFeature.SendReceive2);
+        // https://android.googlesource.com/platform/packages/modules/adb/+/91768a57b7138166e0a3d11f79cd55909dda7014/client/file_sync_client.cpp#1361
+        this.#needPushMkdirWorkaround =
+            this._adb.canUseFeature(AdbFeature.Shell2) && !this.fixedPushMkdir;
+    }
+
+    /**
+     * Acquires a socket from the pool for direct use.
+     *
+     * The socket must be returned to the pool using {@link releaseSocket} after use.
+     *
+     * @returns A socket from the pool
+     */
+    async acquireSocket() {
+        return await this.#socketPool.acquire();
+    }
+
+    /**
+     * Returns a socket to the pool after use.
+     *
+     * @param socket The socket to return to the pool
+     * @param discard Whether to discard the socket instead of returning it to the pool
+     */
+    async releaseSocket(socket: Socket, discard = false) {
+        await this.#socketPool.release(socket, discard);
+    }
+
+    /**
+     * Gets information of a file or folder.
+     *
+     * If `path` points to a symbolic link, the returned information is about the link itself (with `type` being `LinuxFileType.Link`).
+     */
+    lstat(path: string): Promise<Stat.Stat> {
+        return Stat.lstat(this.#socketPool, path, {
+            version: this.#supportsStat2 ? 2 : 1,
+        });
+    }
+
+    /**
+     * Gets the information of a file or folder.
+     *
+     * If `path` points to a symbolic link, it will be resolved and the returned information is about the target (with `type` being `LinuxFileType.File` or `LinuxFileType.Directory`).
+     */
+    stat(path: string) {
+        if (!this.#supportsStat2) {
+            throw new Error("Not supported");
+        }
+
+        return Stat.stat(this.#socketPool, path);
+    }
+
+    /**
+     * Checks if `path` is a directory, or a symbolic link to a directory.
+     *
+     * This uses `lstat` internally, thus works on all Android versions.
+     */
+    async isDirectory(path: string): Promise<boolean> {
+        try {
+            await this.lstat(path + "/");
+            return true;
+        } catch (e) {
+            if (e instanceof AdbSyncError) {
+                return false;
+            } else {
+                throw e;
+            }
+        }
+    }
+
+    opendir(path: string): AsyncGenerator<OpenDir.Entry, void, void> {
+        return OpenDir.opendir(this.#socketPool, path, {
+            version: this.supportsLs2 ? 2 : 1,
+        });
+    }
+
+    async readdir(path: string) {
+        // TODO: Convert to `Array.fromAsync`
+        const results: OpenDir.Entry[] = [];
+        for await (const entry of this.opendir(path)) {
+            results.push(entry);
+        }
+        return results;
+    }
+
+    /**
+     * Reads the content of a file on device.
+     *
+     * @param filename The full path of the file on device to read.
+     * @returns A `ReadableStream` that contains the file content.
+     */
+    read(filename: string): ReadableStream<Uint8Array> {
+        return Receive.stream(this.#socketPool, filename);
+    }
+
+    /**
+     * Writes a file on device. If the file name already exists, it will be overwritten.
+     *
+     * @param options The content and options of the file to write.
+     */
+    async write(options: WriteOptions): Promise<void> {
+        if (this.needPushMkdirWorkaround) {
+            // It may fail if `filename` already exists.
+            // Ignore the result.
+            // TODO: sync: test push mkdir workaround (need an Android 8 device)
+            await this._adb.subprocess.noneProtocol
+                .spawn(["mkdir", "-p", escapeArg(dirname(options.filename))])
+                .wait();
+        }
+
+        if (this.supportsSendReceive2) {
+            if (options.compression === undefined) {
+                options.compression = Compression.chooseFormat(
+                    this._adb,
+                    Compression.Mode.Compress,
+                );
+            } else if (
+                !Compression.canUseFormat(
+                    this._adb,
+                    options.compression,
+                    Compression.Mode.Compress,
+                )
+            ) {
+                throw new Error(
+                    `Compression type ${options.compression} is not supported`,
+                );
+            }
+        }
+
+        await Send.send({
+            version: this.supportsSendReceive2 ? 2 : 1,
+            pool: this.#socketPool,
+            ...options,
+        });
+    }
+}

package/src/service/sync/socket-pool.ts

@@ -0,0 +1,97 @@
+import type { Adb } from "../../adb.js";
+
+import { Error as AdbSyncError, Socket } from "./socket.js";
+
+export class SocketPool {
+    readonly #adb: Adb;
+    readonly #maxSize: number;
+    readonly #idleTimeout: number;
+    readonly #availableSockets: Socket[] = [];
+    readonly #inUseSockets = new Set<Socket>();
+    #closed = false;
+
+    constructor(adb: Adb, maxSize = 4, idleTimeout = 60000) {
+        this.#adb = adb;
+        this.#maxSize = maxSize;
+        this.#idleTimeout = idleTimeout;
+    }
+
+    async acquire(): Promise<Socket> {
+        if (this.#closed) {
+            throw new Error("SocketPool is closed");
+        }
+        // Try to reuse an available socket
+        if (this.#availableSockets.length > 0) {
+            const socket = this.#availableSockets.pop()!;
+            // Clear the idle timeout
+            socket.clearIdleTimer();
+            this.#inUseSockets.add(socket);
+            return socket;
+        }
+
+        // Create a new socket
+        const adbSocket = await this.#adb.createSocket("sync:");
+        if (this.#closed) {
+            await adbSocket.close();
+            throw new Error("SocketPool is closed");
+        }
+        const socket = new Socket(adbSocket, this.#adb.maxPayloadSize);
+        this.#inUseSockets.add(socket);
+        return socket;
+    }
+
+    async withSocket<T>(fn: (socket: Socket) => Promise<T>): Promise<T> {
+        const socket = await this.acquire();
+        try {
+            const result = await fn(socket);
+            await this.release(socket);
+            return result;
+        } catch (e) {
+            await this.release(socket, !(e instanceof AdbSyncError));
+            throw e;
+        }
+    }
+
+    async release(socket: Socket, discard = false): Promise<void> {
+        this.#inUseSockets.delete(socket);
+
+        // If discarding or we already have enough sockets in the pool, close this socket
+        if (discard || this.#availableSockets.length >= this.#maxSize) {
+            await socket.close();
+            return;
+        }
+
+        // Otherwise, return it to the pool and start idle timeout
+        socket.startIdleTimer(this.#idleTimeout, () => {
+            this.#closeIdleSocket(socket);
+        });
+        this.#availableSockets.push(socket);
+    }
+
+    #closeIdleSocket(socket: Socket): void {
+        // Remove from available sockets
+        const index = this.#availableSockets.indexOf(socket);
+        if (index !== -1) {
+            this.#availableSockets.splice(index, 1);
+            void socket.close();
+        }
+    }
+
+    async dispose(): Promise<void> {
+        this.#closed = true;
+
+        // Close all available sockets
+        const closePromises = this.#availableSockets.map((socket) =>
+            socket.close(),
+        );
+        this.#availableSockets.length = 0;
+
+        // Force close all in-use sockets immediately
+        const inUseClosePromises = Array.from(this.#inUseSockets).map(
+            (socket) => socket.close(),
+        );
+        this.#inUseSockets.clear();
+
+        await Promise.all([...closePromises, ...inUseClosePromises]);
+    }
+}