io.appium.settings 8.0.9 → 8.1.0

package/lib/commands/types.ts

@@ -37,6 +37,76 @@ export interface StartMediaProjectionRecordingOpts {
   filename?: string;
 }
 
+/**
+ * Options for choosing the local TCP port used to bridge a live stream's
+ * on-device socket to the host via `adb forward`. At most one of these may be
+ * provided; if neither is given, an OS-assigned ephemeral port is used.
+ */
+export interface LocalPortOpts {
+  /** Use this exact local TCP port. Throws if it's already in use. */
+  localPort?: number;
+  /** Pick the first free local TCP port from this inclusive [min, max] range. */
+  localPortRange?: [number, number];
+}
+
+/**
+ * Options for starting a live JPEG frame streaming session.
+ */
+export interface StartJpegStreamOpts extends LocalPortOpts {
+  /** Frames per second requested; actual rate may be lower on constrained devices. Default 60. */
+  fps?: number;
+  /** JPEG quality, 1-100. Default 80. */
+  quality?: number;
+  /** Percentage (1-100) to scale the captured frame to before JPEG encoding. Default 100. */
+  scale?: number;
+}
+
+/**
+ * Options for starting a live H.264/HEVC video streaming session.
+ */
+export interface StartVideoStreamOpts extends LocalPortOpts {
+  /** Default 'h264'. */
+  codec?: 'h264' | 'hevc';
+  /** Default 30. */
+  fps?: number;
+  /** Default 4000000 (4 Mbps). */
+  bitrate?: number;
+  /** Same "WxH" syntax/allowed values as StartMediaProjectionRecordingOpts.resolution. */
+  resolution?: string;
+  /** Interleave an AAC audio track captured via AudioPlaybackCaptureConfiguration. Default false. */
+  audio?: boolean;
+}
+
+/**
+ * A single JPEG-encoded frame of the device screen.
+ */
+export interface JpegFrame {
+  /** The raw JPEG-encoded image bytes. */
+  data: Buffer;
+  /** Monotonically increasing frame sequence number. */
+  sequence: number;
+  /** Frame timestamp, in microseconds since the stream session started. */
+  timestampMicros: number;
+}
+
+/**
+ * A single encoded access unit from a live video stream: either a video NAL
+ * unit (Annex-B, H.264/HEVC) or an AAC audio unit (ADTS-framed).
+ */
+export interface AccessUnit {
+  track: 'video' | 'audio';
+  /** The raw access unit bytes. */
+  data: Buffer;
+  /** Monotonically increasing sequence number, per track. */
+  sequence: number;
+  /** Presentation timestamp, in microseconds since the stream session started. */
+  timestampMicros: number;
+  /** Whether this is a video keyframe (always false for audio units). */
+  isKeyFrame: boolean;
+  /** Whether this unit carries codec configuration data (e.g. SPS/PPS) rather than actual samples. */
+  isConfig: boolean;
+}
+
 /**
  * Supported locale information.
  */

package/lib/commands/video-stream.ts

@@ -0,0 +1,179 @@
+import {randomUUID} from 'node:crypto';
+
+import type {ADB} from 'appium-adb';
+import {waitForCondition} from 'asyncbox';
+
+import type {SettingsApp} from '../client.js';
+import {
+  SETTINGS_HELPER_ID,
+  STREAMING_ACTIVITY_NAME,
+  VIDEO_STREAM_ACTION_START,
+  VIDEO_STREAM_ACTION_STOP,
+  VIDEO_STREAM_SERVICE_NAME,
+} from '../constants.js';
+import {StreamTrack, StreamTransport, validateLocalPortOpts} from '../utils/index.js';
+import type {AccessUnit, StartVideoStreamOpts} from './types.js';
+
+const STREAM_STARTUP_TIMEOUT_MS = 3 * 1000;
+const STREAM_STOP_TIMEOUT_MS = 3 * 1000;
+
+/**
+ * Creates a new instance of the live H.264/HEVC video streaming session.
+ * The stream only works since Android API 29+
+ *
+ * @returns The streaming session instance
+ */
+export function makeVideoStreamSession(this: SettingsApp): VideoStreamSession {
+  return new VideoStreamSession(this.adb);
+}
+
+function validateVideoStreamOpts(opts: StartVideoStreamOpts): void {
+  const {codec, fps, bitrate} = opts;
+  if (codec !== undefined && codec !== 'h264' && codec !== 'hevc') {
+    throw new TypeError(`codec must be either 'h264' or 'hevc', got ${codec}`);
+  }
+  if (fps !== undefined && (!Number.isInteger(fps) || fps <= 0)) {
+    throw new TypeError(`fps must be a positive integer, got ${fps}`);
+  }
+  if (bitrate !== undefined && (!Number.isInteger(bitrate) || bitrate <= 0)) {
+    throw new TypeError(`bitrate must be a positive integer, got ${bitrate}`);
+  }
+  validateLocalPortOpts(opts);
+}
+
+/**
+ * Live H.264/HEVC video streaming session for the device screen, with an
+ * optional interleaved AAC audio track.
+ */
+export class VideoStreamSession {
+  private transport: StreamTransport | null = null;
+
+  /**
+   * Creates a new VideoStreamSession instance.
+   *
+   * @param adb - ADB instance for device communication
+   */
+  constructor(private readonly adb: ADB) {}
+
+  /**
+   * Checks if the video stream is currently running.
+   */
+  async isRunning(): Promise<boolean> {
+    const stdout = await this.adb.shell(['dumpsys', 'activity', 'services', VIDEO_STREAM_SERVICE_NAME]);
+    return stdout.includes(VIDEO_STREAM_SERVICE_NAME);
+  }
+
+  /**
+   * Starts the live video stream.
+   * If a stream is already running, this method returns false without starting a new one.
+   *
+   * @param opts Streaming options including codec, fps, bitrate, resolution, audio, and the local port to use
+   * @returns True if the stream was started successfully, false if already running
+   * @throws {Error} If the stream fails to start within the timeout period
+   */
+  async start(opts: StartVideoStreamOpts = {}): Promise<boolean> {
+    validateVideoStreamOpts(opts);
+    if (await this.isRunning()) {
+      return false;
+    }
+
+    const {codec, fps, bitrate, resolution, audio, localPort, localPortRange} = opts;
+    // Socket names are validated on-device against ^[a-zA-Z0-9._-]{1,200}$, so this
+    // must not contain a "/" - unlike VIDEO_STREAM_SERVICE_NAME.
+    const socketName = `${SETTINGS_HELPER_ID}.videostream.${randomUUID()}`;
+    const args = [
+      'am',
+      'start',
+      '-n',
+      STREAMING_ACTIVITY_NAME,
+      '-a',
+      VIDEO_STREAM_ACTION_START,
+      '--es',
+      'socket_name',
+      socketName,
+    ];
+    if (codec) {
+      args.push('--es', 'codec', codec);
+    }
+    if (fps) {
+      args.push('--es', 'fps', `${fps}`);
+    }
+    if (bitrate) {
+      args.push('--es', 'bitrate', `${bitrate}`);
+    }
+    if (resolution) {
+      args.push('--es', 'resolution', resolution);
+    }
+    if (audio !== undefined) {
+      args.push('--es', 'audio', audio ? 'true' : 'false');
+    }
+
+    await this.adb.shell(args);
+    try {
+      await waitForCondition(async () => await this.isRunning(), {
+        waitMs: STREAM_STARTUP_TIMEOUT_MS,
+        intervalMs: 300,
+      });
+    } catch {
+      throw new Error(
+        `The video stream is not running after ${STREAM_STARTUP_TIMEOUT_MS}ms. ` +
+          `Please check the logcat output for more details.`,
+      );
+    }
+
+    this.transport = await StreamTransport.connect(this.adb, socketName, {localPort, localPortRange});
+    return true;
+  }
+
+  /**
+   * Yields access units (video NAL units and, if enabled, AAC audio units) as
+   * they arrive from the device. `start()` must be called before iterating.
+   */
+  async *accessUnits(): AsyncGenerator<AccessUnit> {
+    if (!this.transport) {
+      throw new Error('The video stream has not been started. Call start() first.');
+    }
+    for await (const frame of this.transport.frames()) {
+      yield {
+        track: frame.track === StreamTrack.Audio ? 'audio' : 'video',
+        data: frame.data,
+        sequence: Number(frame.sequence),
+        timestampMicros: Number(frame.timestampMicros),
+        isKeyFrame: frame.isKeyFrame,
+        isConfig: frame.isConfig,
+      };
+    }
+  }
+
+  /**
+   * Stops the currently running video stream.
+   *
+   * @returns True if the stream was stopped successfully, false if it was not running
+   * @throws {Error} If the stream fails to stop within the timeout period
+   */
+  async stop(): Promise<boolean> {
+    try {
+      if (!(await this.isRunning())) {
+        // The on-device service may have already stopped itself (e.g. the client
+        // disconnected), leaving a local transport/adb-forward that still needs closing.
+        return false;
+      }
+
+      await this.adb.shell(['am', 'start', '-n', STREAMING_ACTIVITY_NAME, '-a', VIDEO_STREAM_ACTION_STOP]);
+      try {
+        await waitForCondition(async () => !(await this.isRunning()), {
+          waitMs: STREAM_STOP_TIMEOUT_MS,
+          intervalMs: 300,
+        });
+      } catch {
+        throw new Error(`The attempt to stop the current video stream timed out after ${STREAM_STOP_TIMEOUT_MS}ms`);
+      }
+      return true;
+    } finally {
+      if (this.transport) {
+        await this.transport.close();
+        this.transport = null;
+      }
+    }
+  }
+}

package/lib/constants.ts

@@ -43,3 +43,13 @@ export const RECORDING_SERVICE_NAME = `${SETTINGS_HELPER_ID}/.recorder.RecorderS
 export const RECORDING_ACTIVITY_NAME = `${SETTINGS_HELPER_ID}/io.appium.settings.Settings`;
 export const RECORDING_ACTION_START = `${SETTINGS_HELPER_ID}.recording.ACTION_START`;
 export const RECORDING_ACTION_STOP = `${SETTINGS_HELPER_ID}.recording.ACTION_STOP`;
+
+export const STREAMING_ACTIVITY_NAME = RECORDING_ACTIVITY_NAME;
+
+export const JPEG_STREAM_SERVICE_NAME = `${SETTINGS_HELPER_ID}/.streaming.JpegStreamService`;
+export const JPEG_STREAM_ACTION_START = `${SETTINGS_HELPER_ID}.streaming.jpeg.ACTION_START`;
+export const JPEG_STREAM_ACTION_STOP = `${SETTINGS_HELPER_ID}.streaming.jpeg.ACTION_STOP`;
+
+export const VIDEO_STREAM_SERVICE_NAME = `${SETTINGS_HELPER_ID}/.streaming.VideoStreamService`;
+export const VIDEO_STREAM_ACTION_START = `${SETTINGS_HELPER_ID}.streaming.video.ACTION_START`;
+export const VIDEO_STREAM_ACTION_STOP = `${SETTINGS_HELPER_ID}.streaming.video.ACTION_STOP`;

package/lib/index.ts

@@ -1,4 +1,4 @@
-import {getSettingsApkPath} from './utils.js';
+import {getSettingsApkPath} from './utils/index.js';
 
 export const path = getSettingsApkPath();
 export * from './constants.js';

package/lib/utils/index.ts

@@ -0,0 +1,4 @@
+export {getSettingsApkPath} from './apk-path.js';
+export {createParserState, parseFrames, StreamTrack} from './streaming-protocol.js';
+export type {FrameParserState, StreamFrame} from './streaming-protocol.js';
+export {BoundedFrameQueue, resolveLocalPort, StreamTransport, validateLocalPortOpts} from './streaming-transport.js';

package/lib/utils/streaming-protocol.ts

@@ -0,0 +1,93 @@
+/**
+ * Track discriminator for a streamed frame, matching the on-device
+ * `io.appium.settings.streaming.Frame.Track` enum ordinals.
+ */
+export enum StreamTrack {
+  Jpeg = 0,
+  Video = 1,
+  Audio = 2,
+}
+
+const FLAG_CONFIG = 0x01;
+const FLAG_KEYFRAME = 0x02;
+const FLAG_EOS = 0x04;
+
+/**
+ * A single parsed frame/access-unit read off the streaming socket.
+ */
+export interface StreamFrame {
+  track: StreamTrack;
+  sequence: bigint;
+  timestampMicros: bigint;
+  data: Buffer;
+  isConfig: boolean;
+  isKeyFrame: boolean;
+  isEndOfStream: boolean;
+}
+
+const HEADER_SIZE = 28;
+const MAGIC = Buffer.from('APST', 'ascii');
+
+/**
+ * Incremental parser state, carried across chunk boundaries.
+ */
+export interface FrameParserState {
+  buf: Buffer;
+}
+
+/**
+ * Creates a fresh, empty parser state for use with {@link parseFrames}.
+ *
+ * @returns A new, empty FrameParserState
+ */
+export function createParserState(): FrameParserState {
+  return {buf: Buffer.alloc(0)};
+}
+
+/**
+ * Parses as many complete frames as are available in `state.buf` + `chunk`,
+ * carrying over any trailing partial frame in `state` for the next call.
+ *
+ * Wire format (big-endian, matches `io.appium.settings.streaming.StreamProtocol`):
+ * offset 0 (4 bytes): magic "APST"; offset 4 (1 byte): version;
+ * offset 5 (1 byte): track; offset 6 (1 byte): flags; offset 7: reserved;
+ * offset 8 (8 bytes): sequence; offset 16 (8 bytes): timestampMicros;
+ * offset 24 (4 bytes): payloadLength; followed by `payloadLength` payload bytes.
+ *
+ * @throws {Error} If the stream desynchronizes (unexpected magic bytes)
+ */
+export function* parseFrames(state: FrameParserState, chunk: Buffer): Generator<StreamFrame> {
+  state.buf = state.buf.length ? Buffer.concat([state.buf, chunk]) : chunk;
+  for (;;) {
+    if (state.buf.length < HEADER_SIZE) {
+      return;
+    }
+    if (!state.buf.subarray(0, 4).equals(MAGIC)) {
+      throw new Error('Stream desynchronized: unexpected magic bytes in the streaming protocol header');
+    }
+
+    const payloadLength = state.buf.readUInt32BE(24);
+    const totalLength = HEADER_SIZE + payloadLength;
+    if (state.buf.length < totalLength) {
+      return;
+    }
+
+    const track = state.buf.readUInt8(5) as StreamTrack;
+    const flags = state.buf.readUInt8(6);
+    const sequence = state.buf.readBigUInt64BE(8);
+    const timestampMicros = state.buf.readBigUInt64BE(16);
+    const data = Buffer.from(state.buf.subarray(HEADER_SIZE, totalLength));
+
+    state.buf = state.buf.subarray(totalLength);
+
+    yield {
+      track,
+      sequence,
+      timestampMicros,
+      data,
+      isConfig: (flags & FLAG_CONFIG) !== 0,
+      isKeyFrame: (flags & FLAG_KEYFRAME) !== 0,
+      isEndOfStream: (flags & FLAG_EOS) !== 0,
+    };
+  }
+}