@limrun/appium-xcuitest-driver 10.4.3-lim.1

package/lib/commands/recordscreen.js

@@ -0,0 +1,391 @@
+import _ from 'lodash';
+import {fs, tempDir, logger, util} from 'appium/support';
+import {SubProcess} from 'teen_process';
+import {encodeBase64OrUpload} from '../utils';
+import {WDA_BASE_URL} from 'appium-webdriveragent';
+import {waitForCondition} from 'asyncbox';
+import url from 'url';
+
+/**
+ * Set max timeout for 'reconnect_delay_max' ffmpeg argument usage.
+ * It could have [0-4294] range limitation, thus this value should be less than that right now
+ * to return a better error message.
+ */
+const MAX_RECORDING_TIME_SEC = 4200;
+const DEFAULT_RECORDING_TIME_SEC = 60 * 3;
+const DEFAULT_FPS = 10;
+const DEFAULT_QUALITY = 'medium';
+const DEFAULT_MJPEG_SERVER_PORT = 9100;
+const DEFAULT_VCODEC = 'mjpeg';
+const MP4_EXT = '.mp4';
+const FFMPEG_BINARY = 'ffmpeg';
+const ffmpegLogger = logger.getLogger(FFMPEG_BINARY);
+const QUALITY_MAPPING = {
+  low: 10,
+  medium: 25,
+  high: 75,
+  photo: 100,
+};
+
+const HARDWARE_ACCELERATION_PARAMETERS = {
+  /* https://trac.ffmpeg.org/wiki/HWAccelIntro#VideoToolbox */
+  videoToolbox: {
+    hwaccel: 'videotoolbox',
+    hwaccelOutputFormat: 'videotoolbox_vld',
+    scaleFilterHWAccel: 'scale_vt',
+    videoTypeHWAccel: 'h264_videotoolbox',
+  },
+  /* https://trac.ffmpeg.org/wiki/HWAccelIntro#CUDANVENCNVDEC */
+  cuda: {
+    hwaccel: 'cuda',
+    hwaccelOutputFormat: 'cuda',
+    scaleFilterHWAccel: 'scale_cuda',
+    videoTypeHWAccel: 'h264_nvenc',
+  },
+  /* https://trac.ffmpeg.org/wiki/Hardware/AMF */
+  amf_dx11: {
+    hwaccel: 'd3d11va',
+    hwaccelOutputFormat: 'd3d11',
+    scaleFilterHWAccel: 'scale',
+    videoTypeHWAccel: 'av1_amf',
+  },
+  /* https://trac.ffmpeg.org/wiki/Hardware/QuickSync */
+  qsv: {
+    hwaccel: 'qsv',
+    hwaccelOutputFormat: '',
+    scaleFilterHWAccel: 'scale_qsv',
+    videoTypeHWAccel: 'h264_qsv'
+  },
+  /* https://trac.ffmpeg.org/wiki/Hardware/VAAPI */
+  vaapi: {
+    hwaccel: 'vaapi',
+    hwaccelOutputFormat: 'vaapi',
+    scaleFilterHWAccel: 'scale_vaapi',
+    videoTypeHWAccel: 'h264_vaapi'
+  }
+};
+
+const CAPTURE_START_MARKER = /^\s*frame=/;
+
+export class ScreenRecorder {
+  constructor(udid, log, videoPath, opts = {}) {
+    this.videoPath = videoPath;
+    this.log = log;
+    this.opts = opts;
+    this.udid = udid;
+    this.mainProcess = null;
+    this.timeoutHandler = null;
+  }
+
+  async start(timeoutMs) {
+    try {
+      await fs.which(FFMPEG_BINARY);
+    } catch {
+      throw new Error(
+        `'${FFMPEG_BINARY}' binary is not found in PATH. Install it using 'brew install ffmpeg'. ` +
+          `Check https://www.ffmpeg.org/download.html for more details.`,
+      );
+    }
+
+    const {
+      hardwareAcceleration,
+      remotePort,
+      remoteUrl,
+      videoFps,
+      videoType,
+      videoScale,
+      videoFilters,
+      pixelFormat,
+    } = this.opts;
+
+    const args = [
+      '-f',
+      'mjpeg',
+      // https://github.com/appium/appium/issues/16294
+      '-reconnect',
+      '1',
+      '-reconnect_at_eof',
+      '1',
+      '-reconnect_streamed',
+      '1',
+      '-reconnect_delay_max',
+      `${timeoutMs / 1000 + 1}`,
+    ];
+    const {
+      hwaccel,
+      hwaccelOutputFormat,
+      scaleFilterHWAccel,
+      videoTypeHWAccel
+    } = HARDWARE_ACCELERATION_PARAMETERS[hardwareAcceleration] ?? {};
+
+    if (hwaccel) {
+      args.push('-hwaccel', hwaccel);
+    }
+
+    if (hwaccelOutputFormat) {
+      args.push('-hwaccel_output_format', hwaccelOutputFormat);
+    }
+
+    //Parameter `-r` is optional. See details: https://github.com/appium/appium/issues/12067
+    if (videoFps && videoType === 'libx264' || videoTypeHWAccel) {
+      args.push('-r', videoFps);
+    }
+    const {protocol, hostname} = url.parse(remoteUrl);
+    args.push('-i', `${protocol}//${hostname}:${remotePort}`);
+
+    if (videoFilters || videoScale) {
+      args.push('-vf', videoFilters || `${scaleFilterHWAccel || 'scale'}=${videoScale}`);
+    }
+
+    // Quicktime compatibility via pixelFormat: 'yuv420p'
+    if (pixelFormat) {
+      args.push('-pix_fmt', pixelFormat);
+    }
+    args.push('-vcodec', videoTypeHWAccel || videoType);
+    args.push('-y');
+    args.push(this.videoPath);
+
+    this.mainProcess = new SubProcess(FFMPEG_BINARY, args);
+    let isCaptureStarted = false;
+    this.mainProcess.on('line-stderr', (line) => {
+      if (CAPTURE_START_MARKER.test(line)) {
+        if (!isCaptureStarted) {
+          isCaptureStarted = true;
+        }
+      } else {
+        ffmpegLogger.info(line);
+      }
+    });
+    await this.mainProcess.start(0);
+    const startupTimeout = 5000;
+    try {
+      await waitForCondition(() => isCaptureStarted, {
+        waitMs: startupTimeout,
+        intervalMs: 300,
+      });
+    } catch {
+      this.log.warn(
+        `Screen capture process did not start within ${startupTimeout}ms. Continuing anyway`,
+      );
+    }
+    if (!this.mainProcess.isRunning) {
+      throw new Error(
+        `The screen capture process '${FFMPEG_BINARY}' died unexpectedly. ` +
+          `Check server logs for more details`,
+      );
+    }
+    this.log.info(
+      `Starting screen capture on the device '${
+        this.udid
+      }' with command: '${FFMPEG_BINARY} ${args.join(' ')}'. ` + `Will timeout in ${timeoutMs}ms`,
+    );
+
+    this.timeoutHandler = setTimeout(async () => {
+      if (!(await this.interrupt())) {
+        this.log.warn(
+          `Cannot finish the active screen recording on the device '${this.udid}' after ${timeoutMs}ms timeout`,
+        );
+      }
+    }, timeoutMs);
+  }
+
+  async interrupt(force = false) {
+    let result = true;
+
+    if (this.timeoutHandler) {
+      clearTimeout(this.timeoutHandler);
+      this.timeoutHandler = null;
+    }
+
+    if (this.mainProcess?.isRunning) {
+      const interruptPromise = this.mainProcess.stop(force ? 'SIGTERM' : 'SIGINT');
+      this.mainProcess = null;
+      try {
+        await interruptPromise;
+      } catch (e) {
+        this.log.warn(
+          `Cannot ${force ? 'terminate' : 'interrupt'} ${FFMPEG_BINARY}. ` +
+            `Original error: ${e.message}`,
+        );
+        result = false;
+      }
+    }
+
+    return result;
+  }
+
+  async finish() {
+    await this.interrupt();
+    return this.videoPath;
+  }
+
+  async cleanup() {
+    if (await fs.exists(this.videoPath)) {
+      await fs.rimraf(this.videoPath);
+    }
+  }
+}
+
+/**
+ * Direct Appium to start recording the device screen
+ *
+ * Record the display of devices running iOS Simulator since Xcode 9 or real devices since iOS 11
+ * (ffmpeg utility is required: 'brew install ffmpeg').
+ * It records screen activity to a MPEG-4 file. Audio is not recorded with the video file.
+ * If screen recording has been already started then the command will stop it forcefully and start a new one.
+ * The previously recorded video file will be deleted.
+ *
+ * @param {import('./types').StartRecordingScreenOptions} [options] - The available options.
+ * @returns {Promise<string>} Base64-encoded content of the recorded media file if
+ *                   any screen recording is currently running or an empty string.
+ * @throws {Error} If screen recording has failed to start.
+ * @this {XCUITestDriver}
+ */
+export async function startRecordingScreen(options = {}) {
+  const {
+    videoType = DEFAULT_VCODEC,
+    timeLimit = DEFAULT_RECORDING_TIME_SEC,
+    videoQuality = DEFAULT_QUALITY,
+    videoFps = DEFAULT_FPS,
+    videoFilters,
+    videoScale,
+    forceRestart,
+    pixelFormat,
+    hardwareAcceleration
+  } = options;
+
+  let result = '';
+  if (!forceRestart) {
+    this.log.info(
+      `Checking if there is/was a previous screen recording. ` +
+      `Set 'forceRestart' option to 'true' if you'd like to skip this step.`
+    );
+    result = (await this.stopRecordingScreen(options)) ?? result;
+  }
+
+  const videoPath = await tempDir.path({
+    prefix: `appium_${Math.random().toString(16).substring(2, 8)}`,
+    suffix: MP4_EXT
+  });
+
+  const wdaBaseUrl = this.opts.wdaBaseUrl || WDA_BASE_URL;
+  const screenRecorder = new ScreenRecorder(this.device.udid, this.log, videoPath, {
+    remotePort: this.opts.mjpegServerPort || DEFAULT_MJPEG_SERVER_PORT,
+    remoteUrl: wdaBaseUrl,
+    videoType,
+    videoFilters,
+    videoScale,
+    videoFps,
+    pixelFormat,
+    hardwareAcceleration
+  });
+  if (!(await screenRecorder.interrupt(true))) {
+    throw this.log.errorWithException('Unable to stop screen recording process');
+  }
+  if (this._recentScreenRecorder) {
+    await this._recentScreenRecorder.cleanup();
+    this._recentScreenRecorder = null;
+  }
+
+  const timeoutSeconds = parseFloat(String(timeLimit));
+  if (isNaN(timeoutSeconds) || timeoutSeconds > MAX_RECORDING_TIME_SEC || timeoutSeconds <= 0) {
+    throw this.log.errorWithException(
+      `The timeLimit value must be in range [1, ${MAX_RECORDING_TIME_SEC}] seconds. ` +
+        `The value of '${timeLimit}' has been passed instead.`,
+    );
+  }
+
+  let {mjpegServerScreenshotQuality, mjpegServerFramerate} =
+    /** @type {import('appium-webdriveragent').WDASettings} */ (
+    await this.proxyCommand('/appium/settings', 'GET')
+  );
+  if (videoQuality) {
+    const quality = _.isInteger(videoQuality)
+      ? videoQuality
+      : QUALITY_MAPPING[_.toLower(String(videoQuality))];
+    if (!quality) {
+      throw new Error(
+        `videoQuality value should be one of ${JSON.stringify(
+          _.keys(QUALITY_MAPPING)
+        )} or a number in range 1..100. ` + `'${videoQuality}' is given instead`
+      );
+    }
+    mjpegServerScreenshotQuality = mjpegServerScreenshotQuality !== quality ? quality : undefined;
+  } else {
+    mjpegServerScreenshotQuality = undefined;
+  }
+  if (videoFps) {
+    const fps = parseInt(String(videoFps), 10);
+    if (isNaN(fps)) {
+      throw new Error(
+        `videoFps value should be a valid number in range 1..60. ` +
+        `'${videoFps}' is given instead`
+      );
+    }
+    mjpegServerFramerate = mjpegServerFramerate !== fps ? fps : undefined;
+  } else {
+    mjpegServerFramerate = undefined;
+  }
+  if (util.hasValue(mjpegServerScreenshotQuality) || util.hasValue(mjpegServerFramerate)) {
+    await this.proxyCommand('/appium/settings', 'POST', {
+      settings: {
+        mjpegServerScreenshotQuality,
+        mjpegServerFramerate
+      }
+    });
+  }
+
+  try {
+    await screenRecorder.start(timeoutSeconds * 1000);
+  } catch (e) {
+    await screenRecorder.interrupt(true);
+    await screenRecorder.cleanup();
+    throw e;
+  }
+  this._recentScreenRecorder = screenRecorder;
+
+  return result;
+}
+
+/**
+ * Direct Appium to stop screen recording and return the video
+ *
+ * If no screen recording process is running then the endpoint will try to get
+ * the recently recorded file. If no previously recorded file is found and no
+ * active screen recording processes are running then the method returns an
+ * empty string.
+ *
+ * @param {import('./types').StopRecordingScreenOptions} options - The available
+ * options.
+ * @returns {Promise<string?>} Base64-encoded content of the recorded media
+ * file if `remotePath` parameter is empty or null or an empty string.
+ * @throws {Error} If there was an error while getting the name of a media
+ *                 file or the file content cannot be uploaded to the remote
+ *                 location.
+ * @this {XCUITestDriver}
+ */
+export async function stopRecordingScreen(options = {}) {
+  if (!this._recentScreenRecorder) {
+    this.log.info('Screen recording is not running. There is nothing to stop.');
+    return '';
+  }
+
+  try {
+    const videoPath = await this._recentScreenRecorder.finish();
+    if (!(await fs.exists(videoPath))) {
+      throw this.log.errorWithException(
+        `The screen recorder utility has failed ` +
+        `to store the actual screen recording at '${videoPath}'`
+      );
+    }
+    return await encodeBase64OrUpload(videoPath, options.remotePath, options);
+  } finally {
+    await this._recentScreenRecorder.interrupt(true);
+    await this._recentScreenRecorder.cleanup();
+    this._recentScreenRecorder = null;
+  }
+}
+
+/**
+ * @typedef {import('../driver').XCUITestDriver} XCUITestDriver
+ */

package/lib/commands/screenshots.js

@@ -0,0 +1,137 @@
+import {retryInterval} from 'asyncbox';
+import _ from 'lodash';
+import {errors} from 'appium/driver';
+import {util, imageUtil} from 'appium/support';
+
+/**
+ * @this {XCUITestDriver}
+ * @returns {Promise<string>}
+ */
+export async function getScreenshot() {
+  if (this.isWebContext()) {
+    const webScreenshotMode = (await this.settings.getSettings()).webScreenshotMode;
+    switch (_.toLower(webScreenshotMode)) {
+      case 'page':
+      case 'viewport':
+        return await (/** @type {import('appium-remote-debugger').RemoteDebugger} */ (this.remote)).captureScreenshot({
+          coordinateSystem: /** @type {'Viewport'|'Page'} */ (_.capitalize(webScreenshotMode)),
+        });
+      case 'native':
+      case undefined:
+      case null:
+        break;
+      default:
+        this.log.warn(
+          `The webScreenshotMode setting value '${webScreenshotMode}' is not known. ` +
+          `Supported values are: page, viewport and native. Falling back to the native mode.`
+        );
+        break;
+    }
+  }
+
+  const getScreenshotFromWDA = async () => {
+    this.log.debug(`Taking screenshot with WDA`);
+    const data = await this.proxyCommand('/screenshot', 'GET');
+    if (!_.isString(data)) {
+      throw new Error(`Unable to take screenshot. WDA returned '${JSON.stringify(data)}'`);
+    }
+    return data;
+  };
+
+  // if we've specified an mjpeg server, use that
+  if (this.mjpegStream) {
+    this.log.info(`mjpeg video stream provided, returning latest frame as screenshot`);
+    const data = await this.mjpegStream.lastChunkPNGBase64();
+    if (data) {
+      return data;
+    }
+    this.log.warn(
+      'Tried to get screenshot from active MJPEG stream, but there ' +
+        'was no data yet. Falling back to regular screenshot methods.',
+    );
+  }
+
+  try {
+    return await getScreenshotFromWDA();
+  } catch (err) {
+    this.log.warn(`Error getting screenshot: ${err.message}`);
+  }
+
+  // simulator attempt
+  if (this.isSimulator()) {
+    this.log.info(`Falling back to 'simctl io screenshot' API`);
+    const payload = await /** @type {import('../driver').Simulator} */ (this.device).simctl.getScreenshot();
+    if (!payload) {
+      throw new errors.UnableToCaptureScreen();
+    }
+    return payload;
+  }
+
+  // Retry for real devices only. Fail fast on Simulator if simctl does not work as expected
+  return /** @type {string} */ (await retryInterval(2, 1000, getScreenshotFromWDA));
+}
+
+/**
+ * @this {XCUITestDriver}
+ */
+export async function getElementScreenshot(el) {
+  el = util.unwrapElement(el);
+  if (this.isWebContext()) {
+    const atomsElement = this.getAtomsElement(el);
+    const {width, height} = await this.executeAtom('get_size', [atomsElement]);
+    if (!width || !height) {
+      throw new errors.UnableToCaptureScreen('Cannot take a screenshot of a zero-size element');
+    }
+    const {x, y} = await this.executeAtom('get_top_left_coordinates', [atomsElement]);
+    return await (/** @type {import('appium-remote-debugger').RemoteDebugger} */ (this.remote))
+      .captureScreenshot({rect: {x, y, width, height}});
+  }
+
+  const data = await this.proxyCommand(`/element/${el}/screenshot`, 'GET');
+  if (!_.isString(data)) {
+    throw new errors.UnableToCaptureScreen(
+      `Unable to take an element screenshot. WDA returned: ${JSON.stringify(data)}`,
+    );
+  }
+  return data;
+}
+
+/**
+ * @this {XCUITestDriver}
+ * @returns {Promise<string>}
+ */
+export async function getViewportScreenshot() {
+  if (this.isWebContext()) {
+    return await (/** @type {import('appium-remote-debugger').RemoteDebugger} */ (this.remote))
+      .captureScreenshot();
+  }
+
+  const screenshot = await this.getScreenshot();
+  // if we don't have a status bar, there's nothing to crop, so we can avoid
+  // extra calls and return straight away
+  if ((await this.getStatusBarHeight()) === 0) {
+    return screenshot;
+  }
+
+  const sharp = imageUtil.requireSharp();
+  const {width, height} = await sharp(Buffer.from(screenshot, 'base64')).metadata();
+  if (!width || !height) {
+    throw new errors.UnableToCaptureScreen('The device screenshot is empty');
+  }
+  this.log.debug(`Screenshot dimensions: ${width}x${height}`);
+  const region = await this.getViewportRect();
+  if (region.width + region.left > width) {
+    this.log.info('Viewport region exceeds screenshot width, adjusting region to fit');
+    region.width = width - region.left;
+  }
+  if (region.height + region.top > height) {
+    this.log.info('Viewport region exceeds screenshot height, adjusting region to fit');
+    region.height = height - region.top;
+  }
+  this.log.debug(`Calculated viewport rect: ${JSON.stringify(region)}`);
+  return await imageUtil.cropBase64Image(screenshot, region);
+}
+
+/**
+ * @typedef {import('../driver').XCUITestDriver} XCUITestDriver
+ */

package/lib/commands/simctl.js

@@ -0,0 +1,71 @@
+import { errors } from 'appium/driver';
+
+/**
+ * List of subcommands for `simctl` we provide as mobile simctl command.
+ * They accept 'device' target.
+ */
+const SUBCOMMANDS_HAS_DEVICE = [
+  'boot',
+  'get_app_container',
+  'getenv',
+  'icloud_sync',
+  'install',
+  'install_app_data',
+  'io',
+  'keychain',
+  'launch',
+  'location',
+  'logverbose',
+  'openurl',
+  'pbcopy',
+  'pbpaste',
+  'privacy',
+  'push',
+  'shutdown',
+  'spawn',
+  'status_bar',
+  'terminate',
+  'ui',
+  'uninstall'
+];
+
+/**
+ * Run the given command with arguments as `xcrun simctl` subcommand.
+ * This method works behind the 'simctl' security flag.
+ * @this {XCUITestDriver}
+ * @param {string} command Subcommand to run with `xcrun simctl`
+ * @param {string[]} [args=[]] arguments for the subcommand. The arguments should be after <device> in the help.
+ * @param {number|undefined} timeout - The maximum number of milliseconds
+ * @returns {Promise<SimctlExecResponse>}
+ * @throws {Error} If the simctl subcommand command returns non-zero return code, or the given subcommand was invalid.
+ */
+export async function mobileSimctl(command, args = [], timeout = undefined) {
+  if (!this.isSimulator()) {
+    throw new errors.UnsupportedOperationError(`Only simulator is supported.`);
+  }
+
+  if (!this.opts.udid) {
+    throw new errors.InvalidArgumentError(`Unknown device or simulator UDID: '${this.opts.udid}'`);
+  }
+
+  if (!SUBCOMMANDS_HAS_DEVICE.includes(command)) {
+    throw new errors.InvalidArgumentError(`The given command '${command}' is not supported. ` +
+      `Available subcommands are ${SUBCOMMANDS_HAS_DEVICE.join(',')}`);
+  }
+
+  return await /** @type {import('./../driver').Simulator} */ (this.device).simctl.exec(
+    command,
+    {args: [this.opts.udid, ...args], timeout}
+  );
+}
+
+/**
+ * @typedef {Object} SimctlExecResponse
+ * @property {Buffer<ArrayBufferLike>} stdout The output of standard out.
+ * @property {Buffer<ArrayBufferLike>} stderr The output of standard error.
+ * @property {number} code Return code.
+ */
+
+/**
+ * @typedef {import('../driver').XCUITestDriver} XCUITestDriver
+ */