appium-chromedriver 5.5.2 → 5.6.0

package/lib/storage-client/googleapis.js

@@ -0,0 +1,209 @@
+import _ from 'lodash';
+import xpath from 'xpath';
+import {util, logger} from '@appium/support';
+import {retrieveData} from '../utils';
+import B from 'bluebird';
+import {
+  STORAGE_REQ_TIMEOUT_MS,
+  GOOGLEAPIS_CDN,
+  ARCH,
+  CPU,
+  APPLE_ARM_SUFFIXES,
+} from '../constants';
+import {DOMParser} from '@xmldom/xmldom';
+import path from 'path';
+
+
+const log = logger.getLogger('ChromedriverGoogleapisStorageClient');
+const MAX_PARALLEL_DOWNLOADS = 5;
+
+/**
+ *
+ * @param {Node|Attr} parent
+ * @param {string?} childName
+ * @param {string?} text
+ * @returns
+ */
+export function findChildNode(parent, childName = null, text = null) {
+  if (!childName && !text) {
+    return null;
+  }
+  if (!parent.hasChildNodes()) {
+    return null;
+  }
+
+  for (let childNodeIdx = 0; childNodeIdx < parent.childNodes.length; childNodeIdx++) {
+    const childNode = /** @type {Element|Attr} */ (parent.childNodes[childNodeIdx]);
+    if (childName && !text && childName === childNode.localName) {
+      return childNode;
+    }
+    if (text) {
+      const childText = extractNodeText(childNode);
+      if (!childText) {
+        continue;
+      }
+      if (childName && childName === childNode.localName && text === childText) {
+        return childNode;
+      }
+      if (!childName && text === childText) {
+        return childNode;
+      }
+    }
+  }
+  return null;
+}
+
+/**
+ *
+ * @param {Node?} node
+ * @returns
+ */
+function extractNodeText(node) {
+  return !node || !node.firstChild || !util.hasValue(node.firstChild.nodeValue)
+    ? null
+    : node.firstChild.nodeValue;
+}
+
+/**
+ * Gets additional chromedriver details from chromedriver
+ * release notes
+ *
+ * @param {string} content - Release notes of the corresponding chromedriver
+ * @returns {import('../types').AdditionalDriverDetails}
+ */
+export function parseNotes(content) {
+  const result = {};
+  const versionMatch = /^\s*[-]+ChromeDriver[\D]+([\d.]+)/im.exec(content);
+  if (versionMatch) {
+    result.version = versionMatch[1];
+  }
+  const minBrowserVersionMatch = /^\s*Supports Chrome[\D]+(\d+)/im.exec(content);
+  if (minBrowserVersionMatch) {
+    result.minBrowserVersion = minBrowserVersionMatch[1];
+  }
+  return result;
+}
+
+/**
+ * Parses chromedriver storage XML and returns
+ * the parsed results
+ *
+ * @param {string} xml - The chromedriver storage XML
+ * @param {boolean} shouldParseNotes [true] - If set to `true`
+ * then additional drivers information is going to be parsed
+ * and assigned to `this.mapping`
+ * @returns {Promise<ChromedriverDetailsMapping>}
+ */
+export async function parseGoogleapiStorageXml(xml, shouldParseNotes = true) {
+  const doc = new DOMParser().parseFromString(xml);
+  const driverNodes = /** @type {Array<Node|Attr>} */ (
+    xpath.select(`//*[local-name(.)='Contents']`, doc)
+  );
+  log.debug(`Parsed ${driverNodes.length} entries from storage XML`);
+  if (_.isEmpty(driverNodes)) {
+    throw new Error('Cannot retrieve any valid Chromedriver entries from the storage config');
+  }
+
+  const promises = [];
+  const chunk = [];
+  /** @type {ChromedriverDetailsMapping} */
+  const mapping = {};
+  for (const driverNode of driverNodes) {
+    const k = extractNodeText(findChildNode(driverNode, 'Key'));
+    if (!_.includes(k, '/chromedriver_')) {
+      continue;
+    }
+    const key = String(k);
+
+    const etag = extractNodeText(findChildNode(driverNode, 'ETag'));
+    if (!etag) {
+      log.debug(`The entry '${key}' does not contain the checksum. Skipping it`);
+      continue;
+    }
+
+    const filename = path.basename(key);
+    const osNameMatch = /_([a-z]+)/i.exec(filename);
+    if (!osNameMatch) {
+      log.debug(`The entry '${key}' does not contain valid OS name. Skipping it`);
+      continue;
+    }
+
+    /** @type {ChromedriverDetails} */
+    const cdInfo = {
+      url: `${GOOGLEAPIS_CDN}/${key}`,
+      etag: _.trim(etag, '"'),
+      version: /** @type {string} */ (_.first(key.split('/'))),
+      minBrowserVersion: null,
+      os: {
+        name: osNameMatch[1],
+        arch: filename.includes(ARCH.X64) ? ARCH.X64 : ARCH.X86,
+        cpu: APPLE_ARM_SUFFIXES.some((suffix) => filename.includes(suffix)) ? CPU.ARM : CPU.INTEL,
+      }
+    };
+    mapping[key] = cdInfo;
+
+    const notesPath = `${cdInfo.version}/notes.txt`;
+    const isNotesPresent = !!driverNodes.reduce(
+      (acc, node) => Boolean(acc || findChildNode(node, 'Key', notesPath)),
+      false
+    );
+    if (!isNotesPresent) {
+      cdInfo.minBrowserVersion = null;
+      if (shouldParseNotes) {
+        log.info(`The entry '${key}' does not contain any notes. Skipping it`);
+      }
+      continue;
+    } else if (!shouldParseNotes) {
+      continue;
+    }
+
+    const promise = B.resolve(retrieveAdditionalDriverInfo(key, `${GOOGLEAPIS_CDN}/${notesPath}`, cdInfo));
+    promises.push(promise);
+    chunk.push(promise);
+    if (chunk.length >= MAX_PARALLEL_DOWNLOADS) {
+      await B.any(chunk);
+    }
+    _.remove(chunk, (p) => p.isFulfilled());
+  }
+  await B.all(promises);
+  log.info(`The total count of entries in the mapping: ${_.size(mapping)}`);
+  return mapping;
+}
+
+/**
+ * Downloads chromedriver release notes and puts them
+ * into the dictionary argument
+ *
+ * The method call mutates by merging `AdditionalDriverDetails`
+ * @param {string} driverKey - Driver version plus archive name
+ * @param {string} notesUrl - The URL of chromedriver notes
+ * @param {ChromedriverDetails} infoDict - The dictionary containing driver info.
+ * @param {number} timeout
+ * @throws {Error} if the release notes cannot be downloaded
+ */
+async function retrieveAdditionalDriverInfo(driverKey, notesUrl, infoDict, timeout = STORAGE_REQ_TIMEOUT_MS) {
+  const notes = await retrieveData(
+    notesUrl,
+    {
+      'user-agent': 'appium',
+      accept: '*/*',
+    },
+    {timeout}
+  );
+  const {minBrowserVersion} = parseNotes(notes);
+  if (!minBrowserVersion) {
+    log.debug(
+      `The driver '${driverKey}' does not contain valid release notes at ${notesUrl}. ` +
+        `Skipping it`
+    );
+    return;
+  }
+  infoDict.minBrowserVersion = minBrowserVersion;
+}
+
+/**
+ * @typedef {import('../types').SyncOptions} SyncOptions
+ * @typedef {import('../types').OSInfo} OSInfo
+ * @typedef {import('../types').ChromedriverDetails} ChromedriverDetails
+ * @typedef {import('../types').ChromedriverDetailsMapping} ChromedriverDetailsMapping
+ */

package/lib/{storage-client.js → storage-client/storage-client.js}

@@ -1,23 +1,28 @@
 import {
   getChromedriverDir,
-  CD_CDN,
   retrieveData,
   getOsInfo,
-  OS,
-  X64,
-  X86,
-  APPLE_ARM_SUFFIXES,
   convertToInt,
-} from './utils';
+  getCpuType,
+} from '../utils';
 import _ from 'lodash';
-import xpath from 'xpath';
-import {DOMParser} from '@xmldom/xmldom';
 import B from 'bluebird';
 import path from 'path';
-import os from 'os';
 import {system, fs, logger, tempDir, zip, util, net} from '@appium/support';
+import {
+  STORAGE_REQ_TIMEOUT_MS,
+  GOOGLEAPIS_CDN,
+  USER_AGENT,
+  CHROMELABS_URL,
+  ARCH,
+  OS,
+  CPU,
+} from '../constants';
+import {parseGoogleapiStorageXml} from './googleapis';
+import {parseKnownGoodVersionsWithDownloadsJson} from './chromelabs';
+import {compareVersions} from 'compare-versions';
+import semver from 'semver';
 
-const TIMEOUT_MS = 15000;
 const MAX_PARALLEL_DOWNLOADS = 5;
 
 const log = logger.getLogger('ChromedriverStorageClient');
@@ -33,182 +38,19 @@ async function isCrcOk(src, checksum) {
   return _.toLower(md5) === _.toLower(checksum);
 }
 
-/**
- *
- * @param {Node|Attr} parent
- * @param {string?} childName
- * @param {string?} text
- * @returns
- */
-function findChildNode(parent, childName = null, text = null) {
-  if (!childName && !text) {
-    return null;
-  }
-  if (!parent.hasChildNodes()) {
-    return null;
-  }
-
-  for (let childNodeIdx = 0; childNodeIdx < parent.childNodes.length; childNodeIdx++) {
-    const childNode = /** @type {Element|Attr} */ (parent.childNodes[childNodeIdx]);
-    if (childName && !text && childName === childNode.localName) {
-      return childNode;
-    }
-    if (text) {
-      const childText = extractNodeText(childNode);
-      if (!childText) {
-        continue;
-      }
-      if (childName && childName === childNode.localName && text === childText) {
-        return childNode;
-      }
-      if (!childName && text === childText) {
-        return childNode;
-      }
-    }
-  }
-  return null;
-}
-
-/**
- *
- * @param {Node?} node
- * @returns
- */
-function extractNodeText(node) {
-  return !node || !node.firstChild || !util.hasValue(node.firstChild.nodeValue)
-    ? null
-    : node.firstChild.nodeValue;
-}
-
 export class ChromedriverStorageClient {
   /**
    *
-   * @param {import('./types').ChromedriverStorageClientOpts} args
+   * @param {import('../types').ChromedriverStorageClientOpts} args
    */
   constructor(args = {}) {
-    const {chromedriverDir = getChromedriverDir(), timeout = TIMEOUT_MS} = args;
+    const {chromedriverDir = getChromedriverDir(), timeout = STORAGE_REQ_TIMEOUT_MS} = args;
     this.chromedriverDir = chromedriverDir;
     this.timeout = timeout;
     /** @type {ChromedriverDetailsMapping} */
     this.mapping = {};
   }
 
-  /**
-   * Gets additional chromedriver details from chromedriver
-   * release notes
-   *
-   * @param {string} content - Release notes of the corresponding chromedriver
-   * @returns {import('./types').AdditionalDriverDetails}
-   */
-  parseNotes(content) {
-    const result = {};
-    const versionMatch = /^\s*[-]+ChromeDriver[\D]+([\d.]+)/im.exec(content);
-    if (versionMatch) {
-      result.version = versionMatch[1];
-    }
-    const minBrowserVersionMatch = /^\s*Supports Chrome[\D]+(\d+)/im.exec(content);
-    if (minBrowserVersionMatch) {
-      result.minBrowserVersion = minBrowserVersionMatch[1];
-    }
-    return result;
-  }
-
-  /**
-   * Downloads chromedriver release notes and puts them
-   * into the dictionary argument
-   *
-   * The method call mutates by merging `AdditionalDriverDetails`
-   * @param {string} driverKey - Driver version plus archive name
-   * @param {string} notesUrl - The URL of chromedriver notes
-   * @param {ChromedriverDetails} infoDict - The dictionary containing driver info.
-   * @throws {Error} if the release notes cannot be downloaded
-   */
-  async retrieveAdditionalDriverInfo(driverKey, notesUrl, infoDict) {
-    const notes = await retrieveData(
-      notesUrl,
-      {
-        'user-agent': 'appium',
-        accept: '*/*',
-      },
-      {timeout: this.timeout}
-    );
-    const {minBrowserVersion} = this.parseNotes(notes);
-    if (!minBrowserVersion) {
-      log.debug(
-        `The driver '${driverKey}' does not contain valid release notes at ${notesUrl}. ` +
-          `Skipping it`
-      );
-      return;
-    }
-    infoDict.minBrowserVersion = minBrowserVersion;
-  }
-
-  /**
-   * Parses chromedriver storage XML and stores
-   * the parsed results into `this.mapping`
-   *
-   * @param {Document} doc - The DOM representation
-   * of the chromedriver storage XML
-   * @param {boolean} shouldParseNotes [true] - If set to `true`
-   * then additional drivers information is going to be parsed
-   * and assigned to `this.mapping`
-   */
-  async parseStorageXml(doc, shouldParseNotes = true) {
-    const driverNodes = /** @type {Array<Node|Attr>} */ (
-      xpath.select(`//*[local-name(.)='Contents']`, doc)
-    );
-    log.debug(`Parsed ${driverNodes.length} entries from storage XML`);
-    if (_.isEmpty(driverNodes)) {
-      return;
-    }
-
-    const promises = [];
-    for (const driverNode of driverNodes) {
-      const k = extractNodeText(findChildNode(driverNode, 'Key'));
-      if (!_.includes(k, '/chromedriver_')) {
-        continue;
-      }
-      const key = String(k);
-
-      const etag = extractNodeText(findChildNode(driverNode, 'ETag'));
-      if (!etag) {
-        log.debug(`The entry '${key}' does not contain the checksum. Skipping it`);
-        continue;
-      }
-
-      /** @type {ChromedriverDetails} */
-      const cdInfo = {
-        url: `${CD_CDN}/${key}`,
-        etag: _.trim(etag, '"'),
-        version: /** @type {string} */ (_.first(key.split('/'))),
-        minBrowserVersion: null,
-      };
-      this.mapping[key] = cdInfo;
-
-      const notesPath = `${cdInfo.version}/notes.txt`;
-      const isNotesPresent = !!driverNodes.reduce(
-        (acc, node) => Boolean(acc || findChildNode(node, 'Key', notesPath)),
-        false
-      );
-      if (!isNotesPresent) {
-        cdInfo.minBrowserVersion = null;
-        if (shouldParseNotes) {
-          log.info(`The entry '${key}' does not contain any notes. Skipping it`);
-        }
-        continue;
-      } else if (!shouldParseNotes) {
-        continue;
-      }
-
-      promises.push(this.retrieveAdditionalDriverInfo(key, `${CD_CDN}/${notesPath}`, cdInfo));
-      if (promises.length % MAX_PARALLEL_DOWNLOADS === 0) {
-        await B.all(promises);
-      }
-    }
-    await B.all(promises);
-    log.info(`The total count of entries in the mapping: ${_.size(this.mapping)}`);
-  }
-
   /**
    * Retrieves chromedriver mapping from the storage
    *
@@ -218,17 +60,20 @@ export class ChromedriverStorageClient {
    * @returns {Promise<ChromedriverDetailsMapping>}
    */
   async retrieveMapping(shouldParseNotes = true) {
-    const xml = await retrieveData(
-      CD_CDN,
-      {
-        'user-agent': 'appium',
-        accept: 'application/xml, */*',
-      },
-      {timeout: this.timeout}
-    );
-    const doc = new DOMParser().parseFromString(xml);
-    await this.parseStorageXml(doc, shouldParseNotes);
-    return _.cloneDeep(this.mapping);
+    const [xmlStr, jsonStr] = await B.all([
+      [GOOGLEAPIS_CDN, 'application/xml'],
+      [`${CHROMELABS_URL}/chrome-for-testing/known-good-versions-with-downloads.json`, 'application/json'],
+    ]
+    .map(([url, contentType]) => url
+      ? retrieveData(url, {
+          'user-agent': USER_AGENT,
+          accept: `${contentType}, */*`,
+        }, {timeout: this.timeout})
+      : B.resolve()
+    ));
+    this.mapping = xmlStr ? await parseGoogleapiStorageXml(xmlStr, shouldParseNotes) : {};
+    Object.assign(this.mapping, parseKnownGoodVersionsWithDownloadsJson(jsonStr));
+    return this.mapping;
   }
 
   /**
@@ -328,30 +173,93 @@ export class ChromedriverStorageClient {
 
     if (!_.isEmpty(osInfo)) {
       // Filter out drivers for unsupported system architectures
-      let {name, arch} = osInfo;
-      if (arch === X64 && !driversToSync.some((cdName) => cdName.includes(`_${name}${X64}`))) {
-        // Fall back to x86 build if x64 one is not available for the given OS
-        arch = X86;
+      const {name, arch, cpu = getCpuType()} = osInfo;
+      log.debug(`Selecting chromedrivers whose platform matches to ${name}:${cpu}${arch}`);
+      let result = driversToSync.filter((cdName) => this.doesMatchForOsInfo(cdName, osInfo));
+      if (_.isEmpty(result) && arch === ARCH.X64 && cpu === CPU.INTEL) {
+        // Fallback to X86 if X64 architecture is not available for this driver
+        result = driversToSync.filter((cdName) => this.doesMatchForOsInfo(cdName, {
+          name, arch: ARCH.X86, cpu
+        }));
       }
-      // https://stackoverflow.com/questions/65146751/detecting-apple-silicon-mac-in-javascript
-      if (name === OS.mac && _.includes(_.toLower(os.cpus()[0].model), 'apple')) {
-        for (const armSuffix of APPLE_ARM_SUFFIXES) {
-          if (driversToSync.some((cdName) => cdName.includes(armSuffix))) {
-            // prefer executable for ARM arch if present
-            arch = armSuffix;
-            break;
-          }
-        }
+      if (_.isEmpty(result) && name === OS.MAC && cpu === CPU.ARM) {
+        // Fallback to Intel/Rosetta if ARM architecture is not available for this driver
+        result = driversToSync.filter((cdName) => this.doesMatchForOsInfo(cdName, {
+          name, arch, cpu: CPU.INTEL
+        }));
       }
-      log.debug(`Selecting chromedrivers whose platform matches to ${name}${arch}`);
-      const platformRe = new RegExp(`(\\b|_)${name}${arch}\\b`);
-      driversToSync = driversToSync.filter((cdName) => platformRe.test(cdName));
+      driversToSync = result;
       log.debug(`Got ${util.pluralize('item', driversToSync.length, true)}`);
     }
 
+    if (!_.isEmpty(driversToSync)) {
+      log.debug('Excluding older patches if present');
+      /** @type {{[key: string]: string[]}} */
+      const patchesMap = {};
+      // Older chromedrivers must not be excluded as they follow a different
+      // versioning pattern
+      const versionWithPatchPattern = /\d+\.\d+\.\d+\.\d+/;
+      const selectedVersions = new Set();
+      for (const cdName of driversToSync) {
+        const cdVersion = this.mapping[cdName].version;
+        if (!versionWithPatchPattern.test(cdVersion)) {
+          selectedVersions.add(cdVersion);
+          continue;
+        }
+        const verObj = semver.parse(cdVersion, {loose: true});
+        if (!verObj) {
+          continue;
+        }
+        if (!_.isArray(patchesMap[verObj.major])) {
+          patchesMap[verObj.major] = [];
+        }
+        patchesMap[verObj.major].push(cdVersion);
+      }
+      for (const majorVersion of _.keys(patchesMap)) {
+        if (patchesMap[majorVersion].length <= 1) {
+          continue;
+        }
+        patchesMap[majorVersion].sort(
+          (/** @type {string} */ a, /** @type {string}} */ b) => compareVersions(b, a)
+        );
+      }
+      if (!_.isEmpty(patchesMap)) {
+        log.debug('Versions mapping: ' + JSON.stringify(patchesMap, null, 2));
+        for (const sortedVersions of _.values(patchesMap)) {
+          selectedVersions.add(sortedVersions[0]);
+        }
+        driversToSync = driversToSync.filter(
+          (cdName) => selectedVersions.has(this.mapping[cdName].version)
+        );
+      }
+    }
+
     return driversToSync;
   }
 
+  /**
+   * Checks whether the given chromedriver matches the operating system to run on
+   *
+   * @param {string} cdName
+   * @param {OSInfo} osInfo
+   * @returns {boolean}
+   */
+  doesMatchForOsInfo(cdName, {name, arch, cpu}) {
+    const cdInfo = this.mapping[cdName];
+    if (!cdInfo) {
+      return false;
+    }
+
+    if (cdInfo.os.name !== name || cdInfo.os.arch !== arch) {
+      return false;
+    }
+    if (cpu && cdInfo.os.cpu && this.mapping[cdName].os.cpu !== cpu) {
+      return false;
+    }
+
+    return true;
+  }
+
   /**
    * Retrieves the given chromedriver from the storage
    * and unpacks it into `this.chromedriverDir` folder
@@ -374,7 +282,7 @@ export class ChromedriverStorageClient {
     try {
       await net.downloadFile(url, archivePath, {
         isMetered: false,
-        timeout: TIMEOUT_MS,
+        timeout: STORAGE_REQ_TIMEOUT_MS,
       });
     } catch (e) {
       const err = /** @type {Error} */ (e);
@@ -385,7 +293,7 @@ export class ChromedriverStorageClient {
       log.error(msg);
       return false;
     }
-    if (!(await isCrcOk(archivePath, etag))) {
+    if (etag && !(await isCrcOk(archivePath, etag))) {
       const msg = `The checksum for the downloaded chromedriver '${driverKey}' did not match`;
       if (isStrict) {
         throw new Error(msg);
@@ -442,20 +350,23 @@ export class ChromedriverStorageClient {
      */
     const synchronizedDrivers = [];
     const promises = [];
+    const chunk = [];
     const archivesRoot = await tempDir.openDir();
     try {
       for (const [idx, driverKey] of driversToSync.entries()) {
-        promises.push(
+        const promise = B.resolve(
           (async () => {
             if (await this.retrieveDriver(idx, driverKey, archivesRoot, !_.isEmpty(opts))) {
               synchronizedDrivers.push(driverKey);
             }
           })()
         );
-
-        if (promises.length % MAX_PARALLEL_DOWNLOADS === 0) {
-          await B.all(promises);
+        promises.push(promise);
+        chunk.push(promise);
+        if (chunk.length >= MAX_PARALLEL_DOWNLOADS) {
+          await B.any(chunk);
         }
+        _.remove(chunk, (p) => p.isFulfilled());
       }
       await B.all(promises);
     } finally {
@@ -464,7 +375,7 @@ export class ChromedriverStorageClient {
     if (!_.isEmpty(synchronizedDrivers)) {
       log.info(
         `Successfully synchronized ` +
-          `${util.pluralize('chromedriver', synchronizedDrivers.length, true)}`
+        `${util.pluralize('chromedriver', synchronizedDrivers.length, true)}`
       );
     } else {
       log.info(`No chromedrivers were synchronized`);
@@ -476,8 +387,8 @@ export class ChromedriverStorageClient {
 export default ChromedriverStorageClient;
 
 /**
- * @typedef {import('./types').SyncOptions} SyncOptions
- * @typedef {import('./types').OSInfo} OSInfo
- * @typedef {import('./types').ChromedriverDetails} ChromedriverDetails
- * @typedef {import('./types').ChromedriverDetailsMapping} ChromedriverDetailsMapping
+ * @typedef {import('../types').SyncOptions} SyncOptions
+ * @typedef {import('../types').OSInfo} OSInfo
+ * @typedef {import('../types').ChromedriverDetails} ChromedriverDetails
+ * @typedef {import('../types').ChromedriverDetailsMapping} ChromedriverDetailsMapping
  */

package/lib/types.ts

@@ -46,15 +46,20 @@ export interface SyncOptions {
 export interface OSInfo {
   /**
    * The architecture of the host OS.
-   * Can be either `32` or `64`
+   * Can be either `32`, `64``
    */
   arch: string;
   /**
    *
    * The name of the host OS.
-   * Can be either `mac`, `windows` or `linux`
+   * Can be either `mac`, `win` or `linux`
    */
   name: string;
+  /**
+   * The cpu type of the host OS.
+   * Can be either `intel`, `arm`. `intel` is assumed by default
+   */
+  cpu?: string;
 }
 
 /**
@@ -68,12 +73,13 @@ export interface ChromedriverDetails {
   /**
    * CRC of driver archive
    */
-  etag: string;
+  etag: string | null;
   /**
    * Chromedriver version
    */
   version: string;
   minBrowserVersion: string | null;
+  os: OSInfo;
 }
 
 /**