@eyeo/get-browser-binary 0.14.0 → 0.16.0

package/.gitlab-ci.yml

@@ -29,9 +29,9 @@ test:basic:
 
 test:browsers:linux:
   stage: test
-  image: docker:20.10.16
+  image: docker:24.0.5
   services:
-    - docker:20.10.16-dind
+    - docker:24.0.5-dind
   before_script:
     - docker build -f test/docker/Dockerfile -t browsers .
   script:

package/README.md

@@ -32,7 +32,7 @@ the right side.
 
 - Chromium >= 77 (Chromium ARM >= 92)
 - Firefox >= 68
-- Edge >= 95
+- Edge >= 95 (Windows Edge >= 79)
 
 Note: Installing Edge is not supported on Windows. It is assumed to be installed
 because it is the default browser on that platform. On macOS, only the latest
@@ -133,20 +133,16 @@ docker run --shm-size=512m -e TEST_ARGS="--grep chromium.*latest --timeout 10000
 
 #### ARM architecture (M1/M2 Apple Silicon)
 
-```shell
-docker build -f test/docker/arm64.Dockerfile -t browsers-arm .
-docker run --shm-size=512m -e TEST_ARGS="--grep (chromium|firefox).*latest" -it browsers-arm
-```
+The run is done emulating the AMD architecture. Requirements:
 
-Only latest Chromium and latest Firefox ESR versions are supported. Edge is not
-supported.
+- macOS >= 13 (Ventura)
+- Rosetta
+- The feature "Use Rosetta for x86/amd64 emulation on Apple Silicon" enabled in Docker
 
-Regarding Firefox, it may be possible to run other versions using AMD emulation
-(unstable results):
+The `--platform` option should be used when running the image:
 
 ```shell
-docker build -f test/docker/Dockerfile -t browsers-amd .
-docker run --platform linux/amd64 --shm-size=512m -e TEST_ARGS="--grep firefox.*68.0" -it browsers-amd
+docker run --platform linux/amd64 --shm-size=512m -e TEST_ARGS="--grep chromium.*latest" -it browsers
 ```
 
 ## Building the documentation

package/RELEASE_NOTES.md

@@ -1,3 +1,37 @@
+# 0.16.0
+
+- Starts using selenium's automated driver management (#67)
+- Loads the extension in Firefox by local path instead of being copied to a
+temporary folder (!104)
+- Checks if extensions can be loaded beforehand, by ensuring they contain a
+`manifest.json` file. If not, a new `Extension manifest file not found` error
+will be thrown (#71)
+- Refactors headless runs by not using webdriver's deprecated headless() method
+(#72)
+
+### Testing
+
+- Fixes an issue with `npm test` which would fail when the `grep` option would
+include parenthesis (#69)
+- Adds the browser version information in the "runs" test (#74)
+
+### Notes for integrators
+
+- Please make sure that your selenium-webdriver package version is at least
+4.15.0 (the one used in this release) (#67)
+- If you experience chromedriver issues, try deleting the `/browser-snapshots`
+cache folder.
+
+# 0.15.0
+
+- Fixes the downloads of Chromium versions < 91 by replacing the remaining
+usages of omahaproxy API (now removed) with chromiumdash (#55)
+- Updates to selenium webdriver 4.15.0 (#66)
+
+### Testing
+
+- Uses AMD emulation on ARM docker test runs (!93)
+
 # 0.14.0
 
 - Increases minimum supported browser versions (#64)

package/index.js

@@ -19,7 +19,7 @@ import {Chromium} from "./src/chromium.js";
 import {Firefox} from "./src/firefox.js";
 import {Edge} from "./src/edge.js";
 
-export {download, takeFullPageScreenshot, snapshotsBaseDir}
+export {download, takeFullPageScreenshot, snapshotsBaseDir, getMajorVersion}
   from "./src/utils.js";
 
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@eyeo/get-browser-binary",
-  "version": "0.14.0",
+  "version": "0.16.0",
   "description": "Install browser binaries and matching webdrivers",
   "repository": {
     "type": "git",
@@ -33,7 +33,7 @@
     "geckodriver": "3.1.0",
     "got": "^12.5.3",
     "jimp": "^0.22.4",
-    "selenium-webdriver": "^4.8.0"
+    "selenium-webdriver": "^4.15.0"
   },
   "devDependencies": {
     "eslint": "^8.33.0",

package/src/browser.js

@@ -84,11 +84,11 @@ export class Browser {
   /**
    * @typedef {Object} driverOptions
    * @property {boolean} [headless=true] Run the browser in headless mode,
-   *   or not. In Chromium >= 111, the
+   *   or not. In Chromium >= 111 and Edge >= 114 the
    *   {@link https://developer.chrome.com/articles/new-headless/ new headless mode}
    *   is used.
-   * @property {Array.<string>} [extensionPaths=[]] Loads extensions to the
-   *   browser.
+   * @property {Array.<string>} [extensionPaths=[]] Paths to folders containing
+   *   unpacked extensions which will be loaded to the browser.
    * @property {boolean} [incognito=false] Runs the browser in incognito mode,
    *   or not.
    * @property {boolean} [insecure=false] Forces the browser to accept insecure
@@ -110,7 +110,8 @@ export class Browser {
    *   browser install files to complete. When set to 0 there is no time limit.
    * @return {webdriver}
    * @throws {Error} Unsupported browser version, Unsupported platform, Browser
-   *   download failed, Driver download failed, Unable to start driver.
+   *   download failed, Driver download failed, Unable to start driver, Manifest
+   *   file not found.
    */
   static async getDriver(version, options = {}, downloadTimeout = 0) {
     // to be implemented by the subclass

package/src/chromium.js

@@ -24,9 +24,8 @@ import chrome from "selenium-webdriver/chrome.js";
 import extractZip from "extract-zip";
 
 import {Browser} from "./browser.js";
-import {download, killDriverProcess, getMajorVersion, checkVersion,
-        checkPlatform, errMsg, snapshotsBaseDir, platformArch}
-  from "./utils.js";
+import {download, getMajorVersion, checkVersion, checkPlatform, errMsg,
+        snapshotsBaseDir, platformArch, checkExtensionPaths} from "./utils.js";
 
 /**
  * Browser and webdriver functionality for Chromium.
@@ -85,10 +84,20 @@ export class Chromium extends Browser {
     let url;
     let chromiumBase;
     try {
-      // https://gitlab.com/eyeo/developer-experience/get-browser-binary/-/issues/55
-      if (getMajorVersion(chromiumVersion) < 91) {
-        url = `https://omahaproxy.appspot.com/deps.json?version=${chromiumVersion}`;
-        ({chromium_base_position: chromiumBase} = await got(url).json());
+      let majorVersion = getMajorVersion(chromiumVersion);
+      if (majorVersion < 91) {
+        // Below v91, base branch position only exists once per milestone in
+        // the /fetch_milestones endpoint
+        url = "https://chromiumdash.appspot.com/fetch_milestones?only_branched=true";
+        let data = await got(url).json();
+        for (let {milestone, chromium_main_branch_position: base} of data) {
+          if (milestone == majorVersion) {
+            chromiumBase = base;
+            break;
+          }
+        }
+        if (!chromiumBase)
+          throw new Error(`${errMsg.browserVersionCheck}: ${url}`);
       }
       else {
         url = `https://chromiumdash.appspot.com/fetch_version?version=${chromiumVersion}`;
@@ -106,8 +115,7 @@ export class Chromium extends Browser {
     // Linux example: "Chromium 112.0.5615.49 built on Debian 11.6"
     // Windows example: "114.0.5735.0"
     let versionNumber = installedVersion.split(" ")[1] || installedVersion;
-    let base = await Chromium.#getBase(versionNumber);
-    return {binary, versionNumber, base};
+    return {binary, versionNumber};
   }
 
   /**
@@ -124,18 +132,10 @@ export class Chromium extends Browser {
   static async installBrowser(version = "latest", downloadTimeout = 0) {
     const MIN_VERSION = process.arch == "arm64" ? 92 : 77;
 
-    let binary;
-    let versionNumber;
-    let base;
-
-    // https://gitlab.com/eyeo/developer-experience/get-browser-binary/-/issues/46
-    if (platformArch == "linux-arm64")
-      return await Chromium.#getInstalledBrowserInfo("/usr/bin/chromium");
-
     checkVersion(version, MIN_VERSION, Chromium.#CHANNELS);
-    versionNumber = await Chromium.#getVersionForChannel(version);
+    let versionNumber = await Chromium.#getVersionForChannel(version);
 
-    base = await Chromium.#getBase(versionNumber);
+    let base = await Chromium.#getBase(versionNumber);
     let startBase = base;
     let [platformDir, fileName] = {
       "win32-ia32": ["Win", "chrome-win.zip"],
@@ -148,6 +148,7 @@ export class Chromium extends Browser {
     let browserDir;
     let snapshotsDir = path.join(snapshotsBaseDir, "chromium");
 
+    let binary;
     while (true) {
       browserDir = path.join(snapshotsDir, `chromium-${platformArch}-${base}`);
       binary = Chromium.#getBinaryPath(browserDir);
@@ -186,72 +187,7 @@ export class Chromium extends Browser {
     }
     await extractZip(archive, {dir: browserDir});
 
-    return {binary, versionNumber, base};
-  }
-
-  static async #installDriver(startBase) {
-    let [dir, zip, driverBinary] = {
-      "win32-ia32": ["Win", "chromedriver_win32.zip", "chromedriver.exe"],
-      "win32-x64": ["Win_x64", "chromedriver_win32.zip", "chromedriver.exe"],
-      "linux-x64": ["Linux_x64", "chromedriver_linux64.zip", "chromedriver"],
-      "linux-arm64": ["", "", "chromedriver"],
-      "darwin-x64": ["Mac", "chromedriver_mac64.zip", "chromedriver"],
-      "darwin-arm64": ["Mac_Arm", "chromedriver_mac64.zip", "chromedriver"]
-    }[platformArch];
-
-    let cacheDir = path.join(snapshotsBaseDir, "chromium", "cache",
-                             "chromedriver");
-    let archive = path.join(cacheDir, `${startBase}-${zip}`);
-    try {
-      await fs.promises.access(archive);
-      await extractZip(archive, {dir: cacheDir});
-    }
-    catch (e) { // zip file is either not cached or corrupted
-      if (platformArch == "linux-arm64") {
-        // https://gitlab.com/eyeo/developer-experience/get-browser-binary/-/issues/46
-        // It's unclear how electron releases match Chromium versions. Once that
-        // is figured out, the link below will depend on the Chromium version.
-        // https://stackoverflow.com/questions/38732822/compile-chromedriver-on-arm
-        let url = "https://github.com/electron/electron/releases/download/v26.2.0/chromedriver-v26.2.0-linux-arm64.zip";
-        try {
-          await download(url, archive);
-        }
-        catch (err) {
-          throw new Error(`${errMsg.driverDownload}: ${url}\n${err}`);
-        }
-      }
-      else {
-        let base = startBase;
-        while (true) {
-          let url = `https://commondatastorage.googleapis.com/chromium-browser-snapshots/${dir}/${base}/${zip}`;
-          try {
-            await download(url, archive);
-            break;
-          }
-          catch (err) {
-            if (err.name == "HTTPError") {
-              base--;
-              archive = path.join(cacheDir, `${base}-${zip}`);
-              if (base <= startBase - Chromium.#MAX_VERSION_DECREMENTS)
-                throw new Error(`${errMsg.driverDownload}: Chromium base ${startBase}`);
-            }
-            else {
-              throw new Error(`${errMsg.driverDownload}: ${url}\n${err}`);
-            }
-          }
-        }
-      }
-      await extractZip(archive, {dir: cacheDir});
-    }
-
-    await killDriverProcess("chromedriver");
-    let driverPath = path.join(cacheDir, zip.split(".")[0], driverBinary);
-    if (platformArch == "linux-arm64")
-      driverPath = path.join(cacheDir, driverBinary);
-    await fs.promises.rm(driverPath, {force: true});
-    await extractZip(archive, {dir: cacheDir});
-
-    return driverPath;
+    return {binary, versionNumber};
   }
 
   /** @see Browser.getDriver */
@@ -259,21 +195,24 @@ export class Chromium extends Browser {
     headless = true, extensionPaths = [], incognito = false, insecure = false,
     extraArgs = [], customBrowserBinary
   } = {}, downloadTimeout = 0) {
-    let {binary, versionNumber, base} = customBrowserBinary ?
+    let {binary, versionNumber} = customBrowserBinary ?
       await Chromium.#getInstalledBrowserInfo(customBrowserBinary) :
       await Chromium.installBrowser(version, downloadTimeout);
-    let driverPath = await Chromium.#installDriver(base);
-    let serviceBuilder = new chrome.ServiceBuilder(driverPath);
+
     let options = new chrome.Options().addArguments("no-sandbox", ...extraArgs);
-    if (extensionPaths.length > 0)
+    if (extensionPaths.length > 0) {
+      await checkExtensionPaths(extensionPaths);
       options.addArguments(`load-extension=${extensionPaths.join(",")}`);
+    }
     if (headless) {
-      // New headless mode introduced in Chrome 111
-      // https://developer.chrome.com/articles/new-headless/
-      if (getMajorVersion(versionNumber) >= 111)
+      let majorVersion = getMajorVersion(versionNumber);
+      // https://www.selenium.dev/blog/2023/headless-is-going-away/
+      if (majorVersion >= 109)
         options.addArguments("headless=new");
+      else if (majorVersion >= 96)
+        options.addArguments("headless=chrome");
       else
-        options.headless();
+        options.addArguments("headless");
     }
     if (insecure)
       options.addArguments("ignore-certificate-errors");
@@ -284,7 +223,6 @@ export class Chromium extends Browser {
     let builder = new webdriver.Builder();
     builder.forBrowser("chrome");
     builder.setChromeOptions(options);
-    builder.setChromeService(serviceBuilder);
 
     return builder.build();
   }

package/src/edge.js

@@ -27,7 +27,7 @@ import extractZip from "extract-zip";
 
 import {Browser} from "./browser.js";
 import {download, killDriverProcess, wait, getMajorVersion, checkVersion,
-        checkPlatform, errMsg, snapshotsBaseDir, platformArch}
+        checkPlatform, errMsg, snapshotsBaseDir, checkExtensionPaths}
   from "./utils.js";
 
 let {By} = webdriver;
@@ -93,7 +93,7 @@ export class Edge extends Browser {
         return `${programFiles}\\Microsoft\\Edge\\Application\\msedge.exe`;
       case "linux":
         return channel == "stable" ?
-          "microsoft-edge" : `microsoft-edge-${channel}`;
+          "/usr/bin/microsoft-edge" : `/usr/bin/microsoft-edge-${channel}`;
       case "darwin":
         return `${process.env.HOME}/Applications/${Edge.#darwinApp}.app/Contents/MacOS/${Edge.#darwinApp}`;
       default:
@@ -179,7 +179,7 @@ export class Edge extends Browser {
     return installedVersion.trim().replace(/.*\s/, "");
   }
 
-  static async #installDriver(binary) {
+  static async #installOldWindowsDriver(binary) {
     async function extractEdgeZip(archive, cacheDir, driverPath) {
       await killDriverProcess("msedgedriver");
       await fs.promises.rm(driverPath, {force: true});
@@ -188,17 +188,12 @@ export class Edge extends Browser {
 
     let binaryPath = binary || Edge.#getBinaryPath();
     let versionNumber = await Edge.#getInstalledVersionNumber(binaryPath);
-    let [zip, driverBinary] = {
-      "win32-ia32": ["edgedriver_win32.zip", "msedgedriver.exe"],
-      "win32-x64": ["edgedriver_win64.zip", "msedgedriver.exe"],
-      "linux-x64": ["edgedriver_linux64.zip", "msedgedriver"],
-      "darwin-x64": ["edgedriver_mac64.zip", "msedgedriver"],
-      "darwin-arm64": ["edgedriver_mac64_m1.zip", "msedgedriver"]
-    }[platformArch];
     let cacheDir = path.join(snapshotsBaseDir, "edge", "cache",
                              `edgedriver-${versionNumber}`);
+    let zip = process.arch == "ia32" ?
+      "edgedriver_win32.zip" : "edgedriver_win64.zip";
     let archive = path.join(cacheDir, `${versionNumber}-${zip}`);
-    let driverPath = path.join(cacheDir, driverBinary);
+    let driverPath = path.join(cacheDir, "msedgedriver.exe");
 
     try {
       await fs.promises.access(archive);
@@ -243,29 +238,36 @@ export class Edge extends Browser {
         await Edge.#getInstalledVersionNumber(binary);
     }
 
-    let driverPath = await Edge.#installDriver(binary);
-    let serviceBuilder = new edge.ServiceBuilder(driverPath);
-
     let options = new edge.Options().addArguments("no-sandbox", ...extraArgs);
     if (headless) {
       if (versionNumber && getMajorVersion(versionNumber) >= 114)
         options.addArguments("headless=new");
       else
-        options.headless();
+        options.addArguments("headless");
     }
-    if (extensionPaths.length > 0)
+    if (extensionPaths.length > 0) {
+      await checkExtensionPaths(extensionPaths);
       options.addArguments(`load-extension=${extensionPaths.join(",")}`);
+    }
     if (incognito)
       options.addArguments("inprivate");
     if (insecure)
       options.addArguments("ignore-certificate-errors");
     if (platform == "linux")
-      options.setEdgeChromiumBinaryPath(`/usr/bin/${binary}`);
+      options.setEdgeChromiumBinaryPath(binary);
 
     let builder = new webdriver.Builder();
     builder.forBrowser("MicrosoftEdge");
     builder.setEdgeOptions(options);
-    builder.setEdgeService(serviceBuilder);
+
+    if (getMajorVersion(versionNumber) < 95) {
+      // Selenium's automated driver download doesn't work on old Edge versions.
+      // Support to installing old drivers is only needed by Windows CI jobs.
+      // https://gitlab.com/eyeo/developer-experience/get-browser-binary/-/issues/2#note_1189235804
+      let driverPath = await Edge.#installOldWindowsDriver(binary);
+      let serviceBuilder = new edge.ServiceBuilder(driverPath);
+      builder.setEdgeService(serviceBuilder);
+    }
 
     let driver;
     // On Windows CI, occasionally a SessionNotCreatedError is thrown, likely

package/src/firefox.js

@@ -23,11 +23,12 @@ import fs from "fs";
 import got from "got";
 import webdriver from "selenium-webdriver";
 import firefox from "selenium-webdriver/firefox.js";
+import {Command} from "selenium-webdriver/lib/command.js";
 
 import {Browser} from "./browser.js";
 import {download, extractTar, extractDmg, killDriverProcess, wait,
         getMajorVersion, checkVersion, checkPlatform, errMsg, snapshotsBaseDir,
-        platformArch} from "./utils.js";
+        platformArch, checkExtensionPaths} from "./utils.js";
 
 let {until, By} = webdriver;
 
@@ -98,10 +99,6 @@ export class Firefox extends Browser {
   static async installBrowser(version = "latest", downloadTimeout = 0) {
     const MIN_VERSION = 68;
 
-    // https://gitlab.com/eyeo/developer-experience/get-browser-binary/-/issues/46
-    if (platformArch == "linux-arm64")
-      return await Firefox.#getInstalledBrowserInfo("/usr/bin/firefox");
-
     checkVersion(version, MIN_VERSION, Firefox.#CHANNELS);
     let versionNumber = await Firefox.#getVersionForChannel(version);
 
@@ -156,7 +153,7 @@ export class Firefox extends Browser {
 
     let options = new firefox.Options();
     if (headless)
-      options.headless();
+      options.addArguments("--headless");
     if (incognito)
       options.addArguments("--private");
     if (insecure)
@@ -188,8 +185,18 @@ export class Firefox extends Browser {
     }, 30000, `${errMsg.driverStart}: geckodriver`, 1000);
 
     for (let extensionPath of extensionPaths) {
-      let temporary = true; // Parameter not documented on the webdriver docs
-      await driver.installAddon(extensionPath, temporary);
+      await checkExtensionPaths([extensionPath]);
+
+      // This is based on Selenium's Firefox `installAddon` function. Rather
+      // than setting the "addon" parameter, which is the actual extension
+      // base64 encoded, we need to set the "path" which is the filepath to the
+      // extension. This allows downstream to test upgrade paths by updating the
+      // extension source code and reloading it.
+      // See https://github.com/SeleniumHQ/selenium/blob/trunk/javascript/node/selenium-webdriver/firefox.js
+      await driver.execute(
+        new Command("install addon")
+          .setParameter("path", extensionPath)
+          .setParameter("temporary", true));
     }
     return driver;
   }

package/src/utils.js

@@ -34,7 +34,8 @@ export const errMsg = {
   browserDownload: "Browser download failed",
   browserNotInstalled: "Browser is not installed",
   browserVersionCheck: "Checking the browser version failed",
-  elemNotFound: "HTML element not found"
+  elemNotFound: "HTML element not found",
+  manifestNotFound: "Extension manifest file not found"
 };
 export const platformArch = `${process.platform}-${process.arch}`;
 
@@ -238,6 +239,12 @@ export async function takeFullPageScreenshot(driver, hideScrollbars = true) {
   return fullScreenshot;
 }
 
+/**
+ * Returns the major version of a browser version number.
+ * @param {string} versionNumber Full browser version number.
+ * @return {number} Major version number.
+ * @throws {Error} Unsupported browser version.
+ */
 export function getMajorVersion(versionNumber) {
   let majorVersion = parseInt(versionNumber && versionNumber.split(".")[0], 10);
   if (isNaN(majorVersion))
@@ -258,3 +265,14 @@ export function checkPlatform() {
   if (!["win32", "linux", "darwin"].includes(process.platform))
     throw new Error(`${errMsg.unsupportedPlatform}: ${process.platform}`);
 }
+
+export async function checkExtensionPaths(extensionPaths) {
+  for (let extensionPath of extensionPaths) {
+    try {
+      await fs.promises.access(path.join(extensionPath, "manifest.json"));
+    }
+    catch (err) {
+      throw new Error(`${errMsg.manifestNotFound}: ${extensionPath}`);
+    }
+  }
+}

package/test/browsers.js

@@ -25,8 +25,6 @@ import {BROWSERS, snapshotsBaseDir, takeFullPageScreenshot} from "../index.js";
 import {killDriverProcess} from "../src/utils.js";
 import {TEST_SERVER_URL} from "./test-server.js";
 
-import "geckodriver"; // Required to set the driver path on Windows
-
 const VERSIONS = {
   chromium: ["latest", "77.0.3865.0", "beta", "dev"],
   firefox: ["latest", "68.0", "beta"],
@@ -106,42 +104,21 @@ function getExtension(browser, version) {
   return {extensionPaths, manifest};
 }
 
-async function getCachedTimes(browser) {
-  let armLinux = process.platform == "linux" && process.arch == "arm64";
-  let cacheDir = path.join(snapshotsBaseDir, browser, "cache");
-  let cacheFiles;
-  try {
-    cacheFiles = await fs.promises.readdir(cacheDir);
-  }
-  catch (err) {
-    if (!armLinux)
-      throw err;
-  }
-
-  let browserCacheTime = null;
+async function getInstallFileCTime(browser) {
   // Browsers installed at OS level are not tested
-  if (browser != "edge" && !armLinux) {
-    let installTypes = [".zip", ".dmg", ".bz2"];
-    let browserZip =
-      cacheFiles.find(elem => installTypes.some(type => elem.includes(type)));
-    let browserStat = await fs.promises.stat(path.join(cacheDir, browserZip));
-    browserCacheTime = browserStat.ctimeMs;
-  }
+  if (browser == "edge" && process.platform == "win32")
+    return null;
 
-  let driverCacheTime = null;
-  // geckodriver is installed by npm, that's why Firefox is not tested here
-  // for chromiumdriver on arm linux, see https://gitlab.com/eyeo/developer-experience/get-browser-binary/-/issues/46
-  if (browser != "firefox" && !armLinux) {
-    let driverDir = cacheFiles.find(
-      elem => !elem.endsWith(".zip") && !elem.endsWith(".pkg"));
-    let driverFiles = await fs.promises.readdir(path.join(cacheDir, driverDir));
-    let driverZip = driverFiles.find(elem => elem.endsWith(".zip"));
-    let driverStat =
-      await fs.promises.stat(path.join(cacheDir, driverDir, driverZip));
-    driverCacheTime = driverStat.ctimeMs;
-  }
+  const installTypes = [".zip", ".dmg", ".bz2", ".deb", ".exe"];
+
+  let cacheDir = path.join(snapshotsBaseDir, browser, "cache");
+  let cacheFiles = await fs.promises.readdir(cacheDir);
+  let browserZip =
+    cacheFiles.find(elem => installTypes.some(type => elem.includes(type)));
+  if (!browserZip)
+    throw new Error(`Files in ${cacheDir} don't belong to any known install file types: ${installTypes}`);
 
-  return {browserCTime: browserCacheTime, driverCTime: driverCacheTime};
+  return (await fs.promises.stat(path.join(cacheDir, browserZip))).ctimeMs;
 }
 
 const browserNames = {
@@ -159,6 +136,11 @@ async function basicUrlTest(driver, browser) {
   expect(text).toEqual("Test server basic page");
 }
 
+// Adding the browser version to the test title for logging purposes
+function addVersionToTitle(ctx, version) {
+  ctx.test.title = `${ctx.test.title} [v${version}]`;
+}
+
 for (let browser of Object.keys(BROWSERS)) {
   describe(`Browser: ${browser}`, () => {
     before(async() => {
@@ -171,7 +153,7 @@ for (let browser of Object.keys(BROWSERS)) {
     for (let version of VERSIONS[browser]) {
       describe(`Version: ${version}`, () => {
         let driver = null;
-        let emptyCacheTimes = null;
+        let firstInstallFileCTime = null;
         let customBrowserBinary = null;
         let failedInstall = false;
 
@@ -217,31 +199,32 @@ for (let browser of Object.keys(BROWSERS)) {
           expect(installedVersion).toEqual(
             expect.stringContaining(normalize(versionNumber)));
 
-          // Adding the version number to the test title for logging purposes
-          this.test.title = `${this.test.title} [v${versionNumber}]`;
+          addVersionToTitle(this, versionNumber);
+
+          // Data used in further tests
           customBrowserBinary = binary;
+          firstInstallFileCTime = await getInstallFileCTime(browser);
         });
 
-        it("runs", async() => {
+        it("runs", async function() {
           driver = await BROWSERS[browser].getDriver(version);
           await basicUrlTest(driver, browser);
 
-          // When running all tests, this saves time on the cache test
-          emptyCacheTimes = await getCachedTimes(browser);
+          let browserVersion =
+            (await driver.getCapabilities()).getBrowserVersion();
+          addVersionToTitle(this, browserVersion);
         });
 
-        it("uses cached install files", async() => {
-          if (emptyCacheTimes == null) { // Single test case run
-            driver = await BROWSERS[browser].getDriver(version);
-            emptyCacheTimes = await getCachedTimes(browser);
-            await quitDriver();
-          }
+        // This test depends on running the "installs" test
+        it("uses cached install files", async function() {
+          if (!firstInstallFileCTime)
+            this.skip();
 
           // assigning `driver` to allow the afterEach hook quit the driver
           driver = await BROWSERS[browser].getDriver(version);
-          let existingCacheTimes = await getCachedTimes(browser);
+          let secondInstallFileCTime = await getInstallFileCTime(browser);
 
-          expect(existingCacheTimes).toEqual(emptyCacheTimes);
+          expect(secondInstallFileCTime).toEqual(firstInstallFileCTime);
         });
 
         // This test depends on running the "installs" test
@@ -288,6 +271,11 @@ for (let browser of Object.keys(BROWSERS)) {
           expect(fullImg.bitmap.height).toBeGreaterThan(partImg.bitmap.height);
         });
 
+        // The only reason the geckodriver package is required in package.json
+        // is to install a specific version needed by Firefox 68.0, otherwise it
+        // fails to load extensions. When the oldest Firefox version is bumped,
+        // Selenium's automated driver download should be able to manage it.
+        // https://gitlab.com/eyeo/developer-experience/get-browser-binary/-/issues/44
         it("loads an extension", async() => {
           // Chromium's old headless mode doesn't support loading extensions
           let headless = browser == "firefox" || (browser == "chromium" &&
@@ -324,10 +312,11 @@ for (let browser of Object.keys(BROWSERS)) {
       }
     });
 
-    it("does not run not installed custom browsers", async() => {
-      let customBrowserBinary = "not-installed";
-      await expect(BROWSERS[browser].getDriver("latest", {customBrowserBinary}))
-        .rejects.toThrow(/(Browser is not installed|binary is not a Firefox executable)/);
+    it("does not load an invalid extension", async() => {
+      let extensionPaths = [process.cwd()];
+
+      await expect(BROWSERS[browser].getDriver("latest", {extensionPaths}))
+        .rejects.toThrow(`Extension manifest file not found: ${extensionPaths[0]}`);
     });
   });
 }

package/test/runner.js

@@ -53,10 +53,12 @@ export async function killTestServer() {
 }
 
 async function run() {
-  let args = process.argv.join(" ").split("-- ")[1];
+  let args = process.argv.slice(2); // Keep npm args from "--" onwards
+  args = args.map(v => v.startsWith("-") ? v : `"${v}"`);
+
   await runTestServer();
   try {
-    execSync(`npm run test-suite -- ${args}`, {stdio: "inherit"});
+    execSync(`npm run test-suite ${args.join(" ")}`, {stdio: "inherit"});
   }
   finally {
     await killTestServer();

package/test/docker/arm64.Dockerfile

@@ -1,27 +0,0 @@
-# Duplicated from registry.gitlab.com/eyeo/docker/get-browser-binary:node18
-# https://gitlab.com/eyeo/developer-experience/get-browser-binary/-/issues/46
-FROM node:18-bullseye-slim
-# General packages
-RUN apt-get update && apt-get install -y git procps wget unzip bzip2 gnupg
-# xvfb (headful browser run)
-RUN apt-get install -y libgtk-3-0 libxt6 xvfb libnss3 libxss1
-# General browser dependencies
-RUN apt-get install -y libgconf-2-4 libasound2 libgbm1
-# Edge dependencies
-RUN apt-get install -y fonts-liberation libatomic1 xdg-utils libu2f-udev
-
-# Chromium ARM
-RUN apt-get install -y chromium
-# https://askubuntu.com/questions/1360864/google-chrome-not-launching
-RUN mkdir -p "/root/.config/chromium/Crash Reports/pending/"
-
-# Firefox ARM
-RUN apt-get install -y firefox-esr
-
-COPY package*.json get-browser-binary/
-RUN cd get-browser-binary && npm install
-
-COPY . get-browser-binary/
-
-ENV TEST_ARGS="--grep Browser"
-ENTRYPOINT get-browser-binary/test/docker/entrypoint.sh