@eyeo/get-browser-binary 0.15.0 → 0.17.0

package/.gitlab-ci.yml

@@ -27,42 +27,50 @@ test:basic:
     - npm run lint
     - npm test -- --grep "Utils"
 
-test:browsers:linux:
+.linux:
   stage: test
   image: docker:24.0.5
   services:
     - docker:24.0.5-dind
   before_script:
     - docker build -f test/docker/Dockerfile -t browsers .
+
+test:browsers:linux:
+  extends: .linux
   script:
-    - docker run --shm-size=512m -t browsers
+    - docker run --shm-size=512m -t -e TEST_ARGS="--grep ^.*Browser((?!Version:.dev).)*\$" browsers
 
-test:browsers:windows:
+test:browsers:linux:dev:
+  extends: .linux
+  script:
+    - docker run --shm-size=512m -t -e TEST_ARGS="--grep Version:.dev" browsers
+  allow_failure: true
+
+.windows:
   stage: test
-  variables:
-    CI_PROJECT_ID_MAINSTREAM: 36688302
   before_script:
-    - Invoke-WebRequest
-        -Uri "${Env:CI_API_V4_URL}/projects/${Env:CI_PROJECT_ID_MAINSTREAM}/packages/generic/microsoft-edge/79.0.309/MicrosoftEdgeEnterpriseX64.msi"
-        -Headers @{'JOB-TOKEN' = $Env:CI_JOB_TOKEN}
-        -OutFile 'MicrosoftEdgeEnterpriseX64.msi'
-    - Start-Process msiexec
-        -ArgumentList "/i MicrosoftEdgeEnterpriseX64.msi /norestart /qn" -Wait
-    - choco upgrade -y --no-progress nodejs --version 18.17.1
+    - choco install -y microsoft-edge
     - npm install
+  tags:
+    - saas-windows-medium-amd64
+  cache: {}
+
+test:browsers:windows:
+  extends: .windows
   script:
     # Running Edge tests only on the preinstalled version
     # https://gitlab.com/eyeo/developer-experience/get-browser-binary/-/issues/29
     - npm test -- --grep "edge.*latest"
-    - npm test -- --grep "chromium"
+    - npm test -- --grep "^.*chromium((?!Version:.dev).)*$"
     # Running only a subset of Firefox tests to avoid low OS resources error
     # https://gitlab.com/eyeo/developer-experience/get-browser-binary/-/issues/2
     - npm test -- --grep "firefox.*installs"
-  tags:
-    - shared-windows
-    - windows
-    - windows-1809
-  cache: {}
+
+test:browsers:windows:dev:
+  extends: .windows
+  script:
+    - npm test -- --grep "chromium.*Version:.dev"
+  allow_failure: true
 
 docs:
   stage: docs

package/README.md

@@ -32,7 +32,7 @@ the right side.
 
 - Chromium >= 77 (Chromium ARM >= 92)
 - Firefox >= 68
-- Edge >= 95
+- Edge >= 95 (Windows Edge >= 79)
 
 Note: Installing Edge is not supported on Windows. It is assumed to be installed
 because it is the default browser on that platform. On macOS, only the latest
@@ -43,6 +43,22 @@ Edge version is supported.
 Set the `VERBOSE` environment variable to `"true"` to get verbose logging on
 download requests.
 
+### Offline execution
+
+It is possible to run browsers offline as long as they have been previously
+installed. Example:
+
+```javascript
+// Online
+let {binary} = await BROWSERS[browser].installBrowser(version);
+console.log(binary); // keep the browser binary location to use it offline
+let driver = await BROWSERS[browser].getDriver(version); // let the driver binary download
+
+// Offline
+let customBrowserBinary = "<browser binary location>";
+let driver = await BROWSERS[browser].getDriver(version, {customBrowserBinary});
+```
+
 ## Development
 
 ### Prerequisites

package/RELEASE_NOTES.md

@@ -1,3 +1,39 @@
+# Unreleased 
+
+# 0.17.0
+
+- Stops calling `driver.close()` in `enableExtensionInIncognito()`
+on non-Windows platforms (!114)
+- Add instructions for offline execution (!110)
+- Adds option to setup proxy through PAC file for Firefox (!115)
+
+# 0.16.0
+
+- Starts using selenium's automated driver management. That means additional
+`chromedriver` and `msedgedriver` packages are no longer needed (#67)
+- Loads the extension in Firefox by local path instead of being copied to a
+temporary folder (!104)
+- Checks if extensions can be loaded beforehand, by ensuring they contain a
+`manifest.json` file. If not, a new `Extension manifest file not found` error
+will be thrown (#71)
+- Refactors headless runs by not using webdriver's deprecated headless() method
+(#72)
+
+### Testing
+
+- Fixes an issue with `npm test` which would fail when the `grep` option would
+include parenthesis (#69)
+- Adds the browser version information in the "runs" test (#74)
+
+### Notes for integrators
+
+- Please make sure that your selenium-webdriver package version is at least
+4.15.0 (the one used in this release). Also make sure that you are not using the
+`chromedriver` nor `msedgedriver` packages, otherwise the automated driver
+management may not work as expected (#67)
+- If you experience chromedriver issues, try deleting the `/browser-snapshots`
+cache folder.
+
 # 0.15.0
 
 - Fixes the downloads of Chromium versions < 91 by replacing the remaining

package/index.js

@@ -19,7 +19,7 @@ import {Chromium} from "./src/chromium.js";
 import {Firefox} from "./src/firefox.js";
 import {Edge} from "./src/edge.js";
 
-export {download, takeFullPageScreenshot, snapshotsBaseDir}
+export {download, takeFullPageScreenshot, snapshotsBaseDir, getMajorVersion}
   from "./src/utils.js";
 
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@eyeo/get-browser-binary",
-  "version": "0.15.0",
+  "version": "0.17.0",
   "description": "Install browser binaries and matching webdrivers",
   "repository": {
     "type": "git",

package/src/browser.js

@@ -84,17 +84,19 @@ export class Browser {
   /**
    * @typedef {Object} driverOptions
    * @property {boolean} [headless=true] Run the browser in headless mode,
-   *   or not. In Chromium >= 111, the
+   *   or not. In Chromium >= 111 and Edge >= 114 the
    *   {@link https://developer.chrome.com/articles/new-headless/ new headless mode}
    *   is used.
-   * @property {Array.<string>} [extensionPaths=[]] Loads extensions to the
-   *   browser.
+   * @property {Array.<string>} [extensionPaths=[]] Paths to folders containing
+   *   unpacked extensions which will be loaded to the browser.
    * @property {boolean} [incognito=false] Runs the browser in incognito mode,
    *   or not.
    * @property {boolean} [insecure=false] Forces the browser to accept insecure
    *   certificates, or not.
    * @property {Array.<string>} [extraArgs=[]] Additional arguments to start
    *   the browser with.
+   * @property {string} [proxy] Only for Firefox, path to network
+   *   proxy autoconfig url (PAC file).
    * @property {string} [customBrowserBinary] Path to the browser binary to be
    *   used, instead of the browser installed by installBrowser(). This option
    *   overrides the version parameter in getDriver().
@@ -110,7 +112,8 @@ export class Browser {
    *   browser install files to complete. When set to 0 there is no time limit.
    * @return {webdriver}
    * @throws {Error} Unsupported browser version, Unsupported platform, Browser
-   *   download failed, Driver download failed, Unable to start driver.
+   *   download failed, Driver download failed, Unable to start driver, Manifest
+   *   file not found.
    */
   static async getDriver(version, options = {}, downloadTimeout = 0) {
     // to be implemented by the subclass

package/src/chromium.js

@@ -24,9 +24,8 @@ import chrome from "selenium-webdriver/chrome.js";
 import extractZip from "extract-zip";
 
 import {Browser} from "./browser.js";
-import {download, killDriverProcess, getMajorVersion, checkVersion,
-        checkPlatform, errMsg, snapshotsBaseDir, platformArch}
-  from "./utils.js";
+import {download, getMajorVersion, checkVersion, checkPlatform, errMsg,
+        snapshotsBaseDir, platformArch, checkExtensionPaths} from "./utils.js";
 
 /**
  * Browser and webdriver functionality for Chromium.
@@ -116,8 +115,7 @@ export class Chromium extends Browser {
     // Linux example: "Chromium 112.0.5615.49 built on Debian 11.6"
     // Windows example: "114.0.5735.0"
     let versionNumber = installedVersion.split(" ")[1] || installedVersion;
-    let base = await Chromium.#getBase(versionNumber);
-    return {binary, versionNumber, base};
+    return {binary, versionNumber};
   }
 
   /**
@@ -134,14 +132,10 @@ export class Chromium extends Browser {
   static async installBrowser(version = "latest", downloadTimeout = 0) {
     const MIN_VERSION = process.arch == "arm64" ? 92 : 77;
 
-    let binary;
-    let versionNumber;
-    let base;
-
     checkVersion(version, MIN_VERSION, Chromium.#CHANNELS);
-    versionNumber = await Chromium.#getVersionForChannel(version);
+    let versionNumber = await Chromium.#getVersionForChannel(version);
 
-    base = await Chromium.#getBase(versionNumber);
+    let base = await Chromium.#getBase(versionNumber);
     let startBase = base;
     let [platformDir, fileName] = {
       "win32-ia32": ["Win", "chrome-win.zip"],
@@ -154,6 +148,7 @@ export class Chromium extends Browser {
     let browserDir;
     let snapshotsDir = path.join(snapshotsBaseDir, "chromium");
 
+    let binary;
     while (true) {
       browserDir = path.join(snapshotsDir, `chromium-${platformArch}-${base}`);
       binary = Chromium.#getBinaryPath(browserDir);
@@ -192,54 +187,7 @@ export class Chromium extends Browser {
     }
     await extractZip(archive, {dir: browserDir});
 
-    return {binary, versionNumber, base};
-  }
-
-  static async #installDriver(startBase) {
-    let [dir, zip, driverBinary] = {
-      "win32-ia32": ["Win", "chromedriver_win32.zip", "chromedriver.exe"],
-      "win32-x64": ["Win_x64", "chromedriver_win32.zip", "chromedriver.exe"],
-      "linux-x64": ["Linux_x64", "chromedriver_linux64.zip", "chromedriver"],
-      "darwin-x64": ["Mac", "chromedriver_mac64.zip", "chromedriver"],
-      "darwin-arm64": ["Mac_Arm", "chromedriver_mac64.zip", "chromedriver"]
-    }[platformArch];
-
-    let cacheDir = path.join(snapshotsBaseDir, "chromium", "cache",
-                             "chromedriver");
-    let archive = path.join(cacheDir, `${startBase}-${zip}`);
-    try {
-      await fs.promises.access(archive);
-      await extractZip(archive, {dir: cacheDir});
-    }
-    catch (e) { // zip file is either not cached or corrupted
-      let base = startBase;
-      while (true) {
-        let url = `https://commondatastorage.googleapis.com/chromium-browser-snapshots/${dir}/${base}/${zip}`;
-        try {
-          await download(url, archive);
-          break;
-        }
-        catch (err) {
-          if (err.name == "HTTPError") {
-            base--;
-            archive = path.join(cacheDir, `${base}-${zip}`);
-            if (base <= startBase - Chromium.#MAX_VERSION_DECREMENTS)
-              throw new Error(`${errMsg.driverDownload}: Chromium base ${startBase}`);
-          }
-          else {
-            throw new Error(`${errMsg.driverDownload}: ${url}\n${err}`);
-          }
-        }
-      }
-      await extractZip(archive, {dir: cacheDir});
-    }
-
-    await killDriverProcess("chromedriver");
-    let driverPath = path.join(cacheDir, zip.split(".")[0], driverBinary);
-    await fs.promises.rm(driverPath, {force: true});
-    await extractZip(archive, {dir: cacheDir});
-
-    return driverPath;
+    return {binary, versionNumber};
   }
 
   /** @see Browser.getDriver */
@@ -247,21 +195,24 @@ export class Chromium extends Browser {
     headless = true, extensionPaths = [], incognito = false, insecure = false,
     extraArgs = [], customBrowserBinary
   } = {}, downloadTimeout = 0) {
-    let {binary, versionNumber, base} = customBrowserBinary ?
+    let {binary, versionNumber} = customBrowserBinary ?
       await Chromium.#getInstalledBrowserInfo(customBrowserBinary) :
       await Chromium.installBrowser(version, downloadTimeout);
-    let driverPath = await Chromium.#installDriver(base);
-    let serviceBuilder = new chrome.ServiceBuilder(driverPath);
+
     let options = new chrome.Options().addArguments("no-sandbox", ...extraArgs);
-    if (extensionPaths.length > 0)
+    if (extensionPaths.length > 0) {
+      await checkExtensionPaths(extensionPaths);
       options.addArguments(`load-extension=${extensionPaths.join(",")}`);
+    }
     if (headless) {
-      // New headless mode introduced in Chrome 111
-      // https://developer.chrome.com/articles/new-headless/
-      if (getMajorVersion(versionNumber) >= 111)
+      let majorVersion = getMajorVersion(versionNumber);
+      // https://www.selenium.dev/blog/2023/headless-is-going-away/
+      if (majorVersion >= 109)
         options.addArguments("headless=new");
+      else if (majorVersion >= 96)
+        options.addArguments("headless=chrome");
       else
-        options.headless();
+        options.addArguments("headless");
     }
     if (insecure)
       options.addArguments("ignore-certificate-errors");
@@ -272,7 +223,6 @@ export class Chromium extends Browser {
     let builder = new webdriver.Builder();
     builder.forBrowser("chrome");
     builder.setChromeOptions(options);
-    builder.setChromeService(serviceBuilder);
 
     return builder.build();
   }
@@ -315,8 +265,9 @@ export class Chromium extends Browser {
         setTimeout(() => resolve(enable()), 100);
       });
     }, extensionTitle, errMsg.extensionNotFound);
-    if (version >= 115)
-      // Closing the previously opened new window
+    if (version >= 115 && process.platform == "win32")
+      // Closing the previously opened new window. Needed on Windows to avoid a
+      // further `WebDriverError: disconnected` error
       await driver.close();
 
     await driver.switchTo().window(handle);

package/src/edge.js

@@ -27,7 +27,7 @@ import extractZip from "extract-zip";
 
 import {Browser} from "./browser.js";
 import {download, killDriverProcess, wait, getMajorVersion, checkVersion,
-        checkPlatform, errMsg, snapshotsBaseDir, platformArch}
+        checkPlatform, errMsg, snapshotsBaseDir, checkExtensionPaths}
   from "./utils.js";
 
 let {By} = webdriver;
@@ -93,7 +93,7 @@ export class Edge extends Browser {
         return `${programFiles}\\Microsoft\\Edge\\Application\\msedge.exe`;
       case "linux":
         return channel == "stable" ?
-          "microsoft-edge" : `microsoft-edge-${channel}`;
+          "/usr/bin/microsoft-edge" : `/usr/bin/microsoft-edge-${channel}`;
       case "darwin":
         return `${process.env.HOME}/Applications/${Edge.#darwinApp}.app/Contents/MacOS/${Edge.#darwinApp}`;
       default:
@@ -179,7 +179,7 @@ export class Edge extends Browser {
     return installedVersion.trim().replace(/.*\s/, "");
   }
 
-  static async #installDriver(binary) {
+  static async #installOldWindowsDriver(binary) {
     async function extractEdgeZip(archive, cacheDir, driverPath) {
       await killDriverProcess("msedgedriver");
       await fs.promises.rm(driverPath, {force: true});
@@ -188,17 +188,12 @@ export class Edge extends Browser {
 
     let binaryPath = binary || Edge.#getBinaryPath();
     let versionNumber = await Edge.#getInstalledVersionNumber(binaryPath);
-    let [zip, driverBinary] = {
-      "win32-ia32": ["edgedriver_win32.zip", "msedgedriver.exe"],
-      "win32-x64": ["edgedriver_win64.zip", "msedgedriver.exe"],
-      "linux-x64": ["edgedriver_linux64.zip", "msedgedriver"],
-      "darwin-x64": ["edgedriver_mac64.zip", "msedgedriver"],
-      "darwin-arm64": ["edgedriver_mac64_m1.zip", "msedgedriver"]
-    }[platformArch];
     let cacheDir = path.join(snapshotsBaseDir, "edge", "cache",
                              `edgedriver-${versionNumber}`);
+    let zip = process.arch == "ia32" ?
+      "edgedriver_win32.zip" : "edgedriver_win64.zip";
     let archive = path.join(cacheDir, `${versionNumber}-${zip}`);
-    let driverPath = path.join(cacheDir, driverBinary);
+    let driverPath = path.join(cacheDir, "msedgedriver.exe");
 
     try {
       await fs.promises.access(archive);
@@ -243,29 +238,36 @@ export class Edge extends Browser {
         await Edge.#getInstalledVersionNumber(binary);
     }
 
-    let driverPath = await Edge.#installDriver(binary);
-    let serviceBuilder = new edge.ServiceBuilder(driverPath);
-
     let options = new edge.Options().addArguments("no-sandbox", ...extraArgs);
     if (headless) {
       if (versionNumber && getMajorVersion(versionNumber) >= 114)
         options.addArguments("headless=new");
       else
-        options.headless();
+        options.addArguments("headless");
     }
-    if (extensionPaths.length > 0)
+    if (extensionPaths.length > 0) {
+      await checkExtensionPaths(extensionPaths);
       options.addArguments(`load-extension=${extensionPaths.join(",")}`);
+    }
     if (incognito)
       options.addArguments("inprivate");
     if (insecure)
       options.addArguments("ignore-certificate-errors");
     if (platform == "linux")
-      options.setEdgeChromiumBinaryPath(`/usr/bin/${binary}`);
+      options.setEdgeChromiumBinaryPath(binary);
 
     let builder = new webdriver.Builder();
     builder.forBrowser("MicrosoftEdge");
     builder.setEdgeOptions(options);
-    builder.setEdgeService(serviceBuilder);
+
+    if (getMajorVersion(versionNumber) < 95) {
+      // Selenium's automated driver download doesn't work on old Edge versions.
+      // Support to installing old drivers is only needed by Windows CI jobs.
+      // https://gitlab.com/eyeo/developer-experience/get-browser-binary/-/issues/2#note_1189235804
+      let driverPath = await Edge.#installOldWindowsDriver(binary);
+      let serviceBuilder = new edge.ServiceBuilder(driverPath);
+      builder.setEdgeService(serviceBuilder);
+    }
 
     let driver;
     // On Windows CI, occasionally a SessionNotCreatedError is thrown, likely

package/src/firefox.js

@@ -23,11 +23,12 @@ import fs from "fs";
 import got from "got";
 import webdriver from "selenium-webdriver";
 import firefox from "selenium-webdriver/firefox.js";
+import {Command} from "selenium-webdriver/lib/command.js";
 
 import {Browser} from "./browser.js";
 import {download, extractTar, extractDmg, killDriverProcess, wait,
         getMajorVersion, checkVersion, checkPlatform, errMsg, snapshotsBaseDir,
-        platformArch} from "./utils.js";
+        platformArch, checkExtensionPaths} from "./utils.js";
 
 let {until, By} = webdriver;
 
@@ -139,9 +140,14 @@ export class Firefox extends Browser {
   }
 
   /** @see Browser.getDriver */
+  /** In Firefox you can define url to proxy autoconfig file.
+  * Proxy Autoconfig file determine whether web requests should go through
+  * a proxy server or connect directly. This allow to use localhost mapped
+  * to any url (f.ex testpages.adblockplus.org)
+  */
   static async getDriver(version = "latest", {
     headless = true, extensionPaths = [], incognito = false, insecure = false,
-    extraArgs = [], customBrowserBinary
+    extraArgs = [], customBrowserBinary, proxy
   } = {}, downloadTimeout = 0) {
     let binary;
     let versionNumber;
@@ -152,7 +158,7 @@ export class Firefox extends Browser {
 
     let options = new firefox.Options();
     if (headless)
-      options.headless();
+      options.addArguments("--headless");
     if (incognito)
       options.addArguments("--private");
     if (insecure)
@@ -162,6 +168,10 @@ export class Firefox extends Browser {
     // Enabled by default on Firefox > 68
     if (versionNumber && getMajorVersion(versionNumber) == 68)
       options.setPreference("dom.promise_rejection_events.enabled", true);
+    if (proxy) {
+      options.setPreference("network.proxy.type", 2);
+      options.setPreference("network.proxy.autoconfig_url", proxy);
+    }
 
     options.setBinary(customBrowserBinary || binary);
 
@@ -184,8 +194,18 @@ export class Firefox extends Browser {
     }, 30000, `${errMsg.driverStart}: geckodriver`, 1000);
 
     for (let extensionPath of extensionPaths) {
-      let temporary = true; // Parameter not documented on the webdriver docs
-      await driver.installAddon(extensionPath, temporary);
+      await checkExtensionPaths([extensionPath]);
+
+      // This is based on Selenium's Firefox `installAddon` function. Rather
+      // than setting the "addon" parameter, which is the actual extension
+      // base64 encoded, we need to set the "path" which is the filepath to the
+      // extension. This allows downstream to test upgrade paths by updating the
+      // extension source code and reloading it.
+      // See https://github.com/SeleniumHQ/selenium/blob/trunk/javascript/node/selenium-webdriver/firefox.js
+      await driver.execute(
+        new Command("install addon")
+          .setParameter("path", extensionPath)
+          .setParameter("temporary", true));
     }
     return driver;
   }

package/src/utils.js

@@ -34,7 +34,8 @@ export const errMsg = {
   browserDownload: "Browser download failed",
   browserNotInstalled: "Browser is not installed",
   browserVersionCheck: "Checking the browser version failed",
-  elemNotFound: "HTML element not found"
+  elemNotFound: "HTML element not found",
+  manifestNotFound: "Extension manifest file not found"
 };
 export const platformArch = `${process.platform}-${process.arch}`;
 
@@ -238,6 +239,12 @@ export async function takeFullPageScreenshot(driver, hideScrollbars = true) {
   return fullScreenshot;
 }
 
+/**
+ * Returns the major version of a browser version number.
+ * @param {string} versionNumber Full browser version number.
+ * @return {number} Major version number.
+ * @throws {Error} Unsupported browser version.
+ */
 export function getMajorVersion(versionNumber) {
   let majorVersion = parseInt(versionNumber && versionNumber.split(".")[0], 10);
   if (isNaN(majorVersion))
@@ -258,3 +265,14 @@ export function checkPlatform() {
   if (!["win32", "linux", "darwin"].includes(process.platform))
     throw new Error(`${errMsg.unsupportedPlatform}: ${process.platform}`);
 }
+
+export async function checkExtensionPaths(extensionPaths) {
+  for (let extensionPath of extensionPaths) {
+    try {
+      await fs.promises.access(path.join(extensionPath, "manifest.json"));
+    }
+    catch (err) {
+      throw new Error(`${errMsg.manifestNotFound}: ${extensionPath}`);
+    }
+  }
+}

package/test/browsers.js

@@ -25,14 +25,13 @@ import {BROWSERS, snapshotsBaseDir, takeFullPageScreenshot} from "../index.js";
 import {killDriverProcess} from "../src/utils.js";
 import {TEST_SERVER_URL} from "./test-server.js";
 
-import "geckodriver"; // Required to set the driver path on Windows
-
 const VERSIONS = {
   chromium: ["latest", "77.0.3865.0", "beta", "dev"],
   firefox: ["latest", "68.0", "beta"],
   edge: ["latest", "95.0.1020.40", "beta", "dev"]
 };
 const TEST_URL_BASIC = `${TEST_SERVER_URL}/basic.html`;
+const PROXY_URL_BASIC = "http://testpages.adblockplus.org/basic.html";
 const TEST_URL_LONG = `${TEST_SERVER_URL}/long.html`;
 
 async function switchToHandle(driver, testFn) {
@@ -106,33 +105,22 @@ function getExtension(browser, version) {
   return {extensionPaths, manifest};
 }
 
-async function getCachedTimes(browser) {
-  let cacheDir = path.join(snapshotsBaseDir, browser, "cache");
-  let cacheFiles = await fs.promises.readdir(cacheDir);
-
-  let browserCacheTime = null;
+async function getInstallFileCTime(browser) {
   // Browsers installed at OS level are not tested
-  if (browser != "edge") {
-    let installTypes = [".zip", ".dmg", ".bz2"];
-    let browserZip =
-      cacheFiles.find(elem => installTypes.some(type => elem.includes(type)));
-    let browserStat = await fs.promises.stat(path.join(cacheDir, browserZip));
-    browserCacheTime = browserStat.ctimeMs;
-  }
+  if (browser == "edge" && (process.platform == "win32" ||
+                            process.platform == "darwin"))
+    return null;
 
-  let driverCacheTime = null;
-  // geckodriver is installed by npm, that's why Firefox is not tested here
-  if (browser != "firefox") {
-    let driverDir = cacheFiles.find(
-      elem => !elem.endsWith(".zip") && !elem.endsWith(".pkg"));
-    let driverFiles = await fs.promises.readdir(path.join(cacheDir, driverDir));
-    let driverZip = driverFiles.find(elem => elem.endsWith(".zip"));
-    let driverStat =
-      await fs.promises.stat(path.join(cacheDir, driverDir, driverZip));
-    driverCacheTime = driverStat.ctimeMs;
-  }
+  const installTypes = [".zip", ".dmg", ".bz2", ".deb", ".exe"];
+
+  let cacheDir = path.join(snapshotsBaseDir, browser, "cache");
+  let cacheFiles = await fs.promises.readdir(cacheDir);
+  let browserZip =
+    cacheFiles.find(elem => installTypes.some(type => elem.includes(type)));
+  if (!browserZip)
+    throw new Error(`Files in ${cacheDir} don't belong to any known install file types: ${installTypes}`);
 
-  return {browserCTime: browserCacheTime, driverCTime: driverCacheTime};
+  return (await fs.promises.stat(path.join(cacheDir, browserZip))).ctimeMs;
 }
 
 const browserNames = {
@@ -150,6 +138,19 @@ async function basicUrlTest(driver, browser) {
   expect(text).toEqual("Test server basic page");
 }
 
+// Adding the browser version to the test title for logging purposes
+function addVersionToTitle(ctx, version) {
+  ctx.test.title = `${ctx.test.title} [v${version}]`;
+}
+
+function isOldHeadlessMode(browser, version) {
+  // Chromium's old headless mode doesn't support loading extensions
+  return browser != "firefox" && (
+    !["latest", "beta", "dev"].includes(version) ||
+    // Edge on the Windows CI is v79, that is old headless
+    (browser == "edge" && process.platform == "win32"));
+}
+
 for (let browser of Object.keys(BROWSERS)) {
   describe(`Browser: ${browser}`, () => {
     before(async() => {
@@ -162,7 +163,7 @@ for (let browser of Object.keys(BROWSERS)) {
     for (let version of VERSIONS[browser]) {
       describe(`Version: ${version}`, () => {
         let driver = null;
-        let emptyCacheTimes = null;
+        let firstInstallFileCTime = null;
         let customBrowserBinary = null;
         let failedInstall = false;
 
@@ -177,6 +178,12 @@ for (let browser of Object.keys(BROWSERS)) {
             await killDriverProcess("chromedriver");
           else if (browser == "firefox")
             await killDriverProcess("geckodriver");
+
+          // https://gitlab.com/eyeo/developer-experience/get-browser-binary/-/issues/77
+          if (process.platform == "win32" && browser == "chromium")
+            await killDriverProcess("chrome");
+          else if (process.platform == "win32" && browser == "edge")
+            await killDriverProcess("msedge");
         }
 
         beforeEach(function() {
@@ -208,31 +215,32 @@ for (let browser of Object.keys(BROWSERS)) {
           expect(installedVersion).toEqual(
             expect.stringContaining(normalize(versionNumber)));
 
-          // Adding the version number to the test title for logging purposes
-          this.test.title = `${this.test.title} [v${versionNumber}]`;
+          addVersionToTitle(this, versionNumber);
+
+          // Data used in further tests
           customBrowserBinary = binary;
+          firstInstallFileCTime = await getInstallFileCTime(browser);
         });
 
-        it("runs", async() => {
+        it("runs", async function() {
           driver = await BROWSERS[browser].getDriver(version);
           await basicUrlTest(driver, browser);
 
-          // When running all tests, this saves time on the cache test
-          emptyCacheTimes = await getCachedTimes(browser);
+          let browserVersion =
+            (await driver.getCapabilities()).getBrowserVersion();
+          addVersionToTitle(this, browserVersion);
         });
 
-        it("uses cached install files", async() => {
-          if (emptyCacheTimes == null) { // Single test case run
-            driver = await BROWSERS[browser].getDriver(version);
-            emptyCacheTimes = await getCachedTimes(browser);
-            await quitDriver();
-          }
+        // This test depends on running the "installs" test
+        it("uses cached install files", async function() {
+          if (!firstInstallFileCTime)
+            this.skip();
 
           // assigning `driver` to allow the afterEach hook quit the driver
           driver = await BROWSERS[browser].getDriver(version);
-          let existingCacheTimes = await getCachedTimes(browser);
+          let secondInstallFileCTime = await getInstallFileCTime(browser);
 
-          expect(existingCacheTimes).toEqual(emptyCacheTimes);
+          expect(secondInstallFileCTime).toEqual(firstInstallFileCTime);
         });
 
         // This test depends on running the "installs" test
@@ -266,6 +274,22 @@ for (let browser of Object.keys(BROWSERS)) {
           expect(sizeSmall).toMeasureLessThan(sizeDefault);
         });
 
+        it("supports proxy", async function() {
+          if (browser != "firefox")
+            this.skip();
+
+          let headless = true;
+          let proxy = "http://localhost:3000/proxy-config.pac";
+          let extraArgs = [];
+          driver = await BROWSERS[browser].getDriver(
+            version, {headless, extraArgs, proxy});
+          await driver.navigate().to(PROXY_URL_BASIC);
+
+          let text = await driver.findElement(By.id("basic")).getText();
+          expect(text).toEqual("Test server basic page");
+          await quitDriver();
+        });
+
         it("takes a full page screenshot", async() => {
           driver = await BROWSERS[browser].getDriver(version);
           await driver.navigate().to(TEST_URL_LONG);
@@ -279,10 +303,13 @@ for (let browser of Object.keys(BROWSERS)) {
           expect(fullImg.bitmap.height).toBeGreaterThan(partImg.bitmap.height);
         });
 
+        // The only reason the geckodriver package is required in package.json
+        // is to install a specific version needed by Firefox 68.0, otherwise it
+        // fails to load extensions. When the oldest Firefox version is bumped,
+        // Selenium's automated driver download should be able to manage it.
+        // https://gitlab.com/eyeo/developer-experience/get-browser-binary/-/issues/44
         it("loads an extension", async() => {
-          // Chromium's old headless mode doesn't support loading extensions
-          let headless = browser == "firefox" || (browser == "chromium" &&
-            ["latest", "beta", "dev"].includes(version));
+          let headless = !isOldHeadlessMode(browser, version);
           let {extensionPaths} = getExtension(browser, version);
 
           driver = await BROWSERS[browser].getDriver(
@@ -302,6 +329,29 @@ for (let browser of Object.keys(BROWSERS)) {
           );
           await getHandle(driver, "/index.html");
         });
+
+        it("updates an extension", async() => {
+          let headless = !isOldHeadlessMode(browser, version);
+          let {extensionPaths, manifest} = getExtension(browser, version);
+
+          let tmpExtensionDir = path.join(snapshotsBaseDir, "extension");
+          await fs.promises.rm(tmpExtensionDir, {recursive: true, force: true});
+          await fs.promises.cp(extensionPaths[0], tmpExtensionDir,
+                               {recursive: true});
+
+          driver = await BROWSERS[browser].getDriver(
+            version, {headless, extensionPaths: [tmpExtensionDir]});
+          await getHandle(driver, "/index.html");
+          let text = await driver.findElement(By.id("title")).getText();
+          expect(text).toEqual(`Browser test extension - ${manifest}`);
+
+          // The page is modified and reloaded to emulate an extension update
+          await fs.promises.cp(path.join(tmpExtensionDir, "update.html"),
+                               path.join(tmpExtensionDir, "index.html"));
+          await driver.navigate().refresh();
+          text = await driver.findElement(By.id("title")).getText();
+          expect(text).toEqual(`Updated test extension - ${manifest}`);
+        });
       });
     }
 
@@ -314,5 +364,12 @@ for (let browser of Object.keys(BROWSERS)) {
           .rejects.toThrow(`Unsupported browser version: ${unsupported}`);
       }
     });
+
+    it("does not load an invalid extension", async() => {
+      let extensionPaths = [process.cwd()];
+
+      await expect(BROWSERS[browser].getDriver("latest", {extensionPaths}))
+        .rejects.toThrow(`Extension manifest file not found: ${extensionPaths[0]}`);
+    });
   });
 }

package/test/extension/mv2/index.html

@@ -1 +1 @@
-<h1>Browser test extension - mv2</h1>
+<h1 id="title">Browser test extension - mv2</h1>

package/test/extension/mv2/update.html

@@ -0,0 +1 @@
+<h1 id="title">Updated test extension - mv2</h1>

package/test/extension/mv3/index.html

@@ -1 +1 @@
-<h1>Browser test extension - mv3</h1>
+<h1 id="title">Browser test extension - mv3</h1>

package/test/extension/mv3/update.html

@@ -0,0 +1 @@
+<h1 id="title">Updated test extension - mv3</h1>

package/test/pages/proxy-config.pac

@@ -0,0 +1,7 @@
+
+function FindProxyForURL(url, host) {
+  if (host === "testpages.adblockplus.org")
+    return "PROXY http://localhost:3000";
+
+  return "DIRECT";
+}

package/test/runner.js

@@ -53,10 +53,12 @@ export async function killTestServer() {
 }
 
 async function run() {
-  let args = process.argv.join(" ").split("-- ")[1];
+  let args = process.argv.slice(2); // Keep npm args from "--" onwards
+  args = args.map(v => v.startsWith("-") ? v : `"${v}"`);
+
   await runTestServer();
   try {
-    execSync(`npm run test-suite -- ${args}`, {stdio: "inherit"});
+    execSync(`npm run test-suite ${args.join(" ")}`, {stdio: "inherit"});
   }
   finally {
     await killTestServer();