@eyeo/get-browser-binary 0.2.0 → 0.3.0

package/.gitlab-ci.yml

@@ -19,7 +19,15 @@ cache:
     paths:
     - node_modules/
 
-test:linux:
+test:basic:
+  stage: test
+  before_script:
+    - npm install
+  script:
+    - npm run lint
+    - npm test -- --grep "Utils"
+
+test:browsers:linux:
   stage: test
   before_script:
     - apt-get update && apt-get install -y procps
@@ -32,13 +40,9 @@ test:linux:
     - rm microsoft.*
     - apt-get update && apt-get install -y microsoft-edge-stable
   script:
-    - npm run lint
-    - xvfb-run -a npm test -- -g "chromium (latest|beta)"
-    - xvfb-run -a npm test -- -g "chromium dev"
-    - xvfb-run -a npm test -- -g "chromium 77"
-    - xvfb-run -a npm test -- -g "(edge|firefox)"
+    - xvfb-run -a npm test -- --grep "Browser"
 
-test:windows:
+test:browsers:windows:
   stage: test
   variables:
     CI_PROJECT_ID_MAINSTREAM: 36688302
@@ -52,11 +56,8 @@ test:windows:
     - choco upgrade -y nodejs --version 16.10.0
     - npm install
   script:
-    - npm test -- -g "chromium latest"
-    - npm test -- -g "chromium beta"
-    - npm test -- -g "chromium dev"
-    - npm test -- -g "chromium 77"
-    - npm test -- -g "edge"
+    - npm test -- --grep "chromium"
+    - npm test -- --grep "edge"
   tags:
     - shared-windows
     - windows

package/README.md

@@ -25,10 +25,9 @@ import {BROWSERS} from "@eyeo/get-browser-binary";
   await driver.navigate().to("https://example.com/");
   await driver.quit();
 })();
-
 ```
 
-[test/runner.js](https://gitlab.com/eyeo/developer-experience/get-browser-binary/-/blob/main/test/runner.js)
+[test/browsers.js](https://gitlab.com/eyeo/developer-experience/get-browser-binary/-/blob/main/test/browsers.js)
 provides other usage examples of the library.
 
 For more information, please refer to the [API documention](https://gitlab.com/eyeo/developer-experience/get-browser-binary/-/jobs/artifacts/main/file/docs/index.html?job=docs).
@@ -48,15 +47,30 @@ the right side.
 npm install
 ```
 
+### Folders to ignore / cache
+
+All browser and webdriver files will be downloaded to the `./browser-snapshots`
+folder, which probably makes sense to be ignored (for instance, by adding it to
+`.gitignore`).
+
+On the other hand, `./browser-snapshots/<browser_name>/cache` will hold all the
+downloaded installation files required by `<browser_name>`. Therefore, it may be
+useful to add `./browser-snapshots/*/cache` to the list of cached folders in
+your CI pipeline configuration.
+
 ## Testing
 
-The `grep` option filters the tests to run with a regular expression. Example:
+Running all tests:
 
 ```shell
-npm test -- -g "chromium latest"
+npm test
 ```
 
-Note: Running all Chromium tests at once is currently not working.
+The `grep` option filters the tests to run with a regular expression. Example:
+
+```shell
+npm test -- --grep "chromium latest"
+```
 
 ## Building the documentation
 

package/RELEASE_NOTES.md

@@ -1,3 +1,11 @@
+# 0.3.0
+
+- Adds incognito mode support to all browsers (#5)
+- Exports the utils download function (#12)
+- Adds the details of the directories to ignore / cache to the README (#13)
+- Fixes Chromium tests execution on all platforms (#3)
+- Updates the geckodriver package (#14)
+
 # 0.2.0
 
 Change `Browser.getDriver` signature to use `driverOptions` parameter.

package/index.js

@@ -16,3 +16,4 @@
  */
 
 export * from "./src/browsers.js";
+export {download} from "./src/utils.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@eyeo/get-browser-binary",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Download browser binaries and matching webdrivers",
   "repository": {
     "type": "git",
@@ -19,7 +19,7 @@
     "dmg": "^0.1.0",
     "extract-zip": "^2.0.1",
     "fs-extra": "^10.0.0",
-    "geckodriver": "^2.0.4",
+    "geckodriver": "^3.0.2",
     "got": "^11.8.2",
     "msedgedriver": "^91.0.0",
     "selenium-webdriver": "^4.2.0"
@@ -32,8 +32,8 @@
     "mocha": "^10.0.0"
   },
   "scripts": {
-    "docs": "jsdoc --readme README.md --destination docs src/browsers.js",
+    "docs": "jsdoc --readme README.md --destination docs src/*.js",
     "lint": "eslint --ext js .",
-    "test": "mocha test/runner.js --"
+    "test": "mocha test/*.js --"
   }
 }

package/src/browsers.js

@@ -27,8 +27,8 @@ import firefox from "selenium-webdriver/firefox.js";
 import command from "selenium-webdriver/lib/command.js";
 import extractZip from "extract-zip";
 
-import {download, extractTar, extractDmg, runWinInstaller, getBrowserVersion}
-  from "./utils.js";
+import {download, extractTar, extractDmg, runWinInstaller, getBrowserVersion,
+        killDriverProcess} from "./utils.js";
 
 /**
  * Root folder where browser and webdriver binaries get downloaded.
@@ -37,9 +37,9 @@ import {download, extractTar, extractDmg, runWinInstaller, getBrowserVersion}
 export let snapshotsBaseDir = path.join(process.cwd(), "browser-snapshots");
 
 let {until, By} = webdriver;
-const ERROR_INCOGNITO_NOT_SUPPORTED = "Incognito mode is not supported";
 const ERROR_DOWNLOAD_NOT_SUPPORTED =
   "Downloading this browser is not supported";
+let platform = `${process.platform}-${process.arch}`;
 
 /**
  * Base class for browser download functionality. Please see subclasses for
@@ -89,6 +89,7 @@ class Browser {
    *   no effect.
    * @param {driverOptions?} options - Options to start the browser with.
    * @return {webdriver}
+   * @throws {Error} Unsupported webdriver version.
    */
   static async getDriver(version, options = {}) {
     // to be implemented by the subclass
@@ -102,7 +103,10 @@ class Browser {
    * @return {webdriver}
    */
   static async enableExtensionInIncognito(driver, extensionTitle) {
-    throw new Error(ERROR_INCOGNITO_NOT_SUPPORTED);
+    // Allowing the extension in incognito mode can't happen programmatically:
+    // https://stackoverflow.com/questions/57419654
+    // https://bugzilla.mozilla.org/show_bug.cgi?id=1729315
+    // That is done through the UI, to be implemented by the subclass
   }
 }
 
@@ -159,17 +163,13 @@ class Chromium extends Browser {
 
     let revision = parseInt(chromiumRevision, 10);
     let startingRevision = revision;
-    let platform = `${process.platform}-${process.arch}`;
-
-    let buildTypes = {
+    let [platformDir, fileName] = {
       "win32-ia32": ["Win", "chrome-win.zip"],
       "win32-x64": ["Win_x64", "chrome-win.zip"],
       "linux-x64": ["Linux_x64", "chrome-linux.zip"],
       "darwin-x64": ["Mac", "chrome-mac.zip"],
       "dawrin-arm64": ["Mac_Arm", "chrome-mac.zip"]
-    };
-
-    let [platformDir, fileName] = buildTypes[platform];
+    }[platform];
     let archive = null;
     let browserDir = null;
     let snapshotsDir = path.join(snapshotsBaseDir, "chromium");
@@ -229,15 +229,13 @@ class Chromium extends Browser {
   }
 
   static async #installDriver(revision, version) {
-    let platform = `${process.platform}-${process.arch}`;
-    let buildTypes = {
+    let [dir, zip, driver] = {
       "win32-ia32": ["Win", "chromedriver_win32.zip", "chromedriver.exe"],
       "win32-x64": ["Win_x64", "chromedriver_win32.zip", "chromedriver.exe"],
       "linux-x64": ["Linux_x64", "chromedriver_linux64.zip", "chromedriver"],
       "darwin-x64": ["Mac", "chromedriver_mac64.zip", "chromedriver"],
       "darwin-arm64": ["Mac_Arm", "chromedriver_mac64.zip", "chromedriver"]
-    };
-    let [dir, zip, driver] = buildTypes[platform];
+    }[platform];
 
     let cacheDir = path.join(snapshotsBaseDir, "chromium", "cache", version);
     let destinationDir = path.join(process.cwd(), "node_modules",
@@ -247,25 +245,17 @@ class Chromium extends Browser {
     await download(`https://commondatastorage.googleapis.com/chromium-browser-snapshots/${dir}/${revision}/${zip}`,
                    archive);
     await extractZip(archive, {dir: cacheDir});
+    // avoid driver copy failing if the file would be locked
+    await killDriverProcess(Chromium.#DRIVER);
     await fs.promises.mkdir(destinationDir, {recursive: true});
-    try {
-      await fs.promises.copyFile(path.join(cacheDir, zip.split(".")[0], driver),
-                                 path.join(destinationDir, driver));
-    }
-    catch (err) {
-      // https://gitlab.com/eyeo/developer-experience/get-browser-binary/-/issues/3
-      if (!err.toString().includes("copyfile"))
-        throw err;
-    }
+    await fs.promises.copyFile(path.join(cacheDir, zip.split(".")[0], driver),
+                               path.join(destinationDir, driver));
   }
 
   /** @see Browser.getDriver */
   static async getDriver(version, {headless = true, extensionPaths = [],
                                    incognito = false, insecure = false,
                                    extraArgs = []} = {}) {
-    if (incognito)
-      throw new Error(ERROR_INCOGNITO_NOT_SUPPORTED);
-
     let {binary, revision, downloadedVersion} =
       await Chromium.downloadBinary(version);
     await Chromium.#installDriver(revision, downloadedVersion);
@@ -277,6 +267,8 @@ class Chromium extends Browser {
       options.headless();
     if (insecure)
       options.addArguments("ignore-certificate-errors");
+    if (incognito)
+      options.addArguments("incognito");
     options.setChromeBinaryPath(binary);
 
     let builder = new webdriver.Builder();
@@ -285,6 +277,41 @@ class Chromium extends Browser {
 
     return builder.build();
   }
+
+  /** @see Browser.enableExtensionInIncognito */
+  static async enableExtensionInIncognito(driver, extensionTitle) {
+    let version = await getBrowserVersion(driver);
+    if (version < 75)
+      // The UI workaround needs a chromedriver >= 75
+      throw new Error(`Only supported on Chromium >= 75. Current version: ${version}`);
+
+    await driver.navigate().to("chrome://extensions");
+    await driver.executeScript(`
+      let enable = () => document.querySelector("extensions-manager").shadowRoot
+        .querySelector("extensions-detail-view").shadowRoot
+        .getElementById("allow-incognito").shadowRoot
+        .getElementById("crToggle").click();
+
+      let extensions = document.querySelector("extensions-manager").shadowRoot
+        .getElementById("items-list").shadowRoot
+        .querySelectorAll("extensions-item");
+
+      return new Promise((resolve, reject) => {
+        let extensionDetails;
+        for (let {shadowRoot} of extensions) {
+          if (shadowRoot.getElementById("name").innerHTML != arguments[0])
+            continue;
+
+          extensionDetails = shadowRoot.getElementById("detailsButton");
+        }
+
+        if (!extensionDetails)
+          reject("Extension was not found");
+
+        extensionDetails.click();
+        setTimeout(() => resolve(enable()), 100);
+      });`, extensionTitle);
+  }
 }
 
 /**
@@ -326,18 +353,18 @@ class Firefox {
   }
 
   static async #downloadFirefox(version) {
-    let {platform} = process;
-    if (platform == "win32")
-      platform += "-" + process.arch;
-    let buildTypes = {
+    let [buildPlatform, fileName] = {
       "win32-ia32": ["win32", `Firefox Setup ${version}.exe`],
       "win32-x64": ["win64", `Firefox Setup ${version}.exe`],
-      "linux": ["linux-x86_64", `firefox-${version}.tar.bz2`],
-      "darwin": ["mac", `Firefox ${version}.dmg`]
-    };
+      "linux-x64": ["linux-x86_64", `firefox-${version}.tar.bz2`],
+      "darwin-x64": ["mac", `Firefox ${version}.dmg`],
+      "darwin-arm64": ["mac", `Firefox ${version}.dmg`]
+    }[platform];
 
     let snapshotsDir = path.join(snapshotsBaseDir, "firefox");
     let browserDir = path.join(snapshotsDir, `firefox-${platform}-${version}`);
+    let archive = path.join(snapshotsDir, "cache", fileName);
+
     try {
       await fs.promises.access(browserDir);
       return Firefox.#getFirefoxBinary(browserDir);
@@ -345,10 +372,6 @@ class Firefox {
     catch (e) {}
 
     await fs.promises.mkdir(path.dirname(browserDir), {recursive: true});
-
-    let [buildPlatform, fileName] = buildTypes[platform];
-    let archive = path.join(snapshotsDir, "cache", fileName);
-
     try {
       await fs.promises.access(archive);
     }
@@ -409,14 +432,10 @@ class Firefox {
 
   /** @see Browser.enableExtensionInIncognito */
   static async enableExtensionInIncognito(driver, extensionTitle) {
-    // Allowing the extension in private browsing can't happen programmatically:
-    // https://bugzilla.mozilla.org/show_bug.cgi?id=1729315
-    // Therefore, that is done through the UI
     let version = await getBrowserVersion(driver);
-    if (version < 87) {
+    if (version < 87)
       // The UI workaround assumes web elements only present on Firefox >= 87
       throw new Error(`Only supported on Firefox >= 87. Current version: ${version}`);
-    }
 
     await driver.navigate().to("about:addons");
     await driver.wait(until.elementLocated(By.name("extension")), 1000).click();
@@ -464,15 +483,13 @@ class Edge {
 
     // Based on "node_modules/msedgedriver/install.js", adding a fallback
     // mechanism when msedgedriver doesn't exist for the latest Edge version.
-    let platform = `${process.platform}-${process.arch}`;
-    let buildTypes = {
+    let [zip, driver] = {
       "win32-ia32": ["edgedriver_win32.zip", "msedgedriver.exe"],
       "win32-x64": ["edgedriver_win64.zip", "msedgedriver.exe"],
       "linux-x64": ["edgedriver_linux64.zip", "msedgedriver"],
       "darwin-x64": ["edgedriver_mac64.zip", "msedgedriver"],
       "darwin-arm64": ["edgedriver_arm64.zip", "msedgedriver"]
-    };
-    let [zip, driver] = buildTypes[platform];
+    }[platform];
     let cacheDir = path.join(snapshotsBaseDir, "edge", "cache");
     let driverBinDir = path.join(process.cwd(), "node_modules", Edge.#DRIVER,
                                  "bin");
@@ -498,17 +515,12 @@ class Edge {
       throw new Error(`msedgedriver was not found for Edge ${version}`);
 
     await extractZip(archive, {dir: cacheDir});
+    // avoid driver copy failing if the file would be locked
+    await killDriverProcess(Edge.#DRIVER);
     for (let destinationDir of [driverBinDir, driverLibDir]) {
       await fs.promises.mkdir(destinationDir, {recursive: true});
-      try {
-        await fs.promises.copyFile(path.join(cacheDir, driver),
-                                   path.join(destinationDir, driver));
-      }
-      catch (err) {
-        // https://gitlab.com/eyeo/developer-experience/get-browser-binary/-/issues/3
-        if (!err.toString().includes("copyfile"))
-          throw err;
-      }
+      await fs.promises.copyFile(path.join(cacheDir, driver),
+                                 path.join(destinationDir, driver));
     }
   }
 
@@ -516,9 +528,6 @@ class Edge {
   static async getDriver(version, {headless = true, extensionPaths = [],
                                    incognito = false, insecure = false,
                                    extraArgs = []} = {}) {
-    if (incognito)
-      throw new Error(ERROR_INCOGNITO_NOT_SUPPORTED);
-
     await Edge.#installDriver();
 
     let args = ["no-sandbox", ...extraArgs];
@@ -526,6 +535,8 @@ class Edge {
       args.push("headless");
     if (extensionPaths.length > 0)
       args.push(`load-extension=${extensionPaths.join(",")}`);
+    if (incognito)
+      args.push("incognito");
 
     let builder = new webdriver.Builder();
     builder.forBrowser("MicrosoftEdge");
@@ -538,6 +549,32 @@ class Edge {
 
     return builder.build();
   }
+
+  /** @see Browser.enableExtensionInIncognito */
+  static async enableExtensionInIncognito(driver, extensionTitle) {
+    let version = await getBrowserVersion(driver);
+    if (version < 79)
+      // The UI workaround needs a chromium based msedgedriver
+      throw new Error(`Only supported on Edge >= 79. Current version: ${version}`);
+
+    await driver.navigate().to("edge://extensions/");
+    for (let elem of await driver.findElements(By.css("[role=listitem]"))) {
+      let text = await elem.getAttribute("innerHTML");
+      if (!text.includes(extensionTitle))
+        continue;
+
+      for (let button of await elem.findElements(By.css("button"))) {
+        text = await elem.getAttribute("innerHTML");
+        if (!text.includes("Details"))
+          continue;
+
+        await button.click();
+        return await driver.findElement(By.id("itemAllowIncognito")).click();
+      }
+      throw new Error("Details button not found");
+    }
+    throw new Error(`Extension "${extensionTitle}" not found`);
+  }
 }
 
 

package/src/utils.js

@@ -25,6 +25,12 @@ import got from "got";
 import fsExtra from "fs-extra";
 import dmg from "dmg";
 
+/**
+ * Downloads url resources.
+ * @param {string} url - The url of the resource to be downloaded.
+ * @param {string} destFile - The destination file path.
+ * @throws {TypeError} Invalid URL.
+ */
 export async function download(url, destFile) {
   let cacheDir = path.dirname(destFile);
 
@@ -37,7 +43,11 @@ export async function download(url, destFile) {
     await promisify(pipeline)(got.stream(url), writable);
   }
   catch (error) {
-    await fs.promises.rm(tempDest, {recursive: true});
+    try {
+      await fs.promises.rm(tempDest, {recursive: true});
+    }
+    catch (e) {}
+
     throw error;
   }
 
@@ -78,3 +88,21 @@ export async function getBrowserVersion(driver) {
   let version = (await driver.getCapabilities()).getBrowserVersion();
   return parseInt(version.split(".")[0], 10);
 }
+
+export async function killDriverProcess(driverName) {
+  let cmd = `kill $(pgrep ${driverName})`;
+  let shell;
+  if (process.platform == "win32") {
+    cmd = `Get-Process -Name ${driverName} | Stop-Process; Start-Sleep 1`;
+    shell = "powershell.exe";
+  }
+
+  try {
+    await promisify(exec)(cmd, {shell});
+  }
+  catch (err) {
+    // Command will fail when the driver process is not found
+    if (!err.toString().includes("Command failed"))
+      throw err;
+  }
+}

package/test/{runner.js → browsers.js}

@@ -18,14 +18,12 @@
 import fs from "fs";
 import expect from "expect";
 import path from "path";
-import {exec} from "child_process";
-import {promisify} from "util";
 
-import {BROWSERS, snapshotsBaseDir} from "../src/browsers.js";
+import {BROWSERS, snapshotsBaseDir} from "../index.js";
+import {killDriverProcess} from "../src/utils.js";
 
-// Required to start the driver on some platforms (e.g. Windows).
+// Required to set the driver path on Windows
 import "chromedriver";
-import "geckodriver";
 import "msedgedriver";
 
 const VERSIONS = {
@@ -64,44 +62,10 @@ async function getHandle(driver, page) {
   return handle;
 }
 
-async function killDriverProcess() {
-  for (let name of ["chromedriver", "msedgedriver"]) {
-    let stdout;
-    if (process.platform == "win32") {
-      try {
-        ({stdout} = await promisify(exec)(
-          `(Get-Process -Name ${name}).Id | Stop-Process`,
-          {shell: "powershell.exe"}
-        ));
-      }
-      catch (err) {
-        if (err.toString().includes("Command failed"))
-          continue; // Command will fail when driver process is not found
-
-        throw err;
-      }
-    }
-    else {
-      try {
-        ({stdout} =
-          await promisify(exec)(`ps -a | grep ${name} | grep -v grep`));
-      }
-      catch (err) {
-        if (err.toString().includes("Command failed"))
-          continue; // Command will fail when driver process is not found
-
-        throw err;
-      }
-      let pid = /\s*(\d*)/.exec(stdout)[1]; // first number after any spaces
-      await promisify(exec)(`kill ${pid}`);
-    }
-  }
-}
-
 for (let browser of Object.keys(BROWSERS)) {
   for (let version of VERSIONS[browser]) {
     describe(`Browser: ${browser} ${version || "latest"}`, function() {
-      this.timeout(15000);
+      this.timeout(20000);
 
       before(async() => {
         try {
@@ -117,8 +81,9 @@ for (let browser of Object.keys(BROWSERS)) {
 
         await driver.quit();
         driver = null;
-        // Some platforms don't immediately kill the webdriver process
-        await killDriverProcess();
+        // Some platforms don't immediately kill the chromedriver process
+        if (browser == "chromium")
+          await killDriverProcess("chromedriver");
       });
 
       it("downloads", async function() {
@@ -163,9 +128,12 @@ for (let browser of Object.keys(BROWSERS)) {
       });
 
       it("loads an extension in incognito mode", async function() {
-        if (browser != "firefox" || version == "68.0")
+        if (browser == "firefox" && version == "68.0")
           this.skip();
 
+        if (process.platform == "win32")
+          this.timeout(30000);
+
         driver = await BROWSERS[browser].getDriver(
           version, {headless: false, extensionPaths, incognito: true});
         await BROWSERS[browser].enableExtensionInIncognito(

package/test/utils.js

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2006-present eyeo GmbH
+ *
+ * This module is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import fs from "fs";
+import expect from "expect";
+import path from "path";
+
+import {snapshotsBaseDir, download} from "../index.js";
+
+describe("Utils", () => {
+  it("defines a browser snapshots folder", () => expect(snapshotsBaseDir)
+    .toBe(path.join(process.cwd(), "browser-snapshots")));
+
+  let destFile = path.join(snapshotsBaseDir, "download-test.txt");
+
+  it("downloads resources", async() => {
+    let url = "https://gitlab.com/eyeo/developer-experience/get-browser-binary/-/raw/main/package.json";
+    await download(url, destFile);
+    let contents = await fs.promises.readFile(destFile, {encoding: "utf8"});
+    expect(contents).toEqual(
+      expect.stringContaining("\"name\": \"@eyeo/get-browser-binary\""));
+  });
+
+  it("does not download invalid resources", async() => {
+    expect(download("invalid", destFile)).rejects
+      .toThrow("TypeError [ERR_INVALID_URL]: Invalid URL");
+  });
+});