@eyeo/get-browser-binary 0.8.0 → 0.9.0

package/.gitlab-ci.yml

@@ -32,12 +32,10 @@ test:browsers:linux:
   image: docker:20.10.16
   services:
     - docker:20.10.16-dind
-  variables:
-    DOCKER_DRIVER: overlay2
   before_script:
     - docker build -f test/docker/Dockerfile -t browsers .
   script:
-    - docker run --shm-size=256m browsers
+    - docker run --shm-size=512m -t browsers
 
 test:browsers:windows:
   stage: test

package/.mocharc.json

@@ -1,3 +1,3 @@
 {
-  "timeout": 50000
+  "timeout": 80000
 }

package/README.md

@@ -88,14 +88,14 @@ Useful to reproduce the CI environment of the `test:browsers:linux` job:
 
 ```shell
 docker build -f test/docker/Dockerfile -t browsers .
-docker run --shm-size=256m -it browsers
+docker run --shm-size=512m -it browsers
 ```
 
 The `grep` and `timeout` options can also be used on Docker via the `TEST_ARGS`
 parameter:
 
 ```shell
-docker run --shm-size=256m -e TEST_ARGS="--grep chromium.*latest --timeout 100000" -it browsers
+docker run --shm-size=512m -e TEST_ARGS="--grep chromium.*latest --timeout 100000" -it browsers
 ```
 
 By default, tests delete the `./browser-snapshots` before each `Browser` suite

package/RELEASE_NOTES.md

@@ -1,3 +1,17 @@
+# 0.9.0
+
+- Fixed a Chromium install issue by increasing the value of
+`MAX_VERSION_DECREMENTS` (!52)
+- `installBrowser()`, `getDriver()` and `download()` have a new optional timeout
+parameter on file downloads (!51)
+
+### Testing
+
+- Added a test checking that downloaded browser-snapshot files are actually
+cached (#35)
+- Tests running the minimum firefox version showed occasional failures. That was
+fixed by increasing the shared memory size of the docker image (!50)
+
 # 0.8.0
 
 - Move `takeFullPageScreenshot` to the utils module (#38)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@eyeo/get-browser-binary",
-  "version": "0.8.0",
+  "version": "0.9.0",
   "description": "Install browser binaries and matching webdrivers",
   "repository": {
     "type": "git",

package/src/browsers.js

@@ -82,11 +82,14 @@ class Browser {
    *   subclasses.
    * @param {string} version - Either full version number or channel/release.
    *   Please find examples on the subclasses.
+   * @param {number?} downloadTimeout - Allowed time in ms for the download of
+   *   install files to complete. When set to 0 there is no time limit. Defaults
+   *   to 0.
    * @return {BrowserBinary}
    * @throws {Error} Unsupported browser version, Unsupported platform, Browser
    *   download failed.
    */
-  static async installBrowser(version) {
+  static async installBrowser(version, downloadTimeout = 0) {
     // to be implemented by the subclass
   }
 
@@ -134,11 +137,14 @@ class Browser {
    * @param {string} version - Either full version number or channel/release.
    *   Please find examples on the subclasses.
    * @param {driverOptions?} options - Options to start the browser with.
+   * @param {number?} downloadTimeout - Allowed time in ms for the download of
+   *   browser install files to complete. When set to 0 there is no time limit.
+   *   Defaults to 0.
    * @return {webdriver}
    * @throws {Error} Unsupported browser version, Unsupported platform, Browser
    *   download failed, Driver download failed, Unable to start driver.
    */
-  static async getDriver(version, options = {}) {
+  static async getDriver(version, options = {}, downloadTimeout = 0) {
     // to be implemented by the subclass
   }
 
@@ -208,13 +214,16 @@ class Chromium extends Browser {
    *   {@link snapshotsBaseDir} folder, ready to go.
    * @param {string} version - Either "latest", "beta", "dev" or a full version
    *   number (i.e. "77.0.3865.0"). Defaults to "latest".
+   * @param {number?} downloadTimeout - Allowed time in ms for the download of
+   *   install files to complete. When set to 0 there is no time limit. Defaults
+   *   to 0.
    * @return {BrowserBinary}
    * @throws {Error} Unsupported browser version, Unsupported platform, Browser
    *   download failed.
    */
-  static async installBrowser(version = "latest") {
+  static async installBrowser(version = "latest", downloadTimeout = 0) {
     const MIN_VERSION = 75;
-    const MAX_VERSION_DECREMENTS = 80;
+    const MAX_VERSION_DECREMENTS = 200;
 
     checkVersion(version, MIN_VERSION, Chromium.#CHANNELS);
     let versionNumber = await Chromium.#getVersionForChannel(version);
@@ -248,23 +257,27 @@ class Chromium extends Browser {
       await fs.promises.mkdir(path.dirname(browserDir), {recursive: true});
 
       archive = path.join(snapshotsDir, "cache", `${base}-${fileName}`);
+      let url = `https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/${platformDir}%2F${base}%2F${fileName}?alt=media`;
       try {
         try {
           await fs.promises.access(archive);
         }
         catch (e) {
-          await download(
-            `https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/${platformDir}%2F${base}%2F${fileName}?alt=media`,
-            archive);
+          await download(url, archive, downloadTimeout);
         }
         break;
       }
-      catch (e) {
-        // Chromium advises decrementing the branch_base_position when no
-        // matching build was found. See https://www.chromium.org/getting-involved/download-chromium
-        base--;
-        if (base <= startBase - MAX_VERSION_DECREMENTS)
-          throw new Error(`${BROWSER_DOWNLOAD_ERROR}: Chromium base ${startBase}`);
+      catch (err) {
+        if (err.name == "HTTPError") {
+          // Chromium advises decrementing the branch_base_position when no
+          // matching build was found. See https://www.chromium.org/getting-involved/download-chromium
+          base--;
+          if (base <= startBase - MAX_VERSION_DECREMENTS)
+            throw new Error(`${BROWSER_DOWNLOAD_ERROR}: Chromium base ${startBase}`);
+        }
+        else {
+          throw new Error(`${BROWSER_DOWNLOAD_ERROR}: ${url}\n${err}`);
+        }
       }
     }
     await extractZip(archive, {dir: browserDir});
@@ -302,10 +315,7 @@ class Chromium extends Browser {
 
     await killDriverProcess("chromedriver");
     let driverPath = path.join(cacheDir, zip.split(".")[0], driverBinary);
-    try {
-      await fs.promises.rm(driverPath, {recursive: true});
-    }
-    catch (e) {} // file does not exist
+    await fs.promises.rm(driverPath, {force: true});
     await extractZip(archive, {dir: cacheDir});
 
     return driverPath;
@@ -315,8 +325,9 @@ class Chromium extends Browser {
   static async getDriver(version = "latest", {
     headless = true, extensionPaths = [], incognito = false, insecure = false,
     extraArgs = []
-  } = {}) {
-    let {binary, versionNumber, base} = await Chromium.installBrowser(version);
+  } = {}, downloadTimeout = 0) {
+    let {binary, versionNumber, base} =
+      await Chromium.installBrowser(version, downloadTimeout);
     let driverPath = await Chromium.#installDriver(base, versionNumber);
     let serviceBuilder = new chrome.ServiceBuilder(driverPath);
     let options = new chrome.Options().addArguments("no-sandbox", ...extraArgs);
@@ -414,11 +425,14 @@ class Firefox extends Browser {
    *   {@link snapshotsBaseDir} folder, ready to go.
    * @param {string} version - Either "latest", "beta" or a full version
    *   number (i.e. "68.0"). Defaults to "latest".
+   * @param {number?} downloadTimeout - Allowed time in ms for the download of
+   *   install files to complete. When set to 0 there is no time limit. Defaults
+   *   to 0.
    * @return {BrowserBinary}
    * @throws {Error} Unsupported browser version, Unsupported platform, Browser
    *   download failed.
    */
-  static async installBrowser(version = "latest") {
+  static async installBrowser(version = "latest", downloadTimeout = 0) {
     const MIN_VERSION = 60;
 
     checkVersion(version, MIN_VERSION, Firefox.#CHANNELS);
@@ -450,7 +464,7 @@ class Firefox extends Browser {
     catch (e) {
       let url = `https://archive.mozilla.org/pub/firefox/releases/${versionNumber}/${buildPlatform}/en-US/${fileName}`;
       try {
-        await download(url, archive);
+        await download(url, archive, downloadTimeout);
       }
       catch (err) {
         throw new Error(`${BROWSER_DOWNLOAD_ERROR}: ${url}\n${err}`);
@@ -465,8 +479,8 @@ class Firefox extends Browser {
   static async getDriver(version = "latest", {
     headless = true, extensionPaths = [], incognito = false, insecure = false,
     extraArgs = []
-  } = {}) {
-    let {binary} = await Firefox.installBrowser(version);
+  } = {}, downloadTimeout = 0) {
+    let {binary} = await Firefox.installBrowser(version, downloadTimeout);
 
     let options = new firefox.Options();
     if (headless)
@@ -601,11 +615,14 @@ class Edge extends Browser {
    *   app (not as a system app). Installing Edge on Windows is not supported.
    * @param {string} version - Either "latest", "beta", "dev" or a full version
    *   number (i.e. "95.0.1020.40"). Defaults to "latest".
+   * @param {number?} downloadTimeout - Allowed time in ms for the download of
+   *   install files to complete. When set to 0 there is no time limit. Defaults
+   *   to 0.
    * @return {BrowserBinary}
    * @throws {Error} Unsupported browser version, Unsupported platform, Browser
    *   download failed.
    */
-  static async installBrowser(version = "latest") {
+  static async installBrowser(version = "latest", downloadTimeout = 0) {
     if (platform == "win32")
       // Edge is mandatory on Windows, can't be uninstalled or downgraded
       // https://support.microsoft.com/en-us/microsoft-edge/why-can-t-i-uninstall-microsoft-edge-ee150b3b-7d7a-9984-6d83-eb36683d526d
@@ -642,7 +659,7 @@ class Edge extends Browser {
     catch (e) {}
 
     try {
-      await download(url, archive);
+      await download(url, archive, downloadTimeout);
     }
     catch (err) {
       throw new Error(`${BROWSER_DOWNLOAD_ERROR}: ${url}\n${err}`);
@@ -653,10 +670,7 @@ class Edge extends Browser {
     }
     else if (platform == "darwin") {
       let appName = Edge.#getDarwinAppName(channel);
-      try {
-        await fs.promises.rm(`${process.env.HOME}/Applications/${appName}.app`, {recursive: true});
-      }
-      catch (e) {}
+      await fs.promises.rm(`${process.env.HOME}/Applications/${appName}.app`, {force: true, recursive: true});
       await promisify(exec)(`installer -pkg ${archive} -target CurrentUserHomeDirectory`);
     }
 
@@ -673,10 +687,7 @@ class Edge extends Browser {
   static async #installDriver() {
     async function extractEdgeZip(archive, cacheDir, driverPath) {
       await killDriverProcess("msedgedriver");
-      try {
-        await fs.promises.rm(driverPath, {recursive: true});
-      }
-      catch (e) {} // file does not exist
+      await fs.promises.rm(driverPath, {force: true});
       await extractZip(archive, {dir: cacheDir});
     }
 
@@ -727,9 +738,9 @@ class Edge extends Browser {
   static async getDriver(version = "latest", {
     headless = true, extensionPaths = [], incognito = false, insecure = false,
     extraArgs = []
-  } = {}) {
+  } = {}, downloadTimeout = 0) {
     if (platform == "linux" || platform == "darwin")
-      await Edge.installBrowser(version);
+      await Edge.installBrowser(version, downloadTimeout);
 
     let driverPath = await Edge.#installDriver();
     let serviceBuilder = new edge.ServiceBuilder(driverPath);

package/src/utils.js

@@ -25,31 +25,43 @@ import got from "got";
 import dmg from "dmg";
 import Jimp from "jimp";
 
+
 /**
  * Downloads url resources.
  * @param {string} url - The url of the resource to be downloaded.
  * @param {string} destFile - The destination file path.
- * @throws {TypeError} Invalid URL.
+ * @param {number?} timeout - Allowed time in ms for the download to complete.
+ *   When set to 0 there is no time limit. Defaults to 0.
+ * @throws {TypeError} Invalid URL, Download timeout.
  */
-export async function download(url, destFile) {
+export async function download(url, destFile, timeout = 0) {
   let cacheDir = path.dirname(destFile);
-
   await fs.promises.mkdir(cacheDir, {recursive: true});
 
   let tempDest = `${destFile}-${process.pid}`;
   let writable = fs.createWriteStream(tempDest);
-
+  let timeoutID;
   try {
-    await promisify(pipeline)(got.stream(url), writable);
+    let downloading = promisify(pipeline)(got.stream(url), writable);
+    if (timeout == 0) {
+      await downloading;
+    }
+    else {
+      let timeoutPromise = new Promise((resolve, reject) => {
+        timeoutID = setTimeout(
+          () => reject(`Download timeout after ${timeout}ms`), timeout);
+      });
+      await Promise.race([downloading, timeoutPromise]);
+    }
   }
   catch (error) {
-    try {
-      await fs.promises.rm(tempDest, {recursive: true});
-    }
-    catch (e) {}
-
+    await fs.promises.rm(tempDest, {force: true});
     throw error;
   }
+  finally {
+    if (timeoutID)
+      clearTimeout(timeoutID);
+  }
 
   await fs.promises.rename(tempDest, destFile);
 }

package/test/browsers.js

@@ -104,21 +104,47 @@ function getExtension(browser, version) {
   return {extensionPaths, manifest};
 }
 
+async function getCachedTimes(browser) {
+  let cacheDir = path.join(snapshotsBaseDir, browser, "cache");
+  let cacheFiles = await fs.promises.readdir(cacheDir);
+
+  let browserCacheTime = null;
+  // Edge gets installed at OS level, that's why it's not tested here
+  if (browser != "edge") {
+    let installTypes = [".zip", ".dmg", ".bz2"];
+    let browserZip =
+      cacheFiles.find(elem => installTypes.some(type => elem.includes(type)));
+    let browserStat = await fs.promises.stat(path.join(cacheDir, browserZip));
+    browserCacheTime = browserStat.ctimeMs;
+  }
+
+  let driverCacheTime = null;
+  // Firefox install file includes the driver, that's why it's not tested here
+  if (browser != "firefox") {
+    let driverDir = cacheFiles.find(elem => !elem.endsWith(".zip"));
+    let driverFiles = await fs.promises.readdir(path.join(cacheDir, driverDir));
+    let driverZip = driverFiles.find(elem => elem.endsWith(".zip"));
+    let driverStat =
+      await fs.promises.stat(path.join(cacheDir, driverDir, driverZip));
+    driverCacheTime = driverStat.ctimeMs;
+  }
+
+  return {browserCTime: browserCacheTime, driverCTime: driverCacheTime};
+}
+
 for (let browser of Object.keys(BROWSERS)) {
   describe(`Browser: ${browser}`, () => {
     before(async() => {
       if (process.env.TEST_KEEP_SNAPSHOTS == "true")
         return;
 
-      try {
-        await fs.promises.rm(snapshotsBaseDir, {recursive: true});
-      }
-      catch (e) {}
+      await fs.promises.rm(snapshotsBaseDir, {force: true, recursive: true});
     });
 
     for (let version of VERSIONS[browser]) {
       describe(`Version: ${version}`, () => {
         let driver = null;
+        let emptyCacheTimes = null;
 
         async function quitDriver() {
           if (!driver)
@@ -140,7 +166,7 @@ for (let browser of Object.keys(BROWSERS)) {
             this.skip();
 
           let {binary, versionNumber} =
-            await BROWSERS[browser].installBrowser(version);
+            await BROWSERS[browser].installBrowser(version, 60000);
           let browserName = browser == "edge" ? /(edge|Edge)/ : browser;
           expect(binary).toEqual(expect.stringMatching(browserName));
 
@@ -165,6 +191,23 @@ for (let browser of Object.keys(BROWSERS)) {
 
           expect((await driver.getCapabilities()).getBrowserName())
             .toEqual(expect.stringMatching(names[browser]));
+
+          // When running all tests, this saves time on the cache test
+          emptyCacheTimes = await getCachedTimes(browser);
+        });
+
+        it("uses cached install files", async() => {
+          if (emptyCacheTimes == null) { // Single test case run
+            driver = await BROWSERS[browser].getDriver(version);
+            emptyCacheTimes = await getCachedTimes(browser);
+            await quitDriver();
+          }
+
+          // assigning `driver` to allow the afterEach hook quit the driver
+          driver = await BROWSERS[browser].getDriver(version);
+          let existingCacheTimes = await getCachedTimes(browser);
+
+          expect(existingCacheTimes).toEqual(emptyCacheTimes);
         });
 
         it("supports extra args", async() => {

package/test/utils.js

@@ -25,18 +25,34 @@ describe("Utils", () => {
   it("defines a browser snapshots folder", () => expect(snapshotsBaseDir)
     .toBe(path.join(process.cwd(), "browser-snapshots")));
 
+  let resourceUrl = "https://gitlab.com/eyeo/developer-experience/get-browser-binary/-/raw/main/package.json";
   let destFile = path.join(snapshotsBaseDir, "download-test.txt");
+  let expectedFileContents = "\"name\": \"@eyeo/get-browser-binary\"";
 
   it("downloads resources", async() => {
-    let url = "https://gitlab.com/eyeo/developer-experience/get-browser-binary/-/raw/main/package.json";
-    await download(url, destFile);
+    await fs.promises.rm(destFile, {force: true});
+    await download(resourceUrl, destFile);
     let contents = await fs.promises.readFile(destFile, {encoding: "utf8"});
-    expect(contents).toEqual(
-      expect.stringContaining("\"name\": \"@eyeo/get-browser-binary\""));
+    expect(contents).toEqual(expect.stringContaining(expectedFileContents));
   });
 
-  it("does not download invalid resources", async() => {
-    expect(download("invalid", destFile)).rejects
-      .toThrow("TypeError [ERR_INVALID_URL]: Invalid URL");
+  it("downloads resources on long timeout", async() => {
+    await fs.promises.rm(destFile, {force: true});
+    await download(resourceUrl, destFile, 10000);
+    let contents = await fs.promises.readFile(destFile, {encoding: "utf8"});
+    expect(contents).toEqual(expect.stringContaining(expectedFileContents));
+  });
+
+  it("does not download on short timeout", async() => {
+    try {
+      await download(resourceUrl, destFile, 10);
+      throw new Error("Download timeout didn't throw");
+    }
+    catch (err) {
+      expect(err).toBe("Download timeout after 10ms");
+    }
   });
+
+  it("does not download invalid resources", () =>
+    expect(download("invalid", destFile)).rejects.toThrow("Invalid URL"));
 });