@browserless.io/browserless 2.18.0 → 2.20.0-beta-1

package/build/sdk-utils.js

@@ -2,7 +2,15 @@ import { createInterface } from 'readline';
 import debug from 'debug';
 import fs from 'fs/promises';
 import path from 'path';
-import { spawn } from 'child_process';
+import { promisify } from 'util';
+import { exec } from 'child_process';
+const execAsync = promisify(exec);
+const waitForCommand = async (cmd, workingDirectory) => new Promise((resolve, reject) => execAsync(cmd, { cwd: workingDirectory }).then(({ stderr }) => {
+    if (stderr) {
+        return reject(`Error running ${cmd}. See output for more details: \n${stderr}`);
+    }
+    return resolve();
+}));
 export const getArgSwitches = () => {
     return process.argv.reduce((accum, arg, idx) => {
         if (!arg.startsWith('--')) {
@@ -28,7 +36,11 @@ export const getSourceFiles = async (cwd) => {
     const buildDir = path.join(cwd, 'build');
     const files = await fs.readdir(buildDir, { recursive: true });
     const [httpRoutes, webSocketRoutes] = files.reduce(([httpRoutes, webSocketRoutes], file) => {
+        const isInRootDir = !file.includes(path.sep);
         const parsed = path.parse(file);
+        if (isInRootDir) {
+            return [httpRoutes, webSocketRoutes];
+        }
         if (parsed.name.endsWith('http')) {
             httpRoutes.push(path.join(buildDir, file));
         }
@@ -58,50 +70,10 @@ export const prompt = async (question) => {
         });
     });
 };
+// Exceptions are not caught, since any error would result in a crash regardless
 export const installDependencies = async (workingDirectory) => {
-    await new Promise((resolve, reject) => {
-        spawn('npm', ['i'], {
-            cwd: workingDirectory,
-            stdio: 'inherit',
-        }).once('close', (code) => {
-            if (code === 0) {
-                return resolve();
-            }
-            return reject(`Error when installing dependencies, see output for more details`);
-        });
-    });
-    await new Promise((resolve, reject) => {
-        spawn('npx', 'playwright-core install --with-deps chromium firefox webkit'.split(' '), {
-            cwd: workingDirectory,
-            stdio: 'inherit',
-        }).once('close', (code) => {
-            if (code === 0) {
-                return resolve();
-            }
-            return reject(`Error when installing dependencies, see output for more details`);
-        });
-    });
+    await waitForCommand('npm install', workingDirectory);
+    await waitForCommand('npx playwright-core install --with-deps chromium firefox webkit', workingDirectory);
 };
-export const buildDockerImage = async (cmd, projectDir) => new Promise((resolve, reject) => {
-    const [docker, ...args] = cmd.split(' ');
-    spawn(docker, args, {
-        cwd: projectDir,
-        stdio: 'inherit',
-    }).once('close', (code) => {
-        if (code === 0) {
-            return resolve();
-        }
-        return reject(`Error when building Docker image, see output for more details`);
-    });
-});
-export const buildTypeScript = async (buildDir, projectDir) => new Promise((resolve, reject) => {
-    spawn('npx', ['tsc', '--outDir', buildDir], {
-        cwd: projectDir,
-        stdio: 'inherit',
-    }).once('close', (code) => {
-        if (code === 0) {
-            return resolve();
-        }
-        return reject(`Error in building TypeScript, see output for more details`);
-    });
-});
+export const buildDockerImage = async (cmd, projectDir) => waitForCommand(cmd, projectDir);
+export const buildTypeScript = async (buildDir, projectDir) => waitForCommand(`npx tsc --outDir ${buildDir}`, projectDir);

package/build/server.js

@@ -241,7 +241,9 @@ export class HTTPServer extends EventEmitter {
     async shutdown() {
         this.logger.info(`HTTP Server is shutting down`);
         await new Promise((r) => this.server.close(r));
-        this.server && this.server.removeAllListeners();
+        if (this.server) {
+            this.server.removeAllListeners();
+        }
         // @ts-ignore garbage collect this reference
         this.server = null;
         this.logger.info(`HTTP Server shutdown complete`);

package/build/shared/json-version.http.js

@@ -25,6 +25,7 @@ export default class ChromiumJSONVersionGetRoute extends HTTPRoute {
             return jsonResponse(res, 200, this.cachedJSON);
         }
         catch (err) {
+            logger.warn(`Error handling request`, err);
             return writeResponse(res, 500, 'There was an error handling your request', contentTypes.text);
         }
     }

package/build/shared/utils/performance/main.js

@@ -20,7 +20,8 @@ export default async ({ browser, context, logger, timeout, }) => {
                 return;
             if (pid)
                 process.kill(pid, 'SIGINT');
-            timeoutId && clearTimeout(timeoutId);
+            if (timeoutId)
+                clearTimeout(timeoutId);
             closed = true;
             timeoutId = null;
         };

package/build/utils.d.ts

@@ -17,7 +17,15 @@ export declare const buildDir: string;
 export declare const tsExtension = ".d.ts";
 export declare const jsonExtension = ".json";
 export declare const jsExtension = ".js";
+export declare const isWin: boolean;
 export declare const id: () => string;
+/**
+ * Normalizes a full-path by adding the `file://` protocol if needed.
+ *
+ * @param filepath - The file path to normalize.
+ * @returns The normalized file path.
+ */
+export declare const normalizeFileProtocol: (filepath: string) => string;
 /**
  * Generates a random, Chromium-compliant page ID with "BLESS"
  * prepended. This prepended text signals to other parts of the

package/build/utils.js

@@ -36,7 +36,22 @@ export const buildDir = path.join(path.resolve(), 'build');
 export const tsExtension = '.d.ts';
 export const jsonExtension = '.json';
 export const jsExtension = '.js';
+export const isWin = process.platform === 'win32';
 export const id = () => crypto.randomUUID();
+/**
+ * Normalizes a full-path by adding the `file://` protocol if needed.
+ *
+ * @param filepath - The file path to normalize.
+ * @returns The normalized file path.
+ */
+export const normalizeFileProtocol = (filepath) => {
+    if (isWin) {
+        if (filepath.startsWith('file:///'))
+            return filepath;
+        return 'file:///' + filepath;
+    }
+    return filepath;
+};
 /**
  * Generates a random, Chromium-compliant page ID with "BLESS"
  * prepended. This prepended text signals to other parts of the

package/docker/chrome/Dockerfile

@@ -26,8 +26,9 @@ RUN echo "ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula sele
     fonts-ubuntu \
     fonts-wqy-zenhei
 
-RUN npx --yes playwright install chrome &&\
-  npx --yes playwright install-deps chrome &&\
+# NOTE it's important to not use npx playwright-core here since it'll likely install
+# a more recent version than we potentially have in our own package.json
+RUN ./node_modules/playwright-core/cli.js install --with-deps chrome &&\
   npm run build &&\
   npm run build:function &&\
   npm prune production &&\

package/docker/chromium/Dockerfile

@@ -26,8 +26,9 @@ RUN echo "ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula sele
     fonts-ubuntu \
     fonts-wqy-zenhei
 
-RUN npx --yes playwright install chromium &&\
-  npx --yes playwright install-deps chromium &&\
+# NOTE it's important to not use npx playwright-core here since it'll likely install
+# a more recent version than we potentially have in our own package.json
+RUN ./node_modules/playwright-core/cli.js install --with-deps chromium &&\
   npm run build &&\
   npm run build:function &&\
   npm prune production &&\

package/docker/firefox/Dockerfile

@@ -26,8 +26,9 @@ RUN echo "ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula sele
     fonts-ubuntu \
     fonts-wqy-zenhei
 
-RUN npx --yes playwright install firefox &&\
-  npx --yes playwright install-deps firefox &&\
+# NOTE it's important to not use npx playwright-core here since it'll likely install
+# a more recent version than we potentially have in our own package.json
+RUN ./node_modules/playwright-core/cli.js install --with-deps firefox &&\
   npm run build &&\
   npm prune production &&\
   chown -R blessuser:blessuser $APP_DIR &&\

package/docker/multi/Dockerfile

@@ -29,12 +29,14 @@ RUN echo "ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula sele
 # Chrome stable is only supported on non-ARM builds
 # so it's installation is conditional on whether or not amd64
 RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then \
-    ./node_modules/.bin/playwright-core install --with-deps chrome; \
+    ./node_modules/playwright-core/cli.js install --with-deps chrome; \
   else \
     rm -rf ./src/routes/chrome; \
   fi
 
-RUN npx --yes playwright install --with-deps chromium firefox webkit &&\
+# NOTE it's important to not use npx playwright-core here since it'll likely install
+# a more recent version than we potentially have in our own package.json
+RUN ./node_modules/playwright-core/cli.js install --with-deps chromium firefox webkit &&\
   npm run build &&\
   npm run build:function &&\
   npm prune production &&\

package/docker/webkit/Dockerfile

@@ -27,8 +27,9 @@ RUN echo "ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula sele
     fonts-ubuntu \
     fonts-wqy-zenhei
 
-RUN npx --yes playwright install webkit &&\
-  npx --yes playwright install-deps webkit &&\
+# NOTE it's important to not use npx playwright-core here since it'll likely install
+# a more recent version than we potentially have in our own package.json
+RUN ./node_modules/playwright-core/cli.js install --with-deps webkit &&\
   npm run build &&\
   npm prune production &&\
   chown -R blessuser:blessuser $APP_DIR &&\

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@browserless.io/browserless",
-  "version": "2.18.0",
+  "version": "2.20.0-beta-1",
   "license": "SSPL",
   "description": "The browserless platform",
   "author": "browserless.io",
@@ -25,9 +25,9 @@
     "dev": "npm run build:dev && env-cmd -f .env node build",
     "install:adblock": "node scripts/install-adblock.js",
     "install:debugger": "node scripts/install-debugger.js",
-    "install:browsers": "npx --yes playwright install chromium firefox webkit chrome",
+    "install:browsers": "npx --no playwright-core install chromium firefox webkit",
     "install:dev": "npm run install:browsers && npm run install:debugger",
-    "lint": "eslint . --ext .ts --fix",
+    "lint": "cross-env ESLINT_USE_FLAT_CONFIG=false eslint . --ext .ts --fix",
     "prepack": "npm run build:dev",
     "prettier": "prettier '{src,functions,scripts,bin,external,.github}/**/*.{js,ts,json,yml,yaml,md}' --log-level error --write",
     "test": "cross-env DEBUG=quiet mocha",
@@ -48,21 +48,21 @@
     "tsconfig.json"
   ],
   "dependencies": {
-    "debug": "^4.3.6",
+    "debug": "^4.3.7",
     "del": "^7.0.0",
     "enjoi": "^9.0.1",
-    "file-type": "^19.4.1",
+    "file-type": "^19.5.0",
     "get-port": "^7.1.0",
     "gradient-string": "^2.0.0",
     "http-proxy": "^1.18.1",
-    "lighthouse": "^12.2.0",
-    "micromatch": "^4.0.7",
+    "lighthouse": "^12.2.1",
+    "micromatch": "^4.0.8",
     "playwright-1.41": "npm:playwright-core@1.41.2",
     "playwright-1.42": "npm:playwright-core@1.42.1",
     "playwright-1.43": "npm:playwright-core@1.43.1",
     "playwright-1.44": "npm:playwright-core@1.44.1",
-    "playwright-core": "^1.46.1",
-    "puppeteer-core": "^23.1.1",
+    "playwright-core": "^1.47.0",
+    "puppeteer-core": "^23.3.0",
     "puppeteer-extra": "^3.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2",
     "queue": "^7.0.0",
@@ -70,33 +70,32 @@
     "tar-fs": "^3.0.6"
   },
   "optionalDependencies": {
-    "@types/chai": "^4.3.17",
+    "@types/chai": "^4.3.18",
     "@types/debug": "^4.1.12",
     "@types/gradient-string": "^1.1.6",
     "@types/http-proxy": "^1.17.15",
     "@types/micromatch": "^4.0.9",
     "@types/mocha": "^10.0.7",
-    "@types/node": "^22.5.0",
+    "@types/node": "^22.5.4",
     "@types/sinon": "^17.0.3",
-    "@typescript-eslint/eslint-plugin": "^7.18.0",
-    "@typescript-eslint/parser": "^7.18.0",
+    "@typescript-eslint/eslint-plugin": "^8.5.0",
+    "@typescript-eslint/parser": "^8.5.0",
     "assert": "^2.0.0",
     "chai": "^5.1.1",
     "cross-env": "^7.0.3",
     "env-cmd": "^10.1.0",
     "esbuild": "^0.23.1",
     "esbuild-plugin-polyfill-node": "^0.3.0",
-    "eslint": "^8.57.0",
-    "eslint-plugin-typescript-sort-keys": "^3.2.0",
+    "eslint": "^9.10.0",
     "extract-zip": "^2.0.1",
     "gunzip-maybe": "^1.4.2",
-    "marked": "^14.0.0",
+    "marked": "^14.1.2",
     "mocha": "^10.7.3",
     "move-file": "^3.1.0",
     "prettier": "^3.3.3",
     "sinon": "^18.0.0",
     "ts-node": "^10.9.2",
-    "typescript": "^5.5.4",
+    "typescript": "^5.6.2",
     "typescript-json-schema": "^0.65.1"
   },
   "playwrightVersions": {
@@ -111,32 +110,23 @@
     "root": true,
     "parser": "@typescript-eslint/parser",
     "plugins": [
-      "@typescript-eslint",
-      "typescript-sort-keys"
+      "@typescript-eslint"
     ],
     "extends": [
       "eslint:recommended",
       "plugin:@typescript-eslint/eslint-recommended",
-      "plugin:@typescript-eslint/recommended",
-      "plugin:typescript-sort-keys/recommended"
+      "plugin:@typescript-eslint/recommended"
     ],
     "ignorePatterns": [
       "node_modules/*",
+      "static/*",
+      "**.spec.ts",
       "build/*",
       ".DS_Store",
       ".no-git/*",
       "*.log"
     ],
     "rules": {
-      "sort-keys": [
-        "error",
-        "asc",
-        {
-          "caseSensitive": true,
-          "natural": false,
-          "minKeys": 2
-        }
-      ],
       "semi": [
         2,
         "always"

package/src/browserless.ts

@@ -25,8 +25,10 @@ import {
   WebKitPlaywright,
   WebSocketRoute,
   availableBrowsers,
+  dedent,
   getRouteFiles,
   makeExternalURL,
+  normalizeFileProtocol,
   printLogo,
   safeParse,
 } from '@browserless.io/browserless';
@@ -313,9 +315,7 @@ export class Browserless extends EventEmitter {
           }),
         );
 
-        const wsImport = `${
-          this.config.getIsWin() ? 'file:///' : ''
-        }${wsRoute}`;
+        const wsImport = normalizeFileProtocol(wsRoute);
         const {
           default: Route,
         }: {
@@ -357,7 +357,8 @@ export class Browserless extends EventEmitter {
         !installedBrowsers.some((b) => b.name === route.browser?.name)
       ) {
         throw new Error(
-          `Couldn't load route "${route.path}" due to missing browser binary for "${route.browser?.name}"`,
+          dedent(`Couldn't load route "${route.path}" due to missing browser binary for "${route.browser?.name}".
+            Installed Browsers: ${installedBrowsers.join(', ')}`),
         );
       }
     });

package/src/browsers/index.ts

@@ -255,12 +255,16 @@ export class BrowserManager {
         },
       });
       if (response.ok) {
+        const externalUrl = new URL(serverAddress);
+        const protocol = externalUrl.protocol === 'https:' ? 'wss' : 'ws';
         const body = await response.json();
         for (const page of body) {
+          const devtoolsFrontendUrl = `/devtools/inspector.html?${protocol}=${externalUrl.host}${externalUrl.pathname}/devtools/page/${page.id}`;
           sessions.push({
             ...sessions[0],
             ...page,
             browserWSEndpoint: wsEndpoint,
+            devtoolsFrontendUrl,
           });
         }
       }
@@ -299,8 +303,8 @@ export class BrowserManager {
         global.setTimeout(() => {
           const session = this.browsers.get(browser);
           if (session) {
-            this.log.trace(`Timer hit for "${session.id}"`),
-              this.close(browser, session);
+            this.log.trace(`Timer hit for "${session.id}"`);
+            this.close(browser, session);
           }
         }, timeout),
       );

package/src/config.ts

@@ -1,4 +1,4 @@
-import { exists, keyLength, untildify } from '@browserless.io/browserless';
+import { exists, isWin as isWindows, keyLength, untildify } from '@browserless.io/browserless';
 import { EventEmitter } from 'events';
 import debug from 'debug';
 import { fileURLToPath } from 'url';
@@ -112,8 +112,8 @@ const getDebug = () => {
 export class Config extends EventEmitter {
   protected readonly debug = getDebug();
   protected readonly host = process.env.HOST ?? 'localhost';
+  protected readonly isWin = isWindows;
   protected external = process.env.PROXY_URL ?? process.env.EXTERNAL;
-  protected readonly isWin = process.platform === 'win32';
 
   protected port = +(process.env.PORT ?? '3000');
 

package/src/limiter.spec.ts

@@ -1,3 +1,4 @@
+/* eslint-disable no-unused-expressions */
 import {
   Config,
   Hooks,

package/src/routes/chrome/tests/content.spec.ts

@@ -1,3 +1,4 @@
+/* eslint-disable no-unused-expressions */
 import {
   Browserless,
   Config,

package/src/routes/chrome/ws/page.ts

@@ -4,6 +4,7 @@ import { default as Page, QuerySchema } from '../../../shared/page.ws.js';
 export default class ChromePageWebSocketRoute extends Page {
   name = BrowserlessRoutes.ChromePageWebSocketRoute;
   browser = ChromeCDP;
+  auth = false;
 }
 
 export { QuerySchema };

package/src/sdk-utils.ts

@@ -2,7 +2,24 @@ import { createInterface } from 'readline';
 import debug from 'debug';
 import fs from 'fs/promises';
 import path from 'path';
-import { spawn } from 'child_process';
+import { promisify } from 'util';
+
+import { exec } from 'child_process';
+
+const execAsync = promisify(exec);
+
+const waitForCommand = async (cmd: string, workingDirectory: string) =>
+  new Promise<void>((resolve, reject) =>
+    execAsync(cmd, { cwd: workingDirectory }).then(({ stderr }) => {
+      if (stderr) {
+        return reject(
+          `Error running ${cmd}. See output for more details: \n${stderr}`,
+        );
+      }
+
+      return resolve();
+    }),
+  );
 
 export const getArgSwitches = () => {
   return process.argv.reduce(
@@ -41,7 +58,13 @@ export const getSourceFiles = async (cwd: string) => {
   const files = await fs.readdir(buildDir, { recursive: true });
   const [httpRoutes, webSocketRoutes] = files.reduce(
     ([httpRoutes, webSocketRoutes], file) => {
+      const isInRootDir = !file.includes(path.sep);
       const parsed = path.parse(file);
+
+      if (isInRootDir) {
+        return [httpRoutes, webSocketRoutes];
+      }
+
       if (parsed.name.endsWith('http')) {
         httpRoutes.push(path.join(buildDir, file));
       }
@@ -81,74 +104,23 @@ export const prompt = async (question: string) => {
   });
 };
 
+// Exceptions are not caught, since any error would result in a crash regardless
 export const installDependencies = async (
   workingDirectory: string,
 ): Promise<void> => {
-  await new Promise<void>((resolve, reject) => {
-    spawn('npm', ['i'], {
-      cwd: workingDirectory,
-      stdio: 'inherit',
-    }).once('close', (code) => {
-      if (code === 0) {
-        return resolve();
-      }
-      return reject(
-        `Error when installing dependencies, see output for more details`,
-      );
-    });
-  });
-  await new Promise<void>((resolve, reject) => {
-    spawn(
-      'npx',
-      'playwright-core install --with-deps chromium firefox webkit'.split(' '),
-      {
-        cwd: workingDirectory,
-        stdio: 'inherit',
-      },
-    ).once('close', (code) => {
-      if (code === 0) {
-        return resolve();
-      }
-      return reject(
-        `Error when installing dependencies, see output for more details`,
-      );
-    });
-  });
+  await waitForCommand('npm install', workingDirectory);
+  await waitForCommand(
+    'npx playwright-core install --with-deps chromium firefox webkit',
+    workingDirectory,
+  );
 };
 
 export const buildDockerImage = async (
   cmd: string,
   projectDir: string,
-): Promise<void> =>
-  new Promise((resolve, reject) => {
-    const [docker, ...args] = cmd.split(' ');
-    spawn(docker, args, {
-      cwd: projectDir,
-      stdio: 'inherit',
-    }).once('close', (code) => {
-      if (code === 0) {
-        return resolve();
-      }
-      return reject(
-        `Error when building Docker image, see output for more details`,
-      );
-    });
-  });
+): Promise<void> => waitForCommand(cmd, projectDir);
 
 export const buildTypeScript = async (
   buildDir: string,
   projectDir: string,
-): Promise<void> =>
-  new Promise((resolve, reject) => {
-    spawn('npx', ['tsc', '--outDir', buildDir], {
-      cwd: projectDir,
-      stdio: 'inherit',
-    }).once('close', (code) => {
-      if (code === 0) {
-        return resolve();
-      }
-      return reject(
-        `Error in building TypeScript, see output for more details`,
-      );
-    });
-  });
+): Promise<void> => waitForCommand(`npx tsc --outDir ${buildDir}`, projectDir);

package/src/server.ts

@@ -415,7 +415,10 @@ export class HTTPServer extends EventEmitter {
   public async shutdown(): Promise<void> {
     this.logger.info(`HTTP Server is shutting down`);
     await new Promise((r) => this.server.close(r));
-    this.server && this.server.removeAllListeners();
+
+    if (this.server) {
+      this.server.removeAllListeners();
+    }
 
     // @ts-ignore garbage collect this reference
     this.server = null;

package/src/shared/json-version.http.ts

@@ -45,6 +45,7 @@ export default class ChromiumJSONVersionGetRoute extends HTTPRoute {
       }
       return jsonResponse(res, 200, this.cachedJSON);
     } catch (err) {
+      logger.warn(`Error handling request`, err);
       return writeResponse(
         res,
         500,

package/src/shared/utils/performance/main.ts

@@ -38,7 +38,7 @@ export default async ({
     const close = (pid?: number) => {
       if (closed) return;
       if (pid) process.kill(pid, 'SIGINT');
-      timeoutId && clearTimeout(timeoutId);
+      if (timeoutId) clearTimeout(timeoutId);
       closed = true;
       timeoutId = null;
     };

package/src/utils.ts

@@ -64,9 +64,26 @@ export const buildDir: string = path.join(path.resolve(), 'build');
 export const tsExtension = '.d.ts';
 export const jsonExtension = '.json';
 export const jsExtension = '.js';
+export const isWin = process.platform === 'win32';
 
 export const id = (): string => crypto.randomUUID();
 
+/**
+ * Normalizes a full-path by adding the `file://` protocol if needed.
+ * 
+ * @param filepath - The file path to normalize.
+ * @returns The normalized file path.
+ */
+export const normalizeFileProtocol = (filepath: string) => {
+  if (isWin) {
+    if (filepath.startsWith('file:///')) return filepath;
+
+    return 'file:///' + filepath;
+  }
+
+  return filepath;
+};
+
 /**
  * Generates a random, Chromium-compliant page ID with "BLESS"
  * prepended. This prepended text signals to other parts of the