@browserless.io/browserless 2.15.0 → 2.16.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@browserless.io/browserless",
-  "version": "2.15.0",
+  "version": "2.16.0",
   "license": "SSPL",
   "description": "The browserless platform",
   "author": "browserless.io",
@@ -51,7 +51,7 @@
     "debug": "^4.3.5",
     "del": "^7.0.0",
     "enjoi": "^9.0.1",
-    "file-type": "^19.0.0",
+    "file-type": "^19.1.1",
     "get-port": "^7.1.0",
     "gradient-string": "^2.0.0",
     "http-proxy": "^1.18.1",
@@ -62,7 +62,7 @@
     "playwright-1.43": "npm:playwright-core@1.43.1",
     "playwright-1.44": "npm:playwright-core@1.44.1",
     "playwright-core": "^1.45.1",
-    "puppeteer-core": "^22.12.1",
+    "puppeteer-core": "^22.13.0",
     "puppeteer-extra": "^3.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2",
     "queue": "^7.0.0",
@@ -76,10 +76,10 @@
     "@types/http-proxy": "^1.17.14",
     "@types/micromatch": "^4.0.9",
     "@types/mocha": "^10.0.7",
-    "@types/node": "^20.14.9",
+    "@types/node": "^20.14.10",
     "@types/sinon": "^17.0.3",
-    "@typescript-eslint/eslint-plugin": "^7.15.0",
-    "@typescript-eslint/parser": "^7.15.0",
+    "@typescript-eslint/eslint-plugin": "^7.16.0",
+    "@typescript-eslint/parser": "^7.16.0",
     "assert": "^2.0.0",
     "chai": "^5.1.1",
     "cross-env": "^7.0.3",
@@ -90,10 +90,10 @@
     "eslint-plugin-typescript-sort-keys": "^3.2.0",
     "extract-zip": "^2.0.1",
     "gunzip-maybe": "^1.4.2",
-    "marked": "^13.0.1",
+    "marked": "^13.0.2",
     "mocha": "^10.6.0",
     "move-file": "^3.1.0",
-    "prettier": "^3.3.2",
+    "prettier": "^3.3.3",
     "sinon": "^18.0.0",
     "ts-node": "^10.9.2",
     "typescript": "^5.5.3",

package/scripts/build-schemas.js

@@ -12,7 +12,7 @@ const moduleMain = path.normalize(import.meta.url).endsWith(process.argv[1]);
 
 /**
  * Find an exported interface in a TypeScript AST
- * 
+ *
  * @param {ts.Node} node The node to search for the exported interface
  * @param {string} interfaceName The name of the interface to search for
  * @returns {ts.InterfaceDeclaration | ts.Identifier | null}
@@ -29,7 +29,11 @@ const findExportedInterface = (node, interfaceName) => {
   }
 
   // Check for re-exported interfaces
-  if (ts.isExportDeclaration(node) && node.exportClause && ts.isNamedExports(node.exportClause)) {
+  if (
+    ts.isExportDeclaration(node) &&
+    node.exportClause &&
+    ts.isNamedExports(node.exportClause)
+  ) {
     const elements = node.exportClause.elements;
     for (const element of elements) {
       if (element.name.text === interfaceName) {
@@ -50,7 +54,7 @@ const findExportedInterface = (node, interfaceName) => {
 
 /**
  * Creates an standard JSON schema file for each route (see https://json-schema.org/specification)
- * 
+ *
  * @param {string[]} externalHTTPRoutes Additional HTTP routes to parse
  * @param {string[]} externalWebSocketRoutes Additional WS routes to parse
  */

package/src/browsers/browsers.cdp.ts

@@ -8,12 +8,12 @@ import {
   chromeExecutablePath,
   noop,
   once,
+  ublockPath,
 } from '@browserless.io/browserless';
 import puppeteer, { Browser, Page, Target } from 'puppeteer-core';
 import { Duplex } from 'stream';
 import { EventEmitter } from 'events';
 import StealthPlugin from 'puppeteer-extra-plugin-stealth';
-import { fileURLToPath } from 'url';
 import getPort from 'get-port';
 import httpProxy from 'http-proxy';
 import path from 'path';
@@ -170,11 +170,29 @@ export class ChromiumCDP extends EventEmitter {
     return this.browser?.process() || null;
   }
 
-  public async launch(laucherOpts: BrowserLauncherOptions): Promise<Browser> {
-    const { options, stealth } = laucherOpts;
+  public async launch({
+    options,
+    stealth,
+  }: BrowserLauncherOptions): Promise<Browser> {
     this.port = await getPort();
     this.logger.info(`${this.constructor.name} got open port ${this.port}`);
-    const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+    const extensionLaunchArgs = options.args?.find((a) =>
+      a.startsWith('--load-extension'),
+    );
+
+    // Remove extension flags as we recompile them below with our own
+    options.args = options.args?.filter(
+      (a) =>
+        !a.startsWith('--load-extension') &&
+        !a.startsWith('--disable-extensions-except'),
+    );
+
+    const extensions = [
+      this.blockAds ? ublockPath : null,
+      extensionLaunchArgs ? extensionLaunchArgs.split('=')[1] : null,
+    ].filter((_) => !!_);
+
     const finalOptions = {
       ...options,
       args: [
@@ -186,21 +204,10 @@ export class ChromiumCDP extends EventEmitter {
       executablePath: this.executablePath,
     };
 
-    if (this.blockAds) {
-      // Necessary to load extensions
-      finalOptions.headless = false;
-
-      const loadExtensionPaths: string = path.join(
-        __dirname,
-        '..',
-        '..',
-        'extensions',
-        'ublock',
-      );
-
+    if (extensions.length) {
       finalOptions.args.push(
-        '--load-extension=' + loadExtensionPaths,
-        '--disable-extensions-except=' + loadExtensionPaths,
+        '--load-extension=' + extensions.join(','),
+        '--disable-extensions-except=' + extensions.join(','),
       );
     }
 

package/src/browsers/browsers.playwright.ts

@@ -27,8 +27,10 @@ class BasePlaywright extends EventEmitter {
   protected proxy = httpProxy.createProxyServer();
   protected browser: playwright.BrowserServer | null = null;
   protected browserWSEndpoint: string | null = null;
-  protected playwrightBrowserType: PlaywrightBrowserTypes = PlaywrightBrowserTypes.chromium;
-  protected executablePath = () => playwright[this.playwrightBrowserType].executablePath();
+  protected playwrightBrowserType: PlaywrightBrowserTypes =
+    PlaywrightBrowserTypes.chromium;
+  protected executablePath = () =>
+    playwright[this.playwrightBrowserType].executablePath();
 
   constructor({
     config,
@@ -107,17 +109,22 @@ class BasePlaywright extends EventEmitter {
         `${this.constructor.name} hasn't been launched yet!`,
       );
     }
-    const browser = await playwright[this.playwrightBrowserType].connect(this.browserWSEndpoint);
+    const browser = await playwright[this.playwrightBrowserType].connect(
+      this.browserWSEndpoint,
+    );
     return await browser.newPage();
   }
 
-  public async launch(laucherOpts: BrowserLauncherOptions): Promise<playwright.BrowserServer> {
+  public async launch(
+    laucherOpts: BrowserLauncherOptions,
+  ): Promise<playwright.BrowserServer> {
     const { options, pwVersion } = laucherOpts;
     this.logger.info(`Launching ${this.constructor.name} Handler`);
 
     const opts = this.makeLaunchOptions(options);
     const versionedPw = await this.config.loadPwVersion(pwVersion!);
-    const browser = await versionedPw[this.playwrightBrowserType].launchServer(opts);
+    const browser =
+      await versionedPw[this.playwrightBrowserType].launchServer(opts);
     const browserWSEndpoint = browser.wsEndpoint();
 
     this.logger.info(

package/src/browsers/index.ts

@@ -535,7 +535,7 @@ export class BrowserManager {
       options: launchOptions as BrowserServerOptions,
       pwVersion,
       req,
-      stealth: launchOptions?.stealth
+      stealth: launchOptions?.stealth,
     });
     await this.hooks.browser({ browser, req });
 

package/src/limiter.spec.ts

@@ -22,6 +22,7 @@ describe(`Limiter`, () => {
     webHooks.callRejectAlertURL.resetHistory();
     webHooks.callTimeoutAlertURL.resetHistory();
     webHooks.callErrorAlertURL.resetHistory();
+
     hooks.before.resetHistory();
     hooks.after.resetHistory();
     hooks.browser.resetHistory();
@@ -119,6 +120,32 @@ describe(`Limiter`, () => {
     expect(handlerTwo.calledOnce).to.be.true;
   });
 
+  it('continues to process jobs even if an earlier job errors', (d) => {
+    const config = new Config();
+    const monitoring = new Monitoring(config);
+    const metrics = new Metrics();
+
+    config.setConcurrent(1);
+    config.setQueued(1);
+    config.setTimeout(-1);
+
+    const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
+    const errorJob = () =>
+      Promise.reject(new Error('Danger, danger. High voltage!'));
+    const okJob = spy();
+
+    const jobOne = limiter.limit(errorJob, asyncNoop, asyncNoop, noop);
+    const jobTwo = limiter.limit(okJob, asyncNoop, asyncNoop, noop);
+
+    jobOne();
+    jobTwo();
+
+    limiter.addEventListener('end', () => {
+      expect(okJob.calledOnce).to.be.true;
+      d(undefined);
+    });
+  });
+
   it('bubbles up errors', async () => {
     const config = new Config();
     const monitoring = new Monitoring(config);

package/src/logger.ts

@@ -23,7 +23,7 @@ export class Logger {
   }
 
   protected get reqInfo() {
-    return this.request ? this.request.socket.remoteAddress ?? 'Unknown' : '';
+    return this.request ? (this.request.socket.remoteAddress ?? 'Unknown') : '';
   }
 
   public trace(...messages: unknown[]) {

package/src/utils.ts

@@ -23,6 +23,7 @@ import { Page } from 'puppeteer-core';
 import { ServerResponse } from 'http';
 import crypto from 'crypto';
 import debug from 'debug';
+import { fileURLToPath } from 'url';
 import gradient from 'gradient-string';
 import { homedir } from 'os';
 import path from 'path';
@@ -33,6 +34,8 @@ const isHTTP = (
   return (writeable as ServerResponse).writeHead !== undefined;
 };
 
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
 const getAuthHeaderToken = (header: string) => {
   if (header.startsWith('Basic')) {
     const username = header.split(/\s+/).pop() || '';
@@ -475,10 +478,11 @@ export const queryParamsToObject = (
 ): Record<string, unknown> =>
   [...params.entries()].reduce(
     (accum, [key, value]) => {
-      accum[key] = value;
+      accum[key] =
+        value === '' || value === undefined || value === null ? true : value;
       return accum;
     },
-    {} as Record<string, string>,
+    {} as ReturnType<typeof queryParamsToObject>,
   );
 
 // eslint-disable-next-line @typescript-eslint/no-empty-function
@@ -844,3 +848,5 @@ export const getCDPClient = (page: Page): CDPSession => {
 
   return typeof c === 'function' ? c.call(page) : c;
 };
+
+export const ublockPath = path.join(__dirname, '..', 'extensions', 'ublock');