@browserless.io/browserless 2.15.0 → 2.16.0

package/CHANGELOG.md

@@ -1,6 +1,12 @@
-# [Latest](https://github.com/browserless/chrome/compare/v2.15.0...main)
+# [Latest](https://github.com/browserless/chrome/compare/v2.16.0...main)
 - Dependency updates.
 
+# [v2.16.0](https://github.com/browserless/chrome/compare/v2.15.0...v2.16.0)
+- Dependency updates.
+- Better extension handling and merging when passing in custom extensions.
+- Fixes a bug where a failed job (error thrown) causes the internal queue to drain.
+- Prettier and other source improvements.
+
 # [v2.15.0](https://github.com/browserless/chrome/compare/v2.14.0...v2.15.0)
 - Bug fix to avoid removing healthy jobs from limiter when a job fails
 - Dependency updates.

package/README.md

@@ -39,7 +39,6 @@ If you've been struggling to deploy headless browsers without running into issue
 1. [Full documentation site](https://docs.browserless.io/)
 2. [Live Debugger (using browserless.io)](https://chrome.browserless.io/)
 3. [Docker](https://github.com/browserless/browserless/pkgs/container/base)
-4. [Slack](https://join.slack.com/t/browserless/shared_invite/enQtMzA3OTMwNjA3MzY1LTRmMWU5NjQ0MTQ2YTE2YmU3MzdjNmVlMmU4MThjM2UxODNmNzNlZjVkY2U2NjdkMzYyNTgyZTBiMmE3Nzg0MzY)
 
 ## Features
 
@@ -54,14 +53,16 @@ If you've been struggling to deploy headless browsers without running into issue
 - Error tolerant: if Chrome dies it won't.
 - Support for running and development on Apple's M1 machines
 
-### Cloud
+### Cloud-only
 
 Our [cloud accounts](https://www.browserless.io/pricing/) include all the general features plus extras, such as:
 
-- Inbuilt [residential proxy](https://www.browserless.io/blog/2023/09/24/residential-proxying/)
-- [/unblock API](https://www.browserless.io/blog/2024/02/26/unblock-api/) for avoiding detectors
-- [Hybrid automations](https://www.browserless.io/blog/2024/03/21/hybrid-automations-for-puppeteer/) for streaming login windows during scripts
-- Ability to upload and run extensions _(coming soon)_
+- Inbuilt [residential proxy](https://www.browserless.io/blog/residential-proxying/)
+- [/unblock API](https://www.browserless.io/blog/unblock-api) for avoiding detectors
+- [Automated captcha solving](https://www.browserless.io/blog/captcha-solving) for getting past mandatory checks
+- [Hybrid automations](https://www.browserless.io/blog/hybrid-automations-for-puppeteer/) for streaming login windows during scripts
+- [/reconnect API](https://www.browserless.io/blog/reconnect-api) for keeping browsers alive for reuse
+- [REST APIs](https://www.browserless.io/feature/rest-apis) for tasks such as retrieving HTML, PDFs or Lighthouse metrics
 - SSO, tokens and user roles
 
 ## How it works

package/build/browsers/browsers.cdp.d.ts

@@ -29,7 +29,7 @@ export declare class ChromiumCDP extends EventEmitter {
     close(): Promise<void>;
     pages(): Promise<Page[]>;
     process(): import("child_process").ChildProcess | null;
-    launch(laucherOpts: BrowserLauncherOptions): Promise<Browser>;
+    launch({ options, stealth, }: BrowserLauncherOptions): Promise<Browser>;
     wsEndpoint(): string | null;
     publicWSEndpoint(token: string | null): string | null;
     proxyPageWebSocket(req: Request, socket: Duplex, head: Buffer): Promise<void>;

package/build/browsers/browsers.cdp.js

@@ -1,8 +1,7 @@
-import { BLESS_PAGE_IDENTIFIER, ServerError, chromeExecutablePath, noop, once, } from '@browserless.io/browserless';
+import { BLESS_PAGE_IDENTIFIER, ServerError, chromeExecutablePath, noop, once, ublockPath, } from '@browserless.io/browserless';
 import puppeteer from 'puppeteer-core';
 import { EventEmitter } from 'events';
 import StealthPlugin from 'puppeteer-extra-plugin-stealth';
-import { fileURLToPath } from 'url';
 import getPort from 'get-port';
 import httpProxy from 'http-proxy';
 import path from 'path';
@@ -111,11 +110,17 @@ export class ChromiumCDP extends EventEmitter {
     process() {
         return this.browser?.process() || null;
     }
-    async launch(laucherOpts) {
-        const { options, stealth } = laucherOpts;
+    async launch({ options, stealth, }) {
         this.port = await getPort();
         this.logger.info(`${this.constructor.name} got open port ${this.port}`);
-        const __dirname = path.dirname(fileURLToPath(import.meta.url));
+        const extensionLaunchArgs = options.args?.find((a) => a.startsWith('--load-extension'));
+        // Remove extension flags as we recompile them below with our own
+        options.args = options.args?.filter((a) => !a.startsWith('--load-extension') &&
+            !a.startsWith('--disable-extensions-except'));
+        const extensions = [
+            this.blockAds ? ublockPath : null,
+            extensionLaunchArgs ? extensionLaunchArgs.split('=')[1] : null,
+        ].filter((_) => !!_);
         const finalOptions = {
             ...options,
             args: [
@@ -126,11 +131,8 @@ export class ChromiumCDP extends EventEmitter {
             ].filter((_) => !!_),
             executablePath: this.executablePath,
         };
-        if (this.blockAds) {
-            // Necessary to load extensions
-            finalOptions.headless = false;
-            const loadExtensionPaths = path.join(__dirname, '..', '..', 'extensions', 'ublock');
-            finalOptions.args.push('--load-extension=' + loadExtensionPaths, '--disable-extensions-except=' + loadExtensionPaths);
+        if (extensions.length) {
+            finalOptions.args.push('--load-extension=' + extensions.join(','), '--disable-extensions-except=' + extensions.join(','));
         }
         const launch = stealth
             ? puppeteerStealth.launch.bind(puppeteerStealth)

package/build/browsers/index.js

@@ -353,7 +353,7 @@ export class BrowserManager {
             options: launchOptions,
             pwVersion,
             req,
-            stealth: launchOptions?.stealth
+            stealth: launchOptions?.stealth,
         });
         await this.hooks.browser({ browser, req });
         const session = {

package/build/limiter.spec.js

@@ -91,6 +91,25 @@ describe(`Limiter`, () => {
         await wait;
         expect(handlerTwo.calledOnce).to.be.true;
     });
+    it('continues to process jobs even if an earlier job errors', (d) => {
+        const config = new Config();
+        const monitoring = new Monitoring(config);
+        const metrics = new Metrics();
+        config.setConcurrent(1);
+        config.setQueued(1);
+        config.setTimeout(-1);
+        const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
+        const errorJob = () => Promise.reject(new Error('Danger, danger. High voltage!'));
+        const okJob = spy();
+        const jobOne = limiter.limit(errorJob, asyncNoop, asyncNoop, noop);
+        const jobTwo = limiter.limit(okJob, asyncNoop, asyncNoop, noop);
+        jobOne();
+        jobTwo();
+        limiter.addEventListener('end', () => {
+            expect(okJob.calledOnce).to.be.true;
+            d(undefined);
+        });
+    });
     it('bubbles up errors', async () => {
         const config = new Config();
         const monitoring = new Monitoring(config);

package/build/logger.js

@@ -20,7 +20,7 @@ export class Logger {
         this._fatal = logger.extend('fatal');
     }
     get reqInfo() {
-        return this.request ? this.request.socket.remoteAddress ?? 'Unknown' : '';
+        return this.request ? (this.request.socket.remoteAddress ?? 'Unknown') : '';
     }
     trace(...messages) {
         this._trace(this.reqInfo, ...messages);

package/build/routes/chrome/http/content.post.body.json

@@ -408,14 +408,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@32994": {
+                "__@toStringTag@11061": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@32994",
+                "__@toStringTag@11061",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -450,13 +450,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@32994": {
+        "__@toStringTag@11061": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@32994",
+        "__@toStringTag@11061",
         "byteLength"
       ]
     },
@@ -466,18 +466,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@33095": {
+        "__@species@11100": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@32994": {
+        "__@toStringTag@11061": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@33095",
-        "__@toStringTag@32994",
+        "__@species@11100",
+        "__@toStringTag@11061",
         "byteLength"
       ]
     },

package/build/routes/chrome/http/pdf.post.body.json

@@ -474,8 +474,12 @@
           "type": "boolean"
         },
         "timeout": {
-          "description": "Timeout in milliseconds. Pass `0` to disable timeout.",
+          "description": "Timeout in milliseconds. Pass `0` to disable timeout.\n\nThe default value can be changed by using {@link Page.setDefaultTimeout}",
           "type": "number"
+        },
+        "waitForFonts": {
+          "description": "If true, waits for `document.fonts.ready` to resolve. This might require\nactivating the page using {@link Page.bringToFront} if the page is in the\nbackground.",
+          "type": "boolean"
         }
       },
       "additionalProperties": false
@@ -549,14 +553,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@88036": {
+                "__@toStringTag@76992": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@88036",
+                "__@toStringTag@76992",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -591,13 +595,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@88036": {
+        "__@toStringTag@76992": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@88036",
+        "__@toStringTag@76992",
         "byteLength"
       ]
     },
@@ -607,18 +611,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@88137": {
+        "__@species@77031": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@88036": {
+        "__@toStringTag@76992": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@88137",
-        "__@toStringTag@88036",
+        "__@species@77031",
+        "__@toStringTag@76992",
         "byteLength"
       ]
     },

package/build/routes/chrome/http/scrape.post.body.json

@@ -455,14 +455,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@121462": {
+                "__@toStringTag@110270": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@121462",
+                "__@toStringTag@110270",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -497,13 +497,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@121462": {
+        "__@toStringTag@110270": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@121462",
+        "__@toStringTag@110270",
         "byteLength"
       ]
     },
@@ -513,18 +513,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@121563": {
+        "__@species@110309": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@121462": {
+        "__@toStringTag@110270": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@121563",
-        "__@toStringTag@121462",
+        "__@species@110309",
+        "__@toStringTag@110270",
         "byteLength"
       ]
     },

package/build/routes/chrome/http/screenshot.post.body.json

@@ -498,14 +498,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@110235": {
+                "__@toStringTag@99117": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@110235",
+                "__@toStringTag@99117",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -540,13 +540,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@110235": {
+        "__@toStringTag@99117": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@110235",
+        "__@toStringTag@99117",
         "byteLength"
       ]
     },
@@ -556,18 +556,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@110336": {
+        "__@species@99156": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@110235": {
+        "__@toStringTag@99117": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@110336",
-        "__@toStringTag@110235",
+        "__@species@99156",
+        "__@toStringTag@99117",
         "byteLength"
       ]
     },

package/build/routes/chromium/http/content.post.body.json

@@ -408,14 +408,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@132742": {
+                "__@toStringTag@121476": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@132742",
+                "__@toStringTag@121476",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -450,13 +450,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@132742": {
+        "__@toStringTag@121476": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@132742",
+        "__@toStringTag@121476",
         "byteLength"
       ]
     },
@@ -466,18 +466,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@132843": {
+        "__@species@121515": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@132742": {
+        "__@toStringTag@121476": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@132843",
-        "__@toStringTag@132742",
+        "__@species@121515",
+        "__@toStringTag@121476",
         "byteLength"
       ]
     },

package/build/routes/chromium/http/pdf.post.body.json

@@ -474,8 +474,12 @@
           "type": "boolean"
         },
         "timeout": {
-          "description": "Timeout in milliseconds. Pass `0` to disable timeout.",
+          "description": "Timeout in milliseconds. Pass `0` to disable timeout.\n\nThe default value can be changed by using {@link Page.setDefaultTimeout}",
           "type": "number"
+        },
+        "waitForFonts": {
+          "description": "If true, waits for `document.fonts.ready` to resolve. This might require\nactivating the page using {@link Page.bringToFront} if the page is in the\nbackground.",
+          "type": "boolean"
         }
       },
       "additionalProperties": false
@@ -549,14 +553,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@220645": {
+                "__@toStringTag@198348": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@220645",
+                "__@toStringTag@198348",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -591,13 +595,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@220645": {
+        "__@toStringTag@198348": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@220645",
+        "__@toStringTag@198348",
         "byteLength"
       ]
     },
@@ -607,18 +611,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@220746": {
+        "__@species@198387": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@220645": {
+        "__@toStringTag@198348": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@220746",
-        "__@toStringTag@220645",
+        "__@species@198387",
+        "__@toStringTag@198348",
         "byteLength"
       ]
     },

package/build/routes/chromium/http/scrape.post.body.json

@@ -455,14 +455,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@254051": {
+                "__@toStringTag@209498": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@254051",
+                "__@toStringTag@209498",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -497,13 +497,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@254051": {
+        "__@toStringTag@209498": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@254051",
+        "__@toStringTag@209498",
         "byteLength"
       ]
     },
@@ -513,18 +513,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@254152": {
+        "__@species@209537": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@254051": {
+        "__@toStringTag@209498": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@254152",
-        "__@toStringTag@254051",
+        "__@species@209537",
+        "__@toStringTag@209498",
         "byteLength"
       ]
     },

package/build/routes/chromium/http/screenshot.post.body.json

@@ -498,14 +498,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@231880": {
+                "__@toStringTag@242645": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@231880",
+                "__@toStringTag@242645",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -540,13 +540,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@231880": {
+        "__@toStringTag@242645": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@231880",
+        "__@toStringTag@242645",
         "byteLength"
       ]
     },
@@ -556,18 +556,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@231981": {
+        "__@species@242684": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@231880": {
+        "__@toStringTag@242645": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@231981",
-        "__@toStringTag@231880",
+        "__@species@242684",
+        "__@toStringTag@242645",
         "byteLength"
       ]
     },

package/build/utils.d.ts

@@ -128,4 +128,5 @@ export declare const fetchTimeout: (input: RequestInfo | URL, initWithTimeout?:
 export declare const untildify: (path: string) => string;
 export declare const printLogo: (docsLink: string, debugURL?: string | boolean) => string;
 export declare const getCDPClient: (page: Page) => CDPSession;
+export declare const ublockPath: string;
 export {};

package/build/utils.js

@@ -3,12 +3,14 @@ import { BLESS_PAGE_IDENTIFIER, ChromeCDP, ChromePlaywright, ChromiumCDP, Chromi
 import playwright from 'playwright-core';
 import crypto from 'crypto';
 import debug from 'debug';
+import { fileURLToPath } from 'url';
 import gradient from 'gradient-string';
 import { homedir } from 'os';
 import path from 'path';
 const isHTTP = (writeable) => {
     return writeable.writeHead !== undefined;
 };
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const getAuthHeaderToken = (header) => {
     if (header.startsWith('Basic')) {
         const username = header.split(/\s+/).pop() || '';
@@ -332,7 +334,8 @@ export const availableBrowsers = Promise.all([
     return availableBrowsers;
 });
 export const queryParamsToObject = (params) => [...params.entries()].reduce((accum, [key, value]) => {
-    accum[key] = value;
+    accum[key] =
+        value === '' || value === undefined || value === null ? true : value;
     return accum;
 }, {});
 // eslint-disable-next-line @typescript-eslint/no-empty-function
@@ -609,3 +612,4 @@ export const getCDPClient = (page) => {
     const c = page._client;
     return typeof c === 'function' ? c.call(page) : c;
 };
+export const ublockPath = path.join(__dirname, '..', 'extensions', 'ublock');