@browserless.io/browserless 2.15.0 → 2.16.0-beta-1

package/README.md

@@ -39,7 +39,6 @@ If you've been struggling to deploy headless browsers without running into issue
 1. [Full documentation site](https://docs.browserless.io/)
 2. [Live Debugger (using browserless.io)](https://chrome.browserless.io/)
 3. [Docker](https://github.com/browserless/browserless/pkgs/container/base)
-4. [Slack](https://join.slack.com/t/browserless/shared_invite/enQtMzA3OTMwNjA3MzY1LTRmMWU5NjQ0MTQ2YTE2YmU3MzdjNmVlMmU4MThjM2UxODNmNzNlZjVkY2U2NjdkMzYyNTgyZTBiMmE3Nzg0MzY)
 
 ## Features
 
@@ -54,14 +53,16 @@ If you've been struggling to deploy headless browsers without running into issue
 - Error tolerant: if Chrome dies it won't.
 - Support for running and development on Apple's M1 machines
 
-### Cloud
+### Cloud-only
 
 Our [cloud accounts](https://www.browserless.io/pricing/) include all the general features plus extras, such as:
 
-- Inbuilt [residential proxy](https://www.browserless.io/blog/2023/09/24/residential-proxying/)
-- [/unblock API](https://www.browserless.io/blog/2024/02/26/unblock-api/) for avoiding detectors
-- [Hybrid automations](https://www.browserless.io/blog/2024/03/21/hybrid-automations-for-puppeteer/) for streaming login windows during scripts
-- Ability to upload and run extensions _(coming soon)_
+- Inbuilt [residential proxy](https://www.browserless.io/blog/residential-proxying/)
+- [/unblock API](https://www.browserless.io/blog/unblock-api) for avoiding detectors
+- [Automated captcha solving](https://www.browserless.io/blog/captcha-solving) for getting past mandatory checks
+- [Hybrid automations](https://www.browserless.io/blog/hybrid-automations-for-puppeteer/) for streaming login windows during scripts
+- [/reconnect API](https://www.browserless.io/blog/reconnect-api) for keeping browsers alive for reuse
+- [REST APIs](https://www.browserless.io/feature/rest-apis) for tasks such as retrieving HTML, PDFs or Lighthouse metrics
 - SSO, tokens and user roles
 
 ## How it works

package/build/browsers/browsers.cdp.d.ts

@@ -29,7 +29,7 @@ export declare class ChromiumCDP extends EventEmitter {
     close(): Promise<void>;
     pages(): Promise<Page[]>;
     process(): import("child_process").ChildProcess | null;
-    launch(laucherOpts: BrowserLauncherOptions): Promise<Browser>;
+    launch({ options, stealth, }: BrowserLauncherOptions): Promise<Browser>;
     wsEndpoint(): string | null;
     publicWSEndpoint(token: string | null): string | null;
     proxyPageWebSocket(req: Request, socket: Duplex, head: Buffer): Promise<void>;

package/build/browsers/browsers.cdp.js

@@ -1,8 +1,7 @@
-import { BLESS_PAGE_IDENTIFIER, ServerError, chromeExecutablePath, noop, once, } from '@browserless.io/browserless';
+import { BLESS_PAGE_IDENTIFIER, ServerError, chromeExecutablePath, noop, once, ublockPath, } from '@browserless.io/browserless';
 import puppeteer from 'puppeteer-core';
 import { EventEmitter } from 'events';
 import StealthPlugin from 'puppeteer-extra-plugin-stealth';
-import { fileURLToPath } from 'url';
 import getPort from 'get-port';
 import httpProxy from 'http-proxy';
 import path from 'path';
@@ -111,11 +110,17 @@ export class ChromiumCDP extends EventEmitter {
     process() {
         return this.browser?.process() || null;
     }
-    async launch(laucherOpts) {
-        const { options, stealth } = laucherOpts;
+    async launch({ options, stealth, }) {
         this.port = await getPort();
         this.logger.info(`${this.constructor.name} got open port ${this.port}`);
-        const __dirname = path.dirname(fileURLToPath(import.meta.url));
+        const extensionLaunchArgs = options.args?.find((a) => a.startsWith('--load-extension'));
+        // Remove extension flags as we recompile them below with our own
+        options.args = options.args?.filter((a) => !a.startsWith('--load-extension') &&
+            !a.startsWith('--disable-extensions-except'));
+        const extensions = [
+            this.blockAds ? ublockPath : null,
+            extensionLaunchArgs ? extensionLaunchArgs.split('=')[1] : null,
+        ].filter((_) => !!_);
         const finalOptions = {
             ...options,
             args: [
@@ -126,11 +131,8 @@ export class ChromiumCDP extends EventEmitter {
             ].filter((_) => !!_),
             executablePath: this.executablePath,
         };
-        if (this.blockAds) {
-            // Necessary to load extensions
-            finalOptions.headless = false;
-            const loadExtensionPaths = path.join(__dirname, '..', '..', 'extensions', 'ublock');
-            finalOptions.args.push('--load-extension=' + loadExtensionPaths, '--disable-extensions-except=' + loadExtensionPaths);
+        if (extensions.length) {
+            finalOptions.args.push('--load-extension=' + extensions.join(','), '--disable-extensions-except=' + extensions.join(','));
         }
         const launch = stealth
             ? puppeteerStealth.launch.bind(puppeteerStealth)

package/build/browsers/index.js

@@ -353,7 +353,7 @@ export class BrowserManager {
             options: launchOptions,
             pwVersion,
             req,
-            stealth: launchOptions?.stealth
+            stealth: launchOptions?.stealth,
         });
         await this.hooks.browser({ browser, req });
         const session = {

package/build/limiter.spec.js

@@ -91,6 +91,25 @@ describe(`Limiter`, () => {
         await wait;
         expect(handlerTwo.calledOnce).to.be.true;
     });
+    it('continues to process jobs even if an earlier job errors', (d) => {
+        const config = new Config();
+        const monitoring = new Monitoring(config);
+        const metrics = new Metrics();
+        config.setConcurrent(1);
+        config.setQueued(1);
+        config.setTimeout(-1);
+        const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
+        const errorJob = () => Promise.reject(new Error('Danger, danger. High voltage!'));
+        const okJob = spy();
+        const jobOne = limiter.limit(errorJob, asyncNoop, asyncNoop, noop);
+        const jobTwo = limiter.limit(okJob, asyncNoop, asyncNoop, noop);
+        jobOne();
+        jobTwo();
+        limiter.addEventListener('end', () => {
+            expect(okJob.calledOnce).to.be.true;
+            d(undefined);
+        });
+    });
     it('bubbles up errors', async () => {
         const config = new Config();
         const monitoring = new Monitoring(config);

package/build/routes/chrome/http/content.post.body.json

@@ -408,14 +408,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@32994": {
+                "__@toStringTag@11070": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@32994",
+                "__@toStringTag@11070",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -450,13 +450,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@32994": {
+        "__@toStringTag@11070": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@32994",
+        "__@toStringTag@11070",
         "byteLength"
       ]
     },
@@ -466,18 +466,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@33095": {
+        "__@species@11171": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@32994": {
+        "__@toStringTag@11070": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@33095",
-        "__@toStringTag@32994",
+        "__@species@11171",
+        "__@toStringTag@11070",
         "byteLength"
       ]
     },

package/build/routes/chrome/http/scrape.post.body.json

@@ -455,14 +455,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@121462": {
+                "__@toStringTag@110224": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@121462",
+                "__@toStringTag@110224",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -497,13 +497,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@121462": {
+        "__@toStringTag@110224": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@121462",
+        "__@toStringTag@110224",
         "byteLength"
       ]
     },
@@ -513,18 +513,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@121563": {
+        "__@species@110325": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@121462": {
+        "__@toStringTag@110224": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@121563",
-        "__@toStringTag@121462",
+        "__@species@110325",
+        "__@toStringTag@110224",
         "byteLength"
       ]
     },

package/build/routes/chrome/http/screenshot.post.body.json

@@ -498,14 +498,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@110235": {
+                "__@toStringTag@121528": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@110235",
+                "__@toStringTag@121528",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -540,13 +540,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@110235": {
+        "__@toStringTag@121528": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@110235",
+        "__@toStringTag@121528",
         "byteLength"
       ]
     },
@@ -556,18 +556,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@110336": {
+        "__@species@121629": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@110235": {
+        "__@toStringTag@121528": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@110336",
-        "__@toStringTag@110235",
+        "__@species@121629",
+        "__@toStringTag@121528",
         "byteLength"
       ]
     },

package/build/routes/chromium/http/pdf.post.body.json

@@ -549,14 +549,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@220645": {
+                "__@toStringTag@187784": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@220645",
+                "__@toStringTag@187784",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -591,13 +591,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@220645": {
+        "__@toStringTag@187784": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@220645",
+        "__@toStringTag@187784",
         "byteLength"
       ]
     },
@@ -607,18 +607,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@220746": {
+        "__@species@187885": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@220645": {
+        "__@toStringTag@187784": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@220746",
-        "__@toStringTag@220645",
+        "__@species@187885",
+        "__@toStringTag@187784",
         "byteLength"
       ]
     },

package/build/routes/chromium/http/scrape.post.body.json

@@ -455,14 +455,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@254051": {
+                "__@toStringTag@231869": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@254051",
+                "__@toStringTag@231869",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -497,13 +497,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@254051": {
+        "__@toStringTag@231869": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@254051",
+        "__@toStringTag@231869",
         "byteLength"
       ]
     },
@@ -513,18 +513,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@254152": {
+        "__@species@231970": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@254051": {
+        "__@toStringTag@231869": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@254152",
-        "__@toStringTag@254051",
+        "__@species@231970",
+        "__@toStringTag@231869",
         "byteLength"
       ]
     },

package/build/routes/chromium/http/screenshot.post.body.json

@@ -498,14 +498,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@231880": {
+                "__@toStringTag@243170": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@231880",
+                "__@toStringTag@243170",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -540,13 +540,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@231880": {
+        "__@toStringTag@243170": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@231880",
+        "__@toStringTag@243170",
         "byteLength"
       ]
     },
@@ -556,18 +556,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@231981": {
+        "__@species@243271": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@231880": {
+        "__@toStringTag@243170": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@231981",
-        "__@toStringTag@231880",
+        "__@species@243271",
+        "__@toStringTag@243170",
         "byteLength"
       ]
     },

package/build/utils.d.ts

@@ -128,4 +128,5 @@ export declare const fetchTimeout: (input: RequestInfo | URL, initWithTimeout?:
 export declare const untildify: (path: string) => string;
 export declare const printLogo: (docsLink: string, debugURL?: string | boolean) => string;
 export declare const getCDPClient: (page: Page) => CDPSession;
+export declare const ublockPath: string;
 export {};

package/build/utils.js

@@ -3,12 +3,14 @@ import { BLESS_PAGE_IDENTIFIER, ChromeCDP, ChromePlaywright, ChromiumCDP, Chromi
 import playwright from 'playwright-core';
 import crypto from 'crypto';
 import debug from 'debug';
+import { fileURLToPath } from 'url';
 import gradient from 'gradient-string';
 import { homedir } from 'os';
 import path from 'path';
 const isHTTP = (writeable) => {
     return writeable.writeHead !== undefined;
 };
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const getAuthHeaderToken = (header) => {
     if (header.startsWith('Basic')) {
         const username = header.split(/\s+/).pop() || '';
@@ -332,7 +334,8 @@ export const availableBrowsers = Promise.all([
     return availableBrowsers;
 });
 export const queryParamsToObject = (params) => [...params.entries()].reduce((accum, [key, value]) => {
-    accum[key] = value;
+    accum[key] =
+        value === '' || value === undefined || value === null ? true : value;
     return accum;
 }, {});
 // eslint-disable-next-line @typescript-eslint/no-empty-function
@@ -609,3 +612,4 @@ export const getCDPClient = (page) => {
     const c = page._client;
     return typeof c === 'function' ? c.call(page) : c;
 };
+export const ublockPath = path.join(__dirname, '..', 'extensions', 'ublock');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@browserless.io/browserless",
-  "version": "2.15.0",
+  "version": "2.16.0-beta-1",
   "license": "SSPL",
   "description": "The browserless platform",
   "author": "browserless.io",
@@ -51,7 +51,7 @@
     "debug": "^4.3.5",
     "del": "^7.0.0",
     "enjoi": "^9.0.1",
-    "file-type": "^19.0.0",
+    "file-type": "^19.1.0",
     "get-port": "^7.1.0",
     "gradient-string": "^2.0.0",
     "http-proxy": "^1.18.1",
@@ -76,10 +76,10 @@
     "@types/http-proxy": "^1.17.14",
     "@types/micromatch": "^4.0.9",
     "@types/mocha": "^10.0.7",
-    "@types/node": "^20.14.9",
+    "@types/node": "^20.14.10",
     "@types/sinon": "^17.0.3",
     "@typescript-eslint/eslint-plugin": "^7.15.0",
-    "@typescript-eslint/parser": "^7.15.0",
+    "@typescript-eslint/parser": "^7.16.0",
     "assert": "^2.0.0",
     "chai": "^5.1.1",
     "cross-env": "^7.0.3",
@@ -90,7 +90,7 @@
     "eslint-plugin-typescript-sort-keys": "^3.2.0",
     "extract-zip": "^2.0.1",
     "gunzip-maybe": "^1.4.2",
-    "marked": "^13.0.1",
+    "marked": "^13.0.2",
     "mocha": "^10.6.0",
     "move-file": "^3.1.0",
     "prettier": "^3.3.2",

package/scripts/build-schemas.js

@@ -12,7 +12,7 @@ const moduleMain = path.normalize(import.meta.url).endsWith(process.argv[1]);
 
 /**
  * Find an exported interface in a TypeScript AST
- * 
+ *
  * @param {ts.Node} node The node to search for the exported interface
  * @param {string} interfaceName The name of the interface to search for
  * @returns {ts.InterfaceDeclaration | ts.Identifier | null}
@@ -29,7 +29,11 @@ const findExportedInterface = (node, interfaceName) => {
   }
 
   // Check for re-exported interfaces
-  if (ts.isExportDeclaration(node) && node.exportClause && ts.isNamedExports(node.exportClause)) {
+  if (
+    ts.isExportDeclaration(node) &&
+    node.exportClause &&
+    ts.isNamedExports(node.exportClause)
+  ) {
     const elements = node.exportClause.elements;
     for (const element of elements) {
       if (element.name.text === interfaceName) {
@@ -50,7 +54,7 @@ const findExportedInterface = (node, interfaceName) => {
 
 /**
  * Creates an standard JSON schema file for each route (see https://json-schema.org/specification)
- * 
+ *
  * @param {string[]} externalHTTPRoutes Additional HTTP routes to parse
  * @param {string[]} externalWebSocketRoutes Additional WS routes to parse
  */

package/src/browsers/browsers.cdp.ts

@@ -8,12 +8,12 @@ import {
   chromeExecutablePath,
   noop,
   once,
+  ublockPath,
 } from '@browserless.io/browserless';
 import puppeteer, { Browser, Page, Target } from 'puppeteer-core';
 import { Duplex } from 'stream';
 import { EventEmitter } from 'events';
 import StealthPlugin from 'puppeteer-extra-plugin-stealth';
-import { fileURLToPath } from 'url';
 import getPort from 'get-port';
 import httpProxy from 'http-proxy';
 import path from 'path';
@@ -170,11 +170,29 @@ export class ChromiumCDP extends EventEmitter {
     return this.browser?.process() || null;
   }
 
-  public async launch(laucherOpts: BrowserLauncherOptions): Promise<Browser> {
-    const { options, stealth } = laucherOpts;
+  public async launch({
+    options,
+    stealth,
+  }: BrowserLauncherOptions): Promise<Browser> {
     this.port = await getPort();
     this.logger.info(`${this.constructor.name} got open port ${this.port}`);
-    const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+    const extensionLaunchArgs = options.args?.find((a) =>
+      a.startsWith('--load-extension'),
+    );
+
+    // Remove extension flags as we recompile them below with our own
+    options.args = options.args?.filter(
+      (a) =>
+        !a.startsWith('--load-extension') &&
+        !a.startsWith('--disable-extensions-except'),
+    );
+
+    const extensions = [
+      this.blockAds ? ublockPath : null,
+      extensionLaunchArgs ? extensionLaunchArgs.split('=')[1] : null,
+    ].filter((_) => !!_);
+
     const finalOptions = {
       ...options,
       args: [
@@ -186,21 +204,10 @@ export class ChromiumCDP extends EventEmitter {
       executablePath: this.executablePath,
     };
 
-    if (this.blockAds) {
-      // Necessary to load extensions
-      finalOptions.headless = false;
-
-      const loadExtensionPaths: string = path.join(
-        __dirname,
-        '..',
-        '..',
-        'extensions',
-        'ublock',
-      );
-
+    if (extensions.length) {
       finalOptions.args.push(
-        '--load-extension=' + loadExtensionPaths,
-        '--disable-extensions-except=' + loadExtensionPaths,
+        '--load-extension=' + extensions.join(','),
+        '--disable-extensions-except=' + extensions.join(','),
       );
     }