@browserless.io/browserless 2.14.0 → 2.16.0-beta-1

package/CHANGELOG.md

@@ -1,11 +1,14 @@
-# [Latest](https://github.com/browserless/chrome/compare/v2.14.0...main)
+# [Latest](https://github.com/browserless/chrome/compare/v2.15.0...main)
+- Dependency updates.
+
+# [v2.15.0](https://github.com/browserless/chrome/compare/v2.14.0...v2.15.0)
+- Bug fix to avoid removing healthy jobs from limiter when a job fails
 - Dependency updates.
 
 # [v2.14.0](https://github.com/browserless/chrome/compare/v2.13.0...v2.14.0)
 **Potentially Breaking**
 - Merged duplicate code in browsers and fixed name `WebkitPlaywright` to `WebKitPlaywright` (capitalized "K").
 - Devtools now bundled into the repository at build-time for the `/debugger` route.
--
 - Dependency updates.
 
 # [v2.13.0](https://github.com/browserless/chrome/compare/v2.12.0...v2.13.0)

package/README.md

@@ -39,7 +39,6 @@ If you've been struggling to deploy headless browsers without running into issue
 1. [Full documentation site](https://docs.browserless.io/)
 2. [Live Debugger (using browserless.io)](https://chrome.browserless.io/)
 3. [Docker](https://github.com/browserless/browserless/pkgs/container/base)
-4. [Slack](https://join.slack.com/t/browserless/shared_invite/enQtMzA3OTMwNjA3MzY1LTRmMWU5NjQ0MTQ2YTE2YmU3MzdjNmVlMmU4MThjM2UxODNmNzNlZjVkY2U2NjdkMzYyNTgyZTBiMmE3Nzg0MzY)
 
 ## Features
 
@@ -54,14 +53,16 @@ If you've been struggling to deploy headless browsers without running into issue
 - Error tolerant: if Chrome dies it won't.
 - Support for running and development on Apple's M1 machines
 
-### Cloud
+### Cloud-only
 
 Our [cloud accounts](https://www.browserless.io/pricing/) include all the general features plus extras, such as:
 
-- Inbuilt [residential proxy](https://www.browserless.io/blog/2023/09/24/residential-proxying/)
-- [/unblock API](https://www.browserless.io/blog/2024/02/26/unblock-api/) for avoiding detectors
-- [Hybrid automations](https://www.browserless.io/blog/2024/03/21/hybrid-automations-for-puppeteer/) for streaming login windows during scripts
-- Ability to upload and run extensions _(coming soon)_
+- Inbuilt [residential proxy](https://www.browserless.io/blog/residential-proxying/)
+- [/unblock API](https://www.browserless.io/blog/unblock-api) for avoiding detectors
+- [Automated captcha solving](https://www.browserless.io/blog/captcha-solving) for getting past mandatory checks
+- [Hybrid automations](https://www.browserless.io/blog/hybrid-automations-for-puppeteer/) for streaming login windows during scripts
+- [/reconnect API](https://www.browserless.io/blog/reconnect-api) for keeping browsers alive for reuse
+- [REST APIs](https://www.browserless.io/feature/rest-apis) for tasks such as retrieving HTML, PDFs or Lighthouse metrics
 - SSO, tokens and user roles
 
 ## How it works

package/build/browserless.d.ts

@@ -1,5 +1,3 @@
-/// <reference types="node" />
-/// <reference types="node" />
 import { Logger as BlessLogger, BrowserHTTPRoute, BrowserManager, BrowserWebsocketRoute, Config, FileSystem, HTTPRoute, HTTPServer, Hooks, Limiter, Metrics, Monitoring, Router, Token, WebHooks, WebSocketRoute } from '@browserless.io/browserless';
 import { EventEmitter } from 'events';
 type routeInstances = HTTPRoute | BrowserHTTPRoute | WebSocketRoute | BrowserWebsocketRoute;

package/build/browsers/browsers.cdp.d.ts

@@ -1,8 +1,3 @@
-/// <reference types="node" />
-/// <reference types="node" />
-/// <reference types="node" />
-/// <reference types="node" />
-/// <reference types="node" />
 import { BrowserLauncherOptions, Config, Logger, Request } from '@browserless.io/browserless';
 import { Browser, Page, Target } from 'puppeteer-core';
 import { Duplex } from 'stream';
@@ -34,7 +29,7 @@ export declare class ChromiumCDP extends EventEmitter {
     close(): Promise<void>;
     pages(): Promise<Page[]>;
     process(): import("child_process").ChildProcess | null;
-    launch(laucherOpts: BrowserLauncherOptions): Promise<Browser>;
+    launch({ options, stealth, }: BrowserLauncherOptions): Promise<Browser>;
     wsEndpoint(): string | null;
     publicWSEndpoint(token: string | null): string | null;
     proxyPageWebSocket(req: Request, socket: Duplex, head: Buffer): Promise<void>;

package/build/browsers/browsers.cdp.js

@@ -1,8 +1,7 @@
-import { BLESS_PAGE_IDENTIFIER, ServerError, chromeExecutablePath, noop, once, } from '@browserless.io/browserless';
+import { BLESS_PAGE_IDENTIFIER, ServerError, chromeExecutablePath, noop, once, ublockPath, } from '@browserless.io/browserless';
 import puppeteer from 'puppeteer-core';
 import { EventEmitter } from 'events';
 import StealthPlugin from 'puppeteer-extra-plugin-stealth';
-import { fileURLToPath } from 'url';
 import getPort from 'get-port';
 import httpProxy from 'http-proxy';
 import path from 'path';
@@ -111,11 +110,17 @@ export class ChromiumCDP extends EventEmitter {
     process() {
         return this.browser?.process() || null;
     }
-    async launch(laucherOpts) {
-        const { options, stealth } = laucherOpts;
+    async launch({ options, stealth, }) {
         this.port = await getPort();
         this.logger.info(`${this.constructor.name} got open port ${this.port}`);
-        const __dirname = path.dirname(fileURLToPath(import.meta.url));
+        const extensionLaunchArgs = options.args?.find((a) => a.startsWith('--load-extension'));
+        // Remove extension flags as we recompile them below with our own
+        options.args = options.args?.filter((a) => !a.startsWith('--load-extension') &&
+            !a.startsWith('--disable-extensions-except'));
+        const extensions = [
+            this.blockAds ? ublockPath : null,
+            extensionLaunchArgs ? extensionLaunchArgs.split('=')[1] : null,
+        ].filter((_) => !!_);
         const finalOptions = {
             ...options,
             args: [
@@ -126,11 +131,8 @@ export class ChromiumCDP extends EventEmitter {
             ].filter((_) => !!_),
             executablePath: this.executablePath,
         };
-        if (this.blockAds) {
-            // Necessary to load extensions
-            finalOptions.headless = false;
-            const loadExtensionPaths = path.join(__dirname, '..', '..', 'extensions', 'ublock');
-            finalOptions.args.push('--load-extension=' + loadExtensionPaths, '--disable-extensions-except=' + loadExtensionPaths);
+        if (extensions.length) {
+            finalOptions.args.push('--load-extension=' + extensions.join(','), '--disable-extensions-except=' + extensions.join(','));
         }
         const launch = stealth
             ? puppeteerStealth.launch.bind(puppeteerStealth)

package/build/browsers/browsers.playwright.d.ts

@@ -1,7 +1,3 @@
-/// <reference types="node" />
-/// <reference types="node" />
-/// <reference types="node" />
-/// <reference types="node" />
 import { BrowserLauncherOptions, BrowserServerOptions, Config, Logger, Request } from '@browserless.io/browserless';
 import playwright, { Page } from 'playwright-core';
 import { Duplex } from 'stream';
@@ -31,19 +27,19 @@ declare class BasePlaywright extends EventEmitter {
     protected makeLaunchOptions(opts: BrowserServerOptions): {
         args: string[];
         executablePath: string;
-        chromiumSandbox?: boolean | undefined;
-        devtools?: boolean | undefined;
-        downloadsPath?: string | undefined;
-        headless?: boolean | undefined;
-        ignoreDefaultArgs?: boolean | string[] | undefined;
+        chromiumSandbox?: boolean;
+        devtools?: boolean;
+        downloadsPath?: string;
+        headless?: boolean;
+        ignoreDefaultArgs?: boolean | string[];
         proxy?: {
-            bypass?: string | undefined;
-            password?: string | undefined;
+            bypass?: string;
+            password?: string;
             server: string;
-            username?: string | undefined;
-        } | undefined;
-        timeout?: number | undefined;
-        tracesDir?: string | undefined;
+            username?: string;
+        };
+        timeout?: number;
+        tracesDir?: string;
     };
     keepUntil(): number;
     isRunning(): boolean;
@@ -70,19 +66,19 @@ export declare class FirefoxPlaywright extends BasePlaywright {
     protected makeLaunchOptions(opts: BrowserServerOptions): {
         args: string[];
         executablePath: string;
-        chromiumSandbox?: boolean | undefined;
-        devtools?: boolean | undefined;
-        downloadsPath?: string | undefined;
-        headless?: boolean | undefined;
-        ignoreDefaultArgs?: boolean | string[] | undefined;
+        chromiumSandbox?: boolean;
+        devtools?: boolean;
+        downloadsPath?: string;
+        headless?: boolean;
+        ignoreDefaultArgs?: boolean | string[];
         proxy?: {
-            bypass?: string | undefined;
-            password?: string | undefined;
+            bypass?: string;
+            password?: string;
             server: string;
-            username?: string | undefined;
-        } | undefined;
-        timeout?: number | undefined;
-        tracesDir?: string | undefined;
+            username?: string;
+        };
+        timeout?: number;
+        tracesDir?: string;
     };
 }
 export declare class WebKitPlaywright extends BasePlaywright {
@@ -90,19 +86,19 @@ export declare class WebKitPlaywright extends BasePlaywright {
     protected makeLaunchOptions(opts: BrowserServerOptions): {
         args: string[];
         executablePath: string;
-        chromiumSandbox?: boolean | undefined;
-        devtools?: boolean | undefined;
-        downloadsPath?: string | undefined;
-        headless?: boolean | undefined;
-        ignoreDefaultArgs?: boolean | string[] | undefined;
+        chromiumSandbox?: boolean;
+        devtools?: boolean;
+        downloadsPath?: string;
+        headless?: boolean;
+        ignoreDefaultArgs?: boolean | string[];
         proxy?: {
-            bypass?: string | undefined;
-            password?: string | undefined;
+            bypass?: string;
+            password?: string;
             server: string;
-            username?: string | undefined;
-        } | undefined;
-        timeout?: number | undefined;
-        tracesDir?: string | undefined;
+            username?: string;
+        };
+        timeout?: number;
+        tracesDir?: string;
     };
 }
 export {};

package/build/browsers/index.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 import { BrowserHTTPRoute, BrowserInstance, BrowserServerOptions, BrowserWebsocketRoute, BrowserlessSession, BrowserlessSessionJSON, CDPJSONPayload, CDPLaunchOptions, ChromiumCDP, Config, Hooks, Logger, Request } from '@browserless.io/browserless';
 import { Page } from 'puppeteer-core';
 export declare class BrowserManager {
@@ -46,7 +45,7 @@ export declare class BrowserManager {
         resolver(val: unknown): void;
         routePath: string | string[];
         startedOn: number;
-        trackingId?: string | undefined;
+        trackingId?: string;
         ttl: number;
         userDataDir: string | null;
     }[]>;

package/build/browsers/index.js

@@ -353,7 +353,7 @@ export class BrowserManager {
             options: launchOptions,
             pwVersion,
             req,
-            stealth: launchOptions?.stealth
+            stealth: launchOptions?.stealth,
         });
         await this.hooks.browser({ browser, req });
         const session = {

package/build/config.d.ts

@@ -1,5 +1,3 @@
-/// <reference types="node" />
-/// <reference types="node" />
 import { EventEmitter } from 'events';
 import playwright from 'playwright-core';
 export declare class Config extends EventEmitter {

package/build/file-system.d.ts

@@ -1,5 +1,3 @@
-/// <reference types="node" />
-/// <reference types="node" />
 import { Config, Logger } from '@browserless.io/browserless';
 import { EventEmitter } from 'events';
 export declare class FileSystem extends EventEmitter {

package/build/hooks.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 import { AfterResponse, BeforeRequest, BrowserHook, PageHook } from '@browserless.io/browserless';
 import { EventEmitter } from 'events';
 export declare class Hooks extends EventEmitter {

package/build/http.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 import { BrowserServerOptions, CDPLaunchOptions } from '@browserless.io/browserless';
 import http from 'http';
 export declare const errorCodes: {

package/build/limiter.d.ts

@@ -18,6 +18,11 @@ export declare class Limiter extends q {
     protected queued: number;
     protected logger: Logger;
     constructor(config: Config, metrics: Metrics, monitor: Monitoring, webhooks: WebHooks, hooks: Hooks);
+    protected _errorHandler({ detail: { error }, }: {
+        detail: {
+            error: unknown;
+        };
+    }): void;
     protected handleEnd(): void;
     protected jobEnd(jobInfo: AfterResponse): void;
     protected handleSuccess({ detail: { job } }: {

package/build/limiter.js

@@ -41,6 +41,9 @@ export class Limiter extends q {
         this.addEventListener('error', this.handleFail.bind(this));
         this.addEventListener('end', this.handleEnd.bind(this));
     }
+    _errorHandler({ detail: { error }, }) {
+        this.logger.error(error);
+    }
     handleEnd() {
         this.logQueue('All jobs complete.');
     }

package/build/limiter.spec.js

@@ -91,6 +91,25 @@ describe(`Limiter`, () => {
         await wait;
         expect(handlerTwo.calledOnce).to.be.true;
     });
+    it('continues to process jobs even if an earlier job errors', (d) => {
+        const config = new Config();
+        const monitoring = new Monitoring(config);
+        const metrics = new Metrics();
+        config.setConcurrent(1);
+        config.setQueued(1);
+        config.setTimeout(-1);
+        const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
+        const errorJob = () => Promise.reject(new Error('Danger, danger. High voltage!'));
+        const okJob = spy();
+        const jobOne = limiter.limit(errorJob, asyncNoop, asyncNoop, noop);
+        const jobTwo = limiter.limit(okJob, asyncNoop, asyncNoop, noop);
+        jobOne();
+        jobTwo();
+        limiter.addEventListener('end', () => {
+            expect(okJob.calledOnce).to.be.true;
+            d(undefined);
+        });
+    });
     it('bubbles up errors', async () => {
         const config = new Config();
         const monitoring = new Monitoring(config);
@@ -103,7 +122,7 @@ describe(`Limiter`, () => {
         const spy = () => new Promise((_r, rej) => rej(error));
         const job = limiter.limit(spy, asyncNoop, asyncNoop, noop);
         await job().catch(noop);
-        limiter.addEventListener('end', (res) => {
+        limiter.addEventListener('error', (res) => {
             // 'end' callback fires before the failed callback hooks do, so nextTick to wait
             process.nextTick(() => {
                 expect(res.detail.error).to.eql(error);

package/build/metrics.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 import { IBrowserlessStats } from '@browserless.io/browserless';
 import { EventEmitter } from 'events';
 export declare class Metrics extends EventEmitter {

package/build/monitoring.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 import { Config, IResourceLoad, Logger } from '@browserless.io/browserless';
 import { EventEmitter } from 'events';
 export declare class Monitoring extends EventEmitter {

package/build/router.d.ts

@@ -1,6 +1,3 @@
-/// <reference types="node" />
-/// <reference types="node" />
-/// <reference types="node" />
 import { BrowserHTTPRoute, BrowserManager, BrowserWebsocketRoute, Config, HTTPRoute, Limiter, Logger, Request, Response, WebSocketRoute } from '@browserless.io/browserless';
 import { EventEmitter } from 'events';
 import stream from 'stream';

package/build/routes/chrome/http/content.post.body.json

@@ -109,7 +109,14 @@
       "type": "string"
     },
     "viewport": {
-      "$ref": "#/definitions/Viewport"
+      "anyOf": [
+        {
+          "$ref": "#/definitions/Viewport"
+        },
+        {
+          "type": "null"
+        }
+      ]
     },
     "waitForEvent": {
       "type": "object",
@@ -401,14 +408,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@11034": {
+                "__@toStringTag@11070": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@11034",
+                "__@toStringTag@11070",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -443,13 +450,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@11034": {
+        "__@toStringTag@11070": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@11034",
+        "__@toStringTag@11070",
         "byteLength"
       ]
     },
@@ -459,18 +466,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@11135": {
+        "__@species@11171": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@11034": {
+        "__@toStringTag@11070": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@11135",
-        "__@toStringTag@11034",
+        "__@species@11171",
+        "__@toStringTag@11070",
         "byteLength"
       ]
     },

package/build/routes/chrome/http/pdf.post.body.json

@@ -111,7 +111,14 @@
       "type": "string"
     },
     "viewport": {
-      "$ref": "#/definitions/Viewport"
+      "anyOf": [
+        {
+          "$ref": "#/definitions/Viewport"
+        },
+        {
+          "type": "null"
+        }
+      ]
     },
     "waitForEvent": {
       "type": "object",
@@ -542,14 +549,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@98673": {
+                "__@toStringTag@88036": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@98673",
+                "__@toStringTag@88036",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -584,13 +591,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@98673": {
+        "__@toStringTag@88036": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@98673",
+        "__@toStringTag@88036",
         "byteLength"
       ]
     },
@@ -600,18 +607,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@98774": {
+        "__@species@88137": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@98673": {
+        "__@toStringTag@88036": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@98774",
-        "__@toStringTag@98673",
+        "__@species@88137",
+        "__@toStringTag@88036",
         "byteLength"
       ]
     },

package/build/routes/chrome/http/scrape.post.body.json

@@ -117,7 +117,14 @@
       "type": "string"
     },
     "viewport": {
-      "$ref": "#/definitions/Viewport"
+      "anyOf": [
+        {
+          "$ref": "#/definitions/Viewport"
+        },
+        {
+          "type": "null"
+        }
+      ]
     },
     "waitForEvent": {
       "type": "object",
@@ -448,14 +455,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@109864": {
+                "__@toStringTag@110224": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@109864",
+                "__@toStringTag@110224",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -490,13 +497,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@109864": {
+        "__@toStringTag@110224": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@109864",
+        "__@toStringTag@110224",
         "byteLength"
       ]
     },
@@ -506,18 +513,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@109965": {
+        "__@species@110325": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@109864": {
+        "__@toStringTag@110224": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@109965",
-        "__@toStringTag@109864",
+        "__@species@110325",
+        "__@toStringTag@110224",
         "byteLength"
       ]
     },

package/build/routes/chrome/http/screenshot.post.body.json

@@ -117,7 +117,14 @@
       "type": "string"
     },
     "viewport": {
-      "$ref": "#/definitions/Viewport"
+      "anyOf": [
+        {
+          "$ref": "#/definitions/Viewport"
+        },
+        {
+          "type": "null"
+        }
+      ]
     },
     "waitForEvent": {
       "type": "object",
@@ -491,14 +498,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@121132": {
+                "__@toStringTag@121528": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@121132",
+                "__@toStringTag@121528",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -533,13 +540,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@121132": {
+        "__@toStringTag@121528": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@121132",
+        "__@toStringTag@121528",
         "byteLength"
       ]
     },
@@ -549,18 +556,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@121233": {
+        "__@species@121629": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@121132": {
+        "__@toStringTag@121528": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@121233",
-        "__@toStringTag@121132",
+        "__@species@121629",
+        "__@toStringTag@121528",
         "byteLength"
       ]
     },