@browserless.io/browserless 2.0.0-beta-8 → 2.1.1

package/build/types.d.ts

@@ -438,4 +438,35 @@ export interface IBrowserlessStats {
     unhealthy: number;
     units: number;
 }
+export interface CDPJSONPayload {
+    /**
+     * The description of the target. Generally the page's title.
+     */
+    description: string;
+    /**
+     * The fully-qualified URL of the Devtools inspector app.
+     */
+    devtoolsFrontendUrl: string;
+    /**
+     * A Unique Id for the underlying target.
+     */
+    id: string;
+    /**
+     * The title of the target. For pages this is the page's title.
+     */
+    title: string;
+    /**
+     * The type of target, generally "page" or "background_page".
+     */
+    type: string;
+    /**
+     * The current URL the target is consuming or visiting.
+     */
+    url: string;
+    /**
+     * The target or page's WebSocket Debugger URL. Primarily used for legacy
+     * libraries to connect and inspect or remote automate this target.
+     */
+    webSocketDebuggerUrl: string;
+}
 export {};

package/build/utils.d.ts

@@ -12,6 +12,15 @@ export declare const tsExtension = ".d.ts";
 export declare const jsonExtension = ".json";
 export declare const jsExtension = ".js";
 export declare const id: () => string;
+/**
+ * Generates a random, Chrome-compliant page ID with "BLESS"
+ * prepended. This prepended text signals to other parts of the
+ * system that this is a Browserless-created ID so it can be appropriately
+ * handled.
+ *
+ * @returns {string} A random Page ID
+ */
+export declare const pageID: () => string;
 export declare const createLogger: (domain: string) => debug.Debugger;
 export declare const dedent: (strings: string | string[], ...values: string[]) => string;
 export declare const isConnected: (connection: Duplex | ServerResponse) => boolean;

package/build/utils.js

@@ -1,5 +1,5 @@
 import * as fs from 'fs/promises';
-import { CDPChromium, PlaywrightChromium, PlaywrightFirefox, PlaywrightWebkit, codes, contentTypes, encodings, encryptionAlgo, encryptionSep, } from '@browserless.io/browserless';
+import { BLESS_PAGE_IDENTIFIER, CDPChromium, PlaywrightChromium, PlaywrightFirefox, PlaywrightWebkit, codes, contentTypes, encodings, encryptionAlgo, encryptionSep, } from '@browserless.io/browserless';
 import playwright from 'playwright-core';
 import crypto from 'crypto';
 import debug from 'debug';
@@ -26,6 +26,21 @@ export const tsExtension = '.d.ts';
 export const jsonExtension = '.json';
 export const jsExtension = '.js';
 export const id = () => crypto.randomUUID();
+/**
+ * Generates a random, Chrome-compliant page ID with "BLESS"
+ * prepended. This prepended text signals to other parts of the
+ * system that this is a Browserless-created ID so it can be appropriately
+ * handled.
+ *
+ * @returns {string} A random Page ID
+ */
+export const pageID = () => {
+    const chars = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ';
+    const id = Array.from({ length: 32 - BLESS_PAGE_IDENTIFIER.length })
+        .map(() => chars[Math.floor(Math.random() * chars.length)])
+        .join('');
+    return `${BLESS_PAGE_IDENTIFIER}${id}`;
+};
 export const createLogger = (domain) => {
     return debug(`browserless.io:${domain}`);
 };
@@ -158,7 +173,7 @@ export const removeNullStringify = (json, allowNull = true) => {
             return value;
         if (value !== null)
             return value;
-    });
+    }, '  ');
 };
 export const jsonOrString = (maybeJson) => safeParse(maybeJson) ?? maybeJson;
 export const readBody = async (req) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@browserless.io/browserless",
-  "version": "2.0.0-beta-8",
+  "version": "2.1.1",
   "license": "SSPL",
   "description": "The browserless platform",
   "author": "browserless.io",
@@ -25,8 +25,7 @@
     "dev": "npm run build:dev && env-cmd -f .env node build",
     "install:adblock": "node scripts/install-adblock.js",
     "install:browsers": "npx --yes playwright install chromium firefox webkit",
-    "install:cdp-json": "node scripts/install-cdp-json.js",
-    "install:dev": "npm run install:browsers && npm run install:cdp-json",
+    "install:dev": "npm run install:browsers",
     "lint": "eslint . --ext .ts --fix",
     "prepack": "npm run build:dev",
     "prettier": "prettier '{src,functions,scripts,bin,external,bin}/**/*.{js,ts,json}' --log-level error --write",
@@ -43,7 +42,6 @@
     "scripts/*",
     "src/*",
     "static/*",
-    "browser.json",
     "CHANGELOG.md",
     "tsconfig.json"
   ],
@@ -51,18 +49,18 @@
     "debug": "^4.3.2",
     "del": "^7.0.0",
     "enjoi": "^9.0.1",
-    "file-type": "^18.7.0",
+    "file-type": "^19.0.0",
     "get-port": "^7.0.0",
     "gradient-string": "^2.0.0",
     "http-proxy": "^1.18.1",
     "lighthouse": "^11.1.0",
     "micromatch": "^4.0.4",
-    "playwright-core": "^1.40.1",
-    "puppeteer-core": "^21.6.1",
+    "playwright-core": "^1.41.2",
+    "puppeteer-core": "^21.10.0",
     "puppeteer-extra": "^3.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2",
     "queue": "^7.0.0",
-    "systeminformation": "^5.21.20"
+    "systeminformation": "^5.21.24"
   },
   "optionalDependencies": {
     "@types/chai": "^4.3.11",
@@ -71,25 +69,24 @@
     "@types/http-proxy": "^1.17.14",
     "@types/micromatch": "^4.0.6",
     "@types/mocha": "^10.0.6",
-    "@types/node": "^20.10.8",
-    "@types/sinon": "^17.0.2",
-    "@typescript-eslint/eslint-plugin": "^6.18.1",
-    "@typescript-eslint/parser": "^6.18.1",
+    "@types/node": "^20.11.16",
+    "@types/sinon": "^17.0.3",
+    "@typescript-eslint/eslint-plugin": "^6.20.0",
+    "@typescript-eslint/parser": "^6.21.0",
     "assert": "^2.0.0",
-    "chai": "^5.0.0",
+    "chai": "^5.0.3",
     "cross-env": "^7.0.3",
     "env-cmd": "^10.1.0",
-    "esbuild": "^0.19.11",
+    "esbuild": "^0.20.0",
     "esbuild-plugin-polyfill-node": "^0.3.0",
     "eslint": "^8.56.0",
     "eslint-plugin-import": "^2.29.1",
     "eslint-plugin-typescript-sort-keys": "^3.1.0",
     "extract-zip": "^2.0.1",
-    "marked": "^11.1.0",
+    "marked": "^12.0.0",
     "mocha": "^10.0.0",
     "move-file": "^3.1.0",
-    "nodemon": "^3.0.2",
-    "prettier": "^3.1.1",
+    "prettier": "^3.2.5",
     "sinon": "^17.0.1",
     "ts-node": "^10.9.2",
     "typescript": "^5.3.3",
@@ -144,7 +141,7 @@
             "multiple",
             "single"
           ],
-          "allowSeparatedGroups": false
+          "allowSeparatedGroups": true
         }
       ]
     }

package/src/browserless.ts

@@ -3,6 +3,7 @@ import {
   BrowserHTTPRoute,
   BrowserManager,
   BrowserWebsocketRoute,
+  CDPChromium,
   Config,
   FileSystem,
   HTTPRoute,
@@ -11,6 +12,9 @@ import {
   Limiter,
   Metrics,
   Monitoring,
+  PlaywrightChromium,
+  PlaywrightFirefox,
+  PlaywrightWebkit,
   Router,
   Token,
   WebHooks,
@@ -158,6 +162,12 @@ export class Browserless {
   public async start() {
     const httpRoutes: Array<HTTPRoute | BrowserHTTPRoute> = [];
     const wsRoutes: Array<WebSocketRoute | BrowserWebsocketRoute> = [];
+    const internalBrowsers = [
+      CDPChromium,
+      PlaywrightFirefox,
+      PlaywrightChromium,
+      PlaywrightWebkit,
+    ];
 
     const [[httpRouteFiles, wsRouteFiles], installedBrowsers] =
       await Promise.all([getRouteFiles(this.config), availableBrowsers]);
@@ -254,11 +264,12 @@ export class Browserless {
       }
     }
 
-    // Validate that browsers are installed and route paths are unique
+    // Validate that we have the browsers they are asking for
     [...httpRoutes, ...wsRoutes].forEach((route) => {
       if (
         'browser' in route &&
         route.browser &&
+        internalBrowsers.includes(route.browser) &&
         !installedBrowsers.some((b) => b.name === route.browser?.name)
       ) {
         throw new Error(

package/src/browsers/cdp-chromium.ts

@@ -314,17 +314,16 @@ export class CDPChromium extends EventEmitter {
 
   public wsEndpoint = (): string | null => this.browserWSEndpoint;
 
-  public publicWSEndpoint = (token: string): string | null => {
+  public publicWSEndpoint = (token: string | null): string | null => {
     if (!this.browserWSEndpoint) {
       return null;
     }
 
+    const serverURL = new URL(this.config.getExternalWebSocketAddress());
     const wsURL = new URL(this.browserWSEndpoint);
-    const serverURL = new URL(this.config.getExternalAddress());
-
     wsURL.hostname = serverURL.hostname;
     wsURL.port = serverURL.port;
-    wsURL.protocol = serverURL.protocol === 'https' ? 'wss' : 'ws';
+
     if (token) {
       wsURL.searchParams.set('token', token);
     }
@@ -345,16 +344,14 @@ export class CDPChromium extends EventEmitter {
       }
       socket.once('close', resolve);
 
-      this.debug(
-        `Proxying ${req.parsed.href} to browser ${this.browserWSEndpoint}`,
-      );
-
       const [page] = await this.browser.pages();
       const pageLocation = `/devtools/page/${this.getPageId(page)}`;
 
       this.debug(`Proxying ${req.parsed.href} to page "${pageLocation}"`);
 
-      req.url = pageLocation;
+      const target = new URL(pageLocation, this.browserWSEndpoint).href;
+
+      req.url = '';
 
       this.proxy.ws(
         req,
@@ -362,7 +359,7 @@ export class CDPChromium extends EventEmitter {
         head,
         {
           changeOrigin: true,
-          target: this.browserWSEndpoint,
+          target,
         },
         (error) => {
           this.debug(`Error proxying session: ${error}`);

package/src/browsers/index.ts

@@ -82,6 +82,69 @@ export class BrowserManager {
     return dataDirPath;
   };
 
+  public getProtocolJSON = async (): Promise<object> => {
+    this.debug(`Launching Chrome to generate /json/protocol results`);
+    const browser = new CDPChromium({
+      blockAds: false,
+      config: this.config,
+      record: false,
+      userDataDir: null,
+    });
+    await browser.launch();
+    const wsEndpoint = browser.wsEndpoint();
+
+    if (!wsEndpoint) {
+      throw new Error('There was an error launching the browser');
+    }
+
+    const { port } = new URL(wsEndpoint);
+    const res = await fetch(`http://127.0.0.1:${port}/json/protocol`);
+    const protocolJSON = await res.json();
+
+    browser.close();
+
+    return protocolJSON;
+  };
+
+  public getVersionJSON = async (): Promise<{
+    Browser: string;
+    'Debugger-Version': string;
+    'Protocol-Version': string;
+    'User-Agent': string;
+    'V8-Version': string;
+    'WebKit-Version': string;
+    webSocketDebuggerUrl: string;
+  }> => {
+    this.debug(`Launching Chrome to generate /json/version results`);
+    const browser = new CDPChromium({
+      blockAds: false,
+      config: this.config,
+      record: false,
+      userDataDir: null,
+    });
+    await browser.launch();
+    const wsEndpoint = browser.wsEndpoint();
+
+    if (!wsEndpoint) {
+      throw new Error('There was an error launching the browser');
+    }
+
+    const { port } = new URL(wsEndpoint);
+    const res = await fetch(`http://127.0.0.1:${port}/json/version`);
+    const meta = await res.json();
+
+    browser.close();
+
+    const { 'WebKit-Version': webkitVersion } = meta;
+    const debuggerVersion = webkitVersion.match(/\s\(@(\b[0-9a-f]{5,40}\b)/)[1];
+
+    return {
+      ...meta,
+      'Debugger-Version': debuggerVersion,
+      webSocketDebuggerUrl: this.config.getExternalWebSocketAddress(),
+    };
+  };
+
   private generateSessionJson = async (
     browser: BrowserInstance,
     session: BrowserlessSession,
@@ -201,8 +264,8 @@ export class BrowserManager {
     if (req.parsed.pathname.includes('/devtools/browser')) {
       const sessions = Array.from(this.browsers);
       const id = req.parsed.pathname.split('/').pop() as string;
-      const browser = sessions.find(
-        ([b]) => b.wsEndpoint()?.includes(req.parsed.pathname),
+      const browser = sessions.find(([b]) =>
+        b.wsEndpoint()?.includes(req.parsed.pathname),
       );
 
       if (browser) {

package/src/browsers/playwright-chromium.ts

@@ -106,17 +106,16 @@ export class PlaywrightChromium extends EventEmitter {
 
   public wsEndpoint = (): string | null => this.browserWSEndpoint;
 
-  public publicWSEndpoint = (token: string): string | null => {
+  public publicWSEndpoint = (token: string | null): string | null => {
     if (!this.browserWSEndpoint) {
       return null;
     }
 
+    const serverURL = new URL(this.config.getExternalWebSocketAddress());
     const wsURL = new URL(this.browserWSEndpoint);
-    const serverURL = new URL(this.config.getExternalAddress());
-
     wsURL.hostname = serverURL.hostname;
     wsURL.port = serverURL.port;
-    wsURL.protocol = serverURL.protocol === 'https' ? 'wss' : 'ws';
+
     if (token) {
       wsURL.searchParams.set('token', token);
     }

package/src/browsers/playwright-firefox.ts

@@ -99,17 +99,16 @@ export class PlaywrightFirefox extends EventEmitter {
 
   public wsEndpoint = (): string | null => this.browserWSEndpoint;
 
-  public publicWSEndpoint = (token: string): string | null => {
+  public publicWSEndpoint = (token: string | null): string | null => {
     if (!this.browserWSEndpoint) {
       return null;
     }
 
+    const serverURL = new URL(this.config.getExternalWebSocketAddress());
     const wsURL = new URL(this.browserWSEndpoint);
-    const serverURL = new URL(this.config.getExternalAddress());
-
     wsURL.hostname = serverURL.hostname;
     wsURL.port = serverURL.port;
-    wsURL.protocol = serverURL.protocol === 'https' ? 'wss' : 'ws';
+
     if (token) {
       wsURL.searchParams.set('token', token);
     }

package/src/browsers/playwright-webkit.ts

@@ -99,17 +99,16 @@ export class PlaywrightWebkit extends EventEmitter {
 
   public wsEndpoint = (): string | null => this.browserWSEndpoint;
 
-  public publicWSEndpoint = (token: string): string | null => {
+  public publicWSEndpoint = (token: string | null): string | null => {
     if (!this.browserWSEndpoint) {
       return null;
     }
 
+    const serverURL = new URL(this.config.getExternalWebSocketAddress());
     const wsURL = new URL(this.browserWSEndpoint);
-    const serverURL = new URL(this.config.getExternalAddress());
-
     wsURL.hostname = serverURL.hostname;
     wsURL.port = serverURL.port;
-    wsURL.protocol = serverURL.protocol === 'https' ? 'wss' : 'ws';
+
     if (token) {
       wsURL.searchParams.set('token', token);
     }

package/src/config.ts

@@ -409,6 +409,23 @@ export class Config extends EventEmitter {
   public getExternalAddress = (): string =>
     this.external ?? this.getServerAddress();
 
+  /**
+   * Returns the the fully-qualified WebSocket URL for the
+   * external address that browserless might be
+   * running behind *or* the server address if
+   * no external URL is provided.
+   *
+   * @returns {string} The URL to reach the server
+   */
+  public getExternalWebSocketAddress = (): string => {
+    const httpAddress = new URL(this.external ?? this.getServerAddress());
+    httpAddress.protocol = httpAddress.protocol.startsWith('https')
+      ? 'wss:'
+      : 'ws:';
+
+    return httpAddress.href;
+  };
+
   /**
    * When CORS is enabled, returns relevant CORS headers
    * to requests and for the OPTIONS call. Values can be

package/src/constants.ts

@@ -2,3 +2,4 @@ export const encryptionAlgo = 'aes-192-cbc';
 export const encryptionSep = '.';
 export const liveURLSep = ':';
 export const keyLength = 24;
+export const BLESS_PAGE_IDENTIFIER = 'BLESS';

package/src/file-system.spec.ts

@@ -4,7 +4,7 @@ import { expect } from 'chai';
 
 const filePath = '/tmp/_browserless_test_fs_';
 
-describe.only('File-System', () => {
+describe('File-System', () => {
   afterEach(async () => unlink(filePath));
 
   it('saves and encodes files', async () => {

package/src/http.ts

@@ -92,6 +92,10 @@ export enum HTTPRoutes {
   content = '/content',
   download = '/download',
   function = '/function',
+  jsonList = '/json/list',
+  jsonNew = '/json/new',
+  jsonProtocol = '/json/protocol',
+  jsonVersion = '/json/version',
   pdf = '/pdf',
   performance = '/performance',
   scrape = '/scrape',

package/src/routes/chromium/http/json-list.ts

@@ -0,0 +1,50 @@
+import {
+  APITags,
+  HTTPRoute,
+  HTTPRoutes,
+  Methods,
+  Request,
+  Response,
+  contentTypes,
+  dedent,
+  jsonResponse,
+} from '@browserless.io/browserless';
+import { getCDPJSONPayload } from '../utils/cdp.js';
+
+/*
+Example Payload from Chrome:
+  [{
+    "description": "",
+    "devtoolsFrontendUrl": "/devtools/inspector.html?ws=localhost:9222/devtools/page/6CA38A3E207BA534C674D1057B19E9CC",
+    "id": "6CA38A3E207BA534C674D1057B19E9CC",
+    "title": "New Tab",
+    "type": "page",
+    "url": "http://localhost:9222/json/list",
+    "webSocketDebuggerUrl": "ws://localhost:9222/devtools/page/6CA38A3E207BA534C674D1057B19E9CC"
+  }]
+*/
+export type ResponseSchema = Array<ReturnType<typeof getCDPJSONPayload>>;
+
+export default class GetJSONList extends HTTPRoute {
+  accepts = [contentTypes.any];
+  auth = true;
+  browser = null;
+  concurrency = false;
+  contentTypes = [contentTypes.json];
+  description = dedent(`
+    Returns a JSON payload that acts as a pass-through to the DevTools /json/list HTTP API in Chromium.
+    Browserless mocks this payload so that remote clients can connect to the underlying "webSocketDebuggerUrl"
+    which will cause Browserless to start the browser and proxy that request into a blank page.
+  `);
+  method = Methods.get;
+  path = HTTPRoutes.jsonList;
+  tags = [APITags.browserAPI];
+
+  handler = async (_req: Request, res: Response): Promise<void> => {
+    const config = this.config();
+    const externalAddress = config.getExternalAddress();
+    const payload = getCDPJSONPayload(externalAddress);
+
+    return jsonResponse(res, 200, [payload] as ResponseSchema);
+  };
+}

package/src/routes/chromium/http/json-new.ts

@@ -0,0 +1,50 @@
+import {
+  APITags,
+  HTTPRoute,
+  HTTPRoutes,
+  Methods,
+  Request,
+  Response,
+  contentTypes,
+  dedent,
+  jsonResponse,
+} from '@browserless.io/browserless';
+import { getCDPJSONPayload } from '../utils/cdp.js';
+
+/*
+Example Payload from Chrome:
+{
+  "description": "",
+  "devtoolsFrontendUrl": "/devtools/inspector.html?ws=localhost:9222/devtools/page/2F76525C32A916DF30C4F37A4970B8BF",
+  "id": "2F76525C32A916DF30C4F37A4970B8BF",
+  "title": "",
+  "type": "page",
+  "url": "about:blank",
+  "webSocketDebuggerUrl": "ws://localhost:9222/devtools/page/2F76525C32A916DF30C4F37A4970B8BF"
+}
+*/
+export type ResponseSchema = ReturnType<typeof getCDPJSONPayload>;
+
+export default class GetJSONList extends HTTPRoute {
+  accepts = [contentTypes.any];
+  auth = true;
+  browser = null;
+  concurrency = false;
+  contentTypes = [contentTypes.json];
+  description = dedent(`
+    Returns a JSON payload that acts as a pass-through to the DevTools /json/list HTTP API in Chromium.
+    Browserless mocks this payload so that remote clients can connect to the underlying "webSocketDebuggerUrl"
+    which will cause Browserless to start the browser and proxy that request into a blank page.
+  `);
+  method = Methods.put;
+  path = HTTPRoutes.jsonNew;
+  tags = [APITags.browserAPI];
+
+  handler = async (_req: Request, res: Response): Promise<void> => {
+    const config = this.config();
+    const externalAddress = config.getExternalAddress();
+    const payload = getCDPJSONPayload(externalAddress);
+
+    return jsonResponse(res, 200, payload);
+  };
+}

package/src/routes/chromium/http/json-protocol-get.ts

@@ -0,0 +1,38 @@
+import {
+  APITags,
+  HTTPRoute,
+  HTTPRoutes,
+  Methods,
+  Request,
+  Response,
+  contentTypes,
+  jsonResponse,
+} from '@browserless.io/browserless';
+
+// @TODO Figure out how to parse the Protocol JSON into a TS definition
+// for our openapi docs.
+export type ResponseSchema = object;
+
+export default class GetJSONVersion extends HTTPRoute {
+  accepts = [contentTypes.any];
+  auth = true;
+  browser = null;
+  concurrency = false;
+  contentTypes = [contentTypes.json];
+  description = `Returns Protocol JSON meta-data that Chrome comes with.`;
+  method = Methods.get;
+  path = HTTPRoutes.jsonProtocol;
+  tags = [APITags.browserAPI];
+
+  private cachedProtocol: object | undefined;
+
+  handler = async (_req: Request, res: Response): Promise<void> => {
+    const browserManager = this.browserManager();
+
+    if (!this.cachedProtocol) {
+      this.cachedProtocol = await browserManager.getProtocolJSON();
+    }
+
+    return jsonResponse(res, 200, this.cachedProtocol);
+  };
+}

package/src/routes/chromium/http/json-version-get.ts

@@ -0,0 +1,55 @@
+import {
+  APITags,
+  BrowserManager,
+  HTTPRoute,
+  HTTPRoutes,
+  Methods,
+  Request,
+  Response,
+  UnwrapPromise,
+  contentTypes,
+  jsonResponse,
+  writeResponse,
+} from '@browserless.io/browserless';
+
+export type ResponseSchema = UnwrapPromise<
+  ReturnType<BrowserManager['getVersionJSON']>
+>;
+
+export default class GetJSONVersion extends HTTPRoute {
+  accepts = [contentTypes.any];
+  auth = true;
+  browser = null;
+  concurrency = false;
+  contentTypes = [contentTypes.json];
+  description = `Returns a JSON payload that acts as a pass-through to the DevTools /json/version protocol in Chrome.`;
+  method = Methods.get;
+  path = HTTPRoutes.jsonVersion;
+  tags = [APITags.browserAPI];
+
+  private cachedJSON: ResponseSchema | undefined;
+
+  handler = async (req: Request, res: Response): Promise<void> => {
+    const baseUrl = req.parsed.host;
+    const protocol = req.parsed.protocol.includes('s') ? 'wss' : 'ws';
+    const browserManager = this.browserManager();
+
+    try {
+      if (!this.cachedJSON) {
+        this.cachedJSON = {
+          ...(await browserManager.getVersionJSON()),
+          webSocketDebuggerUrl: `${protocol}://${baseUrl}`,
+        };
+      }
+
+      return jsonResponse(res, 200, this.cachedJSON);
+    } catch (err) {
+      return writeResponse(
+        res,
+        500,
+        'There was an error handling your request',
+        contentTypes.text,
+      );
+    }
+  };
+}