@browserless.io/browserless 2.1.0 → 2.2.0-beta-2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@browserless.io/browserless",
-  "version": "2.1.0",
+  "version": "2.2.0-beta-2",
   "license": "SSPL",
   "description": "The browserless platform",
   "author": "browserless.io",
@@ -25,8 +25,7 @@
     "dev": "npm run build:dev && env-cmd -f .env node build",
     "install:adblock": "node scripts/install-adblock.js",
     "install:browsers": "npx --yes playwright install chromium firefox webkit",
-    "install:cdp-json": "node scripts/install-cdp-json.js",
-    "install:dev": "npm run install:browsers && npm run install:cdp-json",
+    "install:dev": "npm run install:browsers",
     "lint": "eslint . --ext .ts --fix",
     "prepack": "npm run build:dev",
     "prettier": "prettier '{src,functions,scripts,bin,external,bin}/**/*.{js,ts,json}' --log-level error --write",
@@ -43,7 +42,6 @@
     "scripts/*",
     "src/*",
     "static/*",
-    "browser.json",
     "CHANGELOG.md",
     "tsconfig.json"
   ],
@@ -57,8 +55,8 @@
     "http-proxy": "^1.18.1",
     "lighthouse": "^11.1.0",
     "micromatch": "^4.0.4",
-    "playwright-core": "^1.41.1",
-    "puppeteer-core": "^21.6.1",
+    "playwright-core": "^1.41.2",
+    "puppeteer-core": "^21.10.0",
     "puppeteer-extra": "^3.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2",
     "queue": "^7.0.0",
@@ -71,24 +69,24 @@
     "@types/http-proxy": "^1.17.14",
     "@types/micromatch": "^4.0.6",
     "@types/mocha": "^10.0.6",
-    "@types/node": "^20.11.5",
+    "@types/node": "^20.11.16",
     "@types/sinon": "^17.0.3",
-    "@typescript-eslint/eslint-plugin": "^6.19.1",
-    "@typescript-eslint/parser": "^6.19.1",
+    "@typescript-eslint/eslint-plugin": "^6.20.0",
+    "@typescript-eslint/parser": "^6.21.0",
     "assert": "^2.0.0",
-    "chai": "^5.0.0",
+    "chai": "^5.0.3",
     "cross-env": "^7.0.3",
     "env-cmd": "^10.1.0",
-    "esbuild": "^0.19.11",
+    "esbuild": "^0.20.0",
     "esbuild-plugin-polyfill-node": "^0.3.0",
     "eslint": "^8.56.0",
     "eslint-plugin-import": "^2.29.1",
     "eslint-plugin-typescript-sort-keys": "^3.1.0",
     "extract-zip": "^2.0.1",
-    "marked": "^11.1.0",
+    "marked": "^12.0.0",
     "mocha": "^10.0.0",
     "move-file": "^3.1.0",
-    "prettier": "^3.2.4",
+    "prettier": "^3.2.5",
     "sinon": "^17.0.1",
     "ts-node": "^10.9.2",
     "typescript": "^5.3.3",

package/src/browserless.ts

@@ -3,6 +3,7 @@ import {
   BrowserHTTPRoute,
   BrowserManager,
   BrowserWebsocketRoute,
+  CDPChromium,
   Config,
   FileSystem,
   HTTPRoute,
@@ -11,6 +12,9 @@ import {
   Limiter,
   Metrics,
   Monitoring,
+  PlaywrightChromium,
+  PlaywrightFirefox,
+  PlaywrightWebkit,
   Router,
   Token,
   WebHooks,
@@ -158,6 +162,12 @@ export class Browserless {
   public async start() {
     const httpRoutes: Array<HTTPRoute | BrowserHTTPRoute> = [];
     const wsRoutes: Array<WebSocketRoute | BrowserWebsocketRoute> = [];
+    const internalBrowsers = [
+      CDPChromium,
+      PlaywrightFirefox,
+      PlaywrightChromium,
+      PlaywrightWebkit,
+    ];
 
     const [[httpRouteFiles, wsRouteFiles], installedBrowsers] =
       await Promise.all([getRouteFiles(this.config), availableBrowsers]);
@@ -254,11 +264,12 @@ export class Browserless {
       }
     }
 
-    // Validate that browsers are installed and route paths are unique
+    // Validate that we have the browsers they are asking for
     [...httpRoutes, ...wsRoutes].forEach((route) => {
       if (
         'browser' in route &&
         route.browser &&
+        internalBrowsers.includes(route.browser) &&
         !installedBrowsers.some((b) => b.name === route.browser?.name)
       ) {
         throw new Error(

package/src/browsers/cdp-chromium.ts

@@ -314,7 +314,7 @@ export class CDPChromium extends EventEmitter {
 
   public wsEndpoint = (): string | null => this.browserWSEndpoint;
 
-  public publicWSEndpoint = (token: string): string | null => {
+  public publicWSEndpoint = (token: string | null): string | null => {
     if (!this.browserWSEndpoint) {
       return null;
     }
@@ -344,16 +344,14 @@ export class CDPChromium extends EventEmitter {
       }
       socket.once('close', resolve);
 
-      this.debug(
-        `Proxying ${req.parsed.href} to browser ${this.browserWSEndpoint}`,
-      );
-
       const [page] = await this.browser.pages();
       const pageLocation = `/devtools/page/${this.getPageId(page)}`;
 
       this.debug(`Proxying ${req.parsed.href} to page "${pageLocation}"`);
 
-      req.url = pageLocation;
+      const target = new URL(pageLocation, this.browserWSEndpoint).href;
+
+      req.url = '';
 
       this.proxy.ws(
         req,
@@ -361,7 +359,7 @@ export class CDPChromium extends EventEmitter {
         head,
         {
           changeOrigin: true,
-          target: this.browserWSEndpoint,
+          target,
         },
         (error) => {
           this.debug(`Error proxying session: ${error}`);

package/src/browsers/index.ts

@@ -11,27 +11,33 @@ import {
   Config,
   HTTPManagementRoutes,
   NotFound,
+  PlaywrightChromium,
+  PlaywrightFirefox,
+  PlaywrightWebkit,
   Request,
-  ServerError,
   browserHook,
   convertIfBase64,
   createLogger,
   exists,
-  id,
+  generateDataDir,
   makeExternalURL,
   noop,
   pageHook,
   parseBooleanParam,
 } from '@browserless.io/browserless';
-import path, { join } from 'path';
 import { deleteAsync } from 'del';
-import { mkdir } from 'fs/promises';
+import path from 'path';
 
 export class BrowserManager {
   protected browsers: Map<BrowserInstance, BrowserlessSession> = new Map();
   protected launching: Map<string, Promise<unknown>> = new Map();
   protected timers: Map<string, number> = new Map();
   protected debug = createLogger('browser-manager');
+  protected playwrightBrowserNames = [
+    PlaywrightChromium.name,
+    PlaywrightFirefox.name,
+    PlaywrightWebkit.name,
+  ];
 
   constructor(protected config: Config) {}
 
@@ -46,40 +52,28 @@ export class BrowserManager {
     }
   };
 
-  /**
-   * Generates a directory for the user-data-dir contents to be saved in. Uses
-   * the provided sessionId, or creates one when omitted,
-   * and appends it to the name of the directory. If the
-   * directory already exists then no action is taken, verified by run `stat`
-   *
-   * @param sessionId The ID of the session
-   * @returns Promise<string> of the fully-qualified path of the directory
-   */
-  protected generateDataDir = async (
-    sessionId: string = id(),
-  ): Promise<string> => {
-    const baseDirectory = await this.config.getDataDir();
-    const dataDirPath = join(
-      baseDirectory,
-      `browserless-data-dir-${sessionId}`,
-    );
+  public getProtocolJSON = async (): Promise<object> => {
+    this.debug(`Launching Chrome to generate /json/protocol results`);
+    const browser = new CDPChromium({
+      blockAds: false,
+      config: this.config,
+      record: false,
+      userDataDir: null,
+    });
+    await browser.launch();
+    const wsEndpoint = browser.wsEndpoint();
 
-    if (await exists(dataDirPath)) {
-      this.debug(
-        `Data directory already exists, not creating "${dataDirPath}"`,
-      );
-      return dataDirPath;
+    if (!wsEndpoint) {
+      throw new Error('There was an error launching the browser');
     }
 
-    this.debug(`Generating user-data-dir at ${dataDirPath}`);
+    const { port } = new URL(wsEndpoint);
+    const res = await fetch(`http://127.0.0.1:${port}/json/protocol`);
+    const protocolJSON = await res.json();
 
-    await mkdir(dataDirPath, { recursive: true }).catch((err) => {
-      throw new ServerError(
-        `Error creating data-directory "${dataDirPath}": ${err}`,
-      );
-    });
+    browser.close();
 
-    return dataDirPath;
+    return protocolJSON;
   };
 
   public getVersionJSON = async (): Promise<{
@@ -91,6 +85,7 @@ export class BrowserManager {
     'WebKit-Version': string;
     webSocketDebuggerUrl: string;
   }> => {
+    this.debug(`Launching Chrome to generate /json/version results`);
     const browser = new CDPChromium({
       blockAds: false,
       config: this.config,
@@ -280,7 +275,9 @@ export class BrowserManager {
     // unless it's playwright which takes care of its own data-dirs
     const userDataDir =
       manualUserDataDir ||
-      (Browser.name === CDPChromium.name ? await this.generateDataDir() : null);
+      (!this.playwrightBrowserNames.includes(Browser.name)
+        ? await generateDataDir(undefined, this.config)
+        : null);
 
     const proxyServerArg = launchOptions.args?.find((arg) =>
       arg.includes('--proxy-server='),

package/src/browsers/playwright-chromium.ts

@@ -106,7 +106,7 @@ export class PlaywrightChromium extends EventEmitter {
 
   public wsEndpoint = (): string | null => this.browserWSEndpoint;
 
-  public publicWSEndpoint = (token: string): string | null => {
+  public publicWSEndpoint = (token: string | null): string | null => {
     if (!this.browserWSEndpoint) {
       return null;
     }

package/src/browsers/playwright-firefox.ts

@@ -99,7 +99,7 @@ export class PlaywrightFirefox extends EventEmitter {
 
   public wsEndpoint = (): string | null => this.browserWSEndpoint;
 
-  public publicWSEndpoint = (token: string): string | null => {
+  public publicWSEndpoint = (token: string | null): string | null => {
     if (!this.browserWSEndpoint) {
       return null;
     }

package/src/browsers/playwright-webkit.ts

@@ -99,7 +99,7 @@ export class PlaywrightWebkit extends EventEmitter {
 
   public wsEndpoint = (): string | null => this.browserWSEndpoint;
 
-  public publicWSEndpoint = (token: string): string | null => {
+  public publicWSEndpoint = (token: string | null): string | null => {
     if (!this.browserWSEndpoint) {
       return null;
     }

package/src/constants.ts

@@ -2,3 +2,4 @@ export const encryptionAlgo = 'aes-192-cbc';
 export const encryptionSep = '.';
 export const liveURLSep = ':';
 export const keyLength = 24;
+export const BLESS_PAGE_IDENTIFIER = 'BLESS';

package/src/http.ts

@@ -92,6 +92,9 @@ export enum HTTPRoutes {
   content = '/content',
   download = '/download',
   function = '/function',
+  jsonList = '/json/list',
+  jsonNew = '/json/new',
+  jsonProtocol = '/json/protocol',
   jsonVersion = '/json/version',
   pdf = '/pdf',
   performance = '/performance',

package/src/routes/chromium/http/json-list.ts

@@ -0,0 +1,50 @@
+import {
+  APITags,
+  HTTPRoute,
+  HTTPRoutes,
+  Methods,
+  Request,
+  Response,
+  contentTypes,
+  dedent,
+  jsonResponse,
+} from '@browserless.io/browserless';
+import { getCDPJSONPayload } from '../utils/cdp.js';
+
+/*
+Example Payload from Chrome:
+  [{
+    "description": "",
+    "devtoolsFrontendUrl": "/devtools/inspector.html?ws=localhost:9222/devtools/page/6CA38A3E207BA534C674D1057B19E9CC",
+    "id": "6CA38A3E207BA534C674D1057B19E9CC",
+    "title": "New Tab",
+    "type": "page",
+    "url": "http://localhost:9222/json/list",
+    "webSocketDebuggerUrl": "ws://localhost:9222/devtools/page/6CA38A3E207BA534C674D1057B19E9CC"
+  }]
+*/
+export type ResponseSchema = Array<ReturnType<typeof getCDPJSONPayload>>;
+
+export default class GetJSONList extends HTTPRoute {
+  accepts = [contentTypes.any];
+  auth = true;
+  browser = null;
+  concurrency = false;
+  contentTypes = [contentTypes.json];
+  description = dedent(`
+    Returns a JSON payload that acts as a pass-through to the DevTools /json/list HTTP API in Chromium.
+    Browserless mocks this payload so that remote clients can connect to the underlying "webSocketDebuggerUrl"
+    which will cause Browserless to start the browser and proxy that request into a blank page.
+  `);
+  method = Methods.get;
+  path = HTTPRoutes.jsonList;
+  tags = [APITags.browserAPI];
+
+  handler = async (_req: Request, res: Response): Promise<void> => {
+    const config = this.config();
+    const externalAddress = config.getExternalAddress();
+    const payload = getCDPJSONPayload(externalAddress);
+
+    return jsonResponse(res, 200, [payload] as ResponseSchema);
+  };
+}

package/src/routes/chromium/http/json-new.ts

@@ -0,0 +1,50 @@
+import {
+  APITags,
+  HTTPRoute,
+  HTTPRoutes,
+  Methods,
+  Request,
+  Response,
+  contentTypes,
+  dedent,
+  jsonResponse,
+} from '@browserless.io/browserless';
+import { getCDPJSONPayload } from '../utils/cdp.js';
+
+/*
+Example Payload from Chrome:
+{
+  "description": "",
+  "devtoolsFrontendUrl": "/devtools/inspector.html?ws=localhost:9222/devtools/page/2F76525C32A916DF30C4F37A4970B8BF",
+  "id": "2F76525C32A916DF30C4F37A4970B8BF",
+  "title": "",
+  "type": "page",
+  "url": "about:blank",
+  "webSocketDebuggerUrl": "ws://localhost:9222/devtools/page/2F76525C32A916DF30C4F37A4970B8BF"
+}
+*/
+export type ResponseSchema = ReturnType<typeof getCDPJSONPayload>;
+
+export default class GetJSONList extends HTTPRoute {
+  accepts = [contentTypes.any];
+  auth = true;
+  browser = null;
+  concurrency = false;
+  contentTypes = [contentTypes.json];
+  description = dedent(`
+    Returns a JSON payload that acts as a pass-through to the DevTools /json/list HTTP API in Chromium.
+    Browserless mocks this payload so that remote clients can connect to the underlying "webSocketDebuggerUrl"
+    which will cause Browserless to start the browser and proxy that request into a blank page.
+  `);
+  method = Methods.put;
+  path = HTTPRoutes.jsonNew;
+  tags = [APITags.browserAPI];
+
+  handler = async (_req: Request, res: Response): Promise<void> => {
+    const config = this.config();
+    const externalAddress = config.getExternalAddress();
+    const payload = getCDPJSONPayload(externalAddress);
+
+    return jsonResponse(res, 200, payload);
+  };
+}

package/src/routes/chromium/http/json-protocol-get.ts

@@ -0,0 +1,38 @@
+import {
+  APITags,
+  HTTPRoute,
+  HTTPRoutes,
+  Methods,
+  Request,
+  Response,
+  contentTypes,
+  jsonResponse,
+} from '@browserless.io/browserless';
+
+// @TODO Figure out how to parse the Protocol JSON into a TS definition
+// for our openapi docs.
+export type ResponseSchema = object;
+
+export default class GetJSONVersion extends HTTPRoute {
+  accepts = [contentTypes.any];
+  auth = true;
+  browser = null;
+  concurrency = false;
+  contentTypes = [contentTypes.json];
+  description = `Returns Protocol JSON meta-data that Chrome comes with.`;
+  method = Methods.get;
+  path = HTTPRoutes.jsonProtocol;
+  tags = [APITags.browserAPI];
+
+  private cachedProtocol: object | undefined;
+
+  handler = async (_req: Request, res: Response): Promise<void> => {
+    const browserManager = this.browserManager();
+
+    if (!this.cachedProtocol) {
+      this.cachedProtocol = await browserManager.getProtocolJSON();
+    }
+
+    return jsonResponse(res, 200, this.cachedProtocol);
+  };
+}

package/src/routes/chromium/http/json-version-get.ts

@@ -1,8 +1,8 @@
 import {
   APITags,
   BrowserManager,
-  HTTPRoutes,
   HTTPRoute,
+  HTTPRoutes,
   Methods,
   Request,
   Response,

package/src/routes/chromium/utils/cdp.ts

@@ -0,0 +1,19 @@
+import { CDPJSONPayload, pageID } from '@browserless.io/browserless';
+
+export const getCDPJSONPayload = (externalAddress: string): CDPJSONPayload => {
+  const id = pageID();
+  const { protocol, host, pathname, href } = new URL(
+    `/devtools/page/${id}`,
+    externalAddress,
+  );
+
+  return {
+    description: '',
+    devtoolsFrontendUrl: `/devtools/inspector.html?${protocol.replace(':', '')}=${host}${pathname}`,
+    id,
+    title: 'New Tab',
+    type: 'page',
+    url: 'about:blank',
+    webSocketDebuggerUrl: href,
+  };
+};

package/src/types.ts

@@ -556,3 +556,41 @@ export interface IBrowserlessStats {
   unhealthy: number;
   units: number;
 }
+
+export interface CDPJSONPayload {
+  /**
+   * The description of the target. Generally the page's title.
+   */
+  description: string;
+
+  /**
+   * The fully-qualified URL of the Devtools inspector app.
+   */
+  devtoolsFrontendUrl: string;
+
+  /**
+   * A Unique Id for the underlying target.
+   */
+  id: string;
+
+  /**
+   * The title of the target. For pages this is the page's title.
+   */
+  title: string;
+
+  /**
+   * The type of target, generally "page" or "background_page".
+   */
+  type: string;
+
+  /**
+   * The current URL the target is consuming or visiting.
+   */
+  url: string;
+
+  /**
+   * The target or page's WebSocket Debugger URL. Primarily used for legacy
+   * libraries to connect and inspect or remote automate this target.
+   */
+  webSocketDebuggerUrl: string;
+}

package/src/utils.ts

@@ -1,5 +1,6 @@
 import * as fs from 'fs/promises';
 import {
+  BLESS_PAGE_IDENTIFIER,
   CDPChromium,
   Config,
   PlaywrightChromium,
@@ -52,6 +53,23 @@ export const jsExtension = '.js';
 
 export const id = (): string => crypto.randomUUID();
 
+/**
+ * Generates a random, Chrome-compliant page ID with "BLESS"
+ * prepended. This prepended text signals to other parts of the
+ * system that this is a Browserless-created ID so it can be appropriately
+ * handled.
+ *
+ * @returns {string} A random Page ID
+ */
+export const pageID = (): string => {
+  const chars = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ';
+  const id = Array.from({ length: 32 - BLESS_PAGE_IDENTIFIER.length })
+    .map(() => chars[Math.floor(Math.random() * chars.length)])
+    .join('');
+
+  return `${BLESS_PAGE_IDENTIFIER}${id}`;
+};
+
 export const createLogger = (domain: string): debug.Debugger => {
   return debug(`browserless.io:${domain}`);
 };
@@ -224,15 +242,45 @@ export const removeNullStringify = (
   json: unknown,
   allowNull = true,
 ): string => {
-  return JSON.stringify(json, (_key, value) => {
-    if (allowNull) return value;
-    if (value !== null) return value;
-  });
+  return JSON.stringify(
+    json,
+    (_key, value) => {
+      if (allowNull) return value;
+      if (value !== null) return value;
+    },
+    '  ',
+  );
 };
 
 export const jsonOrString = (maybeJson: string): unknown | string =>
   safeParse(maybeJson) ?? maybeJson;
 
+export const generateDataDir = async (
+  sessionId: string = id(),
+  config: Config,
+): Promise<string> => {
+  const baseDirectory = await config.getDataDir();
+  const dataDirPath = path.join(
+    baseDirectory,
+    `browserless-data-dir-${sessionId}`,
+  );
+
+  if (await exists(dataDirPath)) {
+    debug(`Data directory already exists, not creating "${dataDirPath}"`);
+    return dataDirPath;
+  }
+
+  debug(`Generating user-data-dir at ${dataDirPath}`);
+
+  await fs.mkdir(dataDirPath, { recursive: true }).catch((err) => {
+    throw new ServerError(
+      `Error creating data-directory "${dataDirPath}": ${err}`,
+    );
+  });
+
+  return dataDirPath;
+};
+
 export const readBody = async (
   req: Request,
 ): Promise<ReturnType<typeof safeParse>> => {