@browserless.io/browserless 2.2.0-beta-8 → 2.3.0-beta-1

package/scripts/install-devtools.js

@@ -0,0 +1,40 @@
+import { rename, rm, unlink } from 'fs/promises';
+import extract from 'extract-zip';
+import { fileURLToPath } from 'url';
+import fs from 'fs';
+import https from 'https';
+import path from 'path';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+// @TODO get this link dynamically
+const devtoolsUrl = `https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac%2F848005%2Fdevtools-frontend.zip?alt=media`;
+
+const staticDir = path.join(__dirname, '..', 'static');
+const zipPath = path.join(staticDir, 'devtools.zip');
+const extractPath = path.join(staticDir, 'devtools-temp');
+const finalPath = path.join(staticDir, 'devtools');
+const deepPath = path.join(
+  extractPath,
+  'devtools-frontend',
+  'resources',
+  'inspector',
+);
+const zipStream = fs.createWriteStream(zipPath);
+
+const cleanup = async () => {
+  return await Promise.all([
+    unlink(zipPath),
+    rm(extractPath, { recursive: true }),
+  ]);
+};
+
+(async () => {
+  await rm(finalPath, { recursive: true }).catch(() => {});
+  await new Promise((resolve, reject) =>
+    https.get(devtoolsUrl, (response) => {
+      response.pipe(zipStream).on('close', resolve).on('error', reject);
+    }),
+  );
+  await extract(zipPath, { dir: extractPath });
+  await rename(deepPath, finalPath);
+})().finally(cleanup);

package/src/browserless.ts

@@ -3,22 +3,23 @@ import {
   BrowserHTTPRoute,
   BrowserManager,
   BrowserWebsocketRoute,
-  CDPChromium,
+  ChromeCDP,
+  ChromiumCDP,
+  ChromiumPlaywright,
   Config,
   FileSystem,
+  FirefoxPlaywright,
   HTTPRoute,
   HTTPServer,
   IBrowserlessStats,
   Limiter,
   Metrics,
   Monitoring,
-  PlaywrightChromium,
-  PlaywrightFirefox,
-  PlaywrightWebkit,
   Router,
   Token,
   WebHooks,
   WebSocketRoute,
+  WebkitPlaywright,
   availableBrowsers,
   createLogger,
   getRouteFiles,
@@ -35,6 +36,12 @@ type Implements<T> = {
   new (...args: unknown[]): T;
 };
 
+type RouteTypes =
+  | typeof HTTPRoute
+  | typeof BrowserHTTPRoute
+  | typeof WebSocketRoute
+  | typeof BrowserWebsocketRoute;
+
 export class Browserless {
   protected debug: debug.Debugger = createLogger('index');
   protected browserManager: BrowserManager;
@@ -47,6 +54,7 @@ export class Browserless {
   protected token: Token;
   protected webhooks: WebHooks;
 
+  DisabledRoutes: RouteTypes[] = [];
   webSocketRouteFiles: string[] = [];
   httpRouteFiles: string[] = [];
   server?: HTTPServer;
@@ -137,6 +145,15 @@ export class Browserless {
     );
   };
 
+  private routeIsDisabled(Route: RouteTypes) {
+    const Stringified = Route.toString();
+    return this.DisabledRoutes.some((r) => r.toString() === Stringified);
+  }
+
+  public disableRoute(Route: RouteTypes) {
+    this.DisabledRoutes.push(Route);
+  }
+
   public addHTTPRoute(httpRouteFilePath: string) {
     this.httpRouteFiles.push(httpRouteFilePath);
   }
@@ -163,13 +180,14 @@ export class Browserless {
     const httpRoutes: Array<HTTPRoute | BrowserHTTPRoute> = [];
     const wsRoutes: Array<WebSocketRoute | BrowserWebsocketRoute> = [];
     const internalBrowsers = [
-      CDPChromium,
-      PlaywrightFirefox,
-      PlaywrightChromium,
-      PlaywrightWebkit,
+      ChromiumCDP,
+      ChromeCDP,
+      FirefoxPlaywright,
+      ChromiumPlaywright,
+      WebkitPlaywright,
     ];
 
-    const [[httpRouteFiles, wsRouteFiles], installedBrowsers] =
+    const [[internalHttpRouteFiles, internalWsRouteFiles], installedBrowsers] =
       await Promise.all([getRouteFiles(this.config), availableBrowsers]);
 
     const docsLink = makeExternalURL(this.config.getExternalAddress(), '/docs');
@@ -178,7 +196,10 @@ export class Browserless {
     this.debug(`Running as user "${userInfo().username}"`);
     this.debug('Starting import of HTTP Routes');
 
-    for (const httpRoute of [...httpRouteFiles, ...this.httpRouteFiles]) {
+    for (const httpRoute of [
+      ...internalHttpRouteFiles,
+      ...this.httpRouteFiles,
+    ]) {
       if (httpRoute.endsWith('js')) {
         const { name } = path.parse(httpRoute);
         const [bodySchema, querySchema] = await Promise.all(
@@ -199,28 +220,33 @@ export class Browserless {
           default: Route,
         }: { default: Implements<HTTPRoute> | Implements<BrowserHTTPRoute> } =
           await import(routeImport + `?cb=${Date.now()}`);
-        const route = new Route(
-          this.browserManager,
-          this.config,
-          this.fileSystem,
-          logger,
-          this.metrics,
-          this.monitoring,
-        );
-        route.bodySchema = safeParse(bodySchema);
-        route.querySchema = safeParse(querySchema);
-        route.config = () => this.config;
-        route.metrics = () => this.metrics;
-        route.monitoring = () => this.monitoring;
-        route.fileSystem = () => this.fileSystem;
-        route.debug = () => logger;
-
-        httpRoutes.push(route);
+        if (!this.routeIsDisabled(Route)) {
+          const route = new Route(
+            this.browserManager,
+            this.config,
+            this.fileSystem,
+            logger,
+            this.metrics,
+            this.monitoring,
+          );
+          route.bodySchema = safeParse(bodySchema);
+          route.querySchema = safeParse(querySchema);
+          route.config = () => this.config;
+          route.metrics = () => this.metrics;
+          route.monitoring = () => this.monitoring;
+          route.fileSystem = () => this.fileSystem;
+          route.debug = () => logger;
+
+          httpRoutes.push(route);
+        }
       }
     }
 
     this.debug('Starting import of WebSocket Routes');
-    for (const wsRoute of [...wsRouteFiles, ...this.webSocketRouteFiles]) {
+    for (const wsRoute of [
+      ...internalWsRouteFiles,
+      ...this.webSocketRouteFiles,
+    ]) {
       if (wsRoute.endsWith('js')) {
         const { name } = path.parse(wsRoute);
         const [, querySchema] = await Promise.all(
@@ -244,23 +270,24 @@ export class Browserless {
             | Implements<WebSocketRoute>
             | Implements<BrowserWebsocketRoute>;
         } = await import(wsImport + `?cb=${Date.now()}`);
+        if (!this.routeIsDisabled(Route)) {
+          const route = new Route(
+            this.browserManager,
+            this.config,
+            this.fileSystem,
+            logger,
+            this.metrics,
+            this.monitoring,
+          );
+          route.querySchema = safeParse(querySchema);
+          route.config = () => this.config;
+          route.metrics = () => this.metrics;
+          route.monitoring = () => this.monitoring;
+          route.fileSystem = () => this.fileSystem;
+          route.debug = () => logger;
 
-        const route = new Route(
-          this.browserManager,
-          this.config,
-          this.fileSystem,
-          logger,
-          this.metrics,
-          this.monitoring,
-        );
-        route.querySchema = safeParse(querySchema);
-        route.config = () => this.config;
-        route.metrics = () => this.metrics;
-        route.monitoring = () => this.monitoring;
-        route.fileSystem = () => this.fileSystem;
-        route.debug = () => logger;
-
-        wsRoutes.push(route);
+          wsRoutes.push(route);
+        }
       }
     }
 
@@ -273,7 +300,7 @@ export class Browserless {
         !installedBrowsers.some((b) => b.name === route.browser?.name)
       ) {
         throw new Error(
-          `Couldn't load route "${route.path}" due to missing browser of "${route.browser?.name}"`,
+          `Couldn't load route "${route.path}" due to missing browser binary for "${route.browser?.name}"`,
         );
       }
     });

package/src/browsers/chrome.cdp.ts

@@ -0,0 +1,10 @@
+import {
+  chromeExecutablePath,
+  createLogger,
+} from '@browserless.io/browserless';
+import { ChromiumCDP } from './chromium.cdp.js';
+
+export class ChromeCDP extends ChromiumCDP {
+  protected executablePath = chromeExecutablePath;
+  protected debug = createLogger('browsers:chrome:cdp');
+}

package/src/browsers/chrome.playwright.ts

@@ -0,0 +1,10 @@
+import {
+  chromeExecutablePath,
+  createLogger,
+} from '@browserless.io/browserless';
+import { ChromiumPlaywright } from './chromium.playwright.js';
+
+export class ChromePlaywright extends ChromiumPlaywright {
+  protected executablePath = chromeExecutablePath;
+  protected debug = createLogger('browsers:chrome:playwright');
+}

package/src/browsers/{cdp-chromium.ts → chromium.cdp.ts}

@@ -1,4 +1,5 @@
 import {
+  BLESS_PAGE_IDENTIFIER,
   CDPLaunchOptions,
   Config,
   Request,
@@ -23,7 +24,7 @@ import puppeteerStealth from 'puppeteer-extra';
 
 puppeteerStealth.use(StealthPlugin());
 
-export class CDPChromium extends EventEmitter {
+export class ChromiumCDP extends EventEmitter {
   protected config: Config;
   protected userDataDir: string | null;
   protected record: boolean;
@@ -32,8 +33,9 @@ export class CDPChromium extends EventEmitter {
   protected browser: Browser | null = null;
   protected browserWSEndpoint: string | null = null;
   protected port?: number;
-  protected debug = createLogger('browsers:cdp:chromium');
+  protected debug = createLogger('browsers:chromium:cdp');
   protected proxy = httpProxy.createProxyServer();
+  protected executablePath = playwright.chromium.executablePath();
 
   constructor({
     userDataDir,
@@ -44,7 +46,7 @@ export class CDPChromium extends EventEmitter {
     blockAds: boolean;
     config: Config;
     record: boolean;
-    userDataDir: CDPChromium['userDataDir'];
+    userDataDir: ChromiumCDP['userDataDir'];
   }) {
     super();
 
@@ -263,7 +265,7 @@ export class CDPChromium extends EventEmitter {
         ...(options.args || []),
         this.userDataDir ? `--user-data-dir=${this.userDataDir}` : '',
       ].filter((_) => !!_),
-      executablePath: playwright.chromium.executablePath(),
+      executablePath: this.executablePath,
     };
 
     if (this.record || this.blockAds) {
@@ -302,7 +304,7 @@ export class CDPChromium extends EventEmitter {
       ? puppeteerStealth.launch.bind(puppeteerStealth)
       : puppeteer.launch.bind(puppeteer);
 
-    this.debug(finalOptions, `Launching CDP Chrome Handler`);
+    this.debug(finalOptions, `Launching CDP Handler`);
     // @ts-ignore mis-matched types from stealth...
     this.browser = (await launch(finalOptions)) as Browser;
     this.browser.on('targetcreated', this.onTargetCreated);
@@ -322,7 +324,7 @@ export class CDPChromium extends EventEmitter {
 
     const serverURL = new URL(this.config.getExternalWebSocketAddress());
     const wsURL = new URL(this.browserWSEndpoint);
-    wsURL.hostname = serverURL.hostname;
+    wsURL.host = serverURL.host;
     wsURL.port = serverURL.port;
 
     if (token) {
@@ -344,16 +346,21 @@ export class CDPChromium extends EventEmitter {
         );
       }
       socket.once('close', resolve);
+      this.debug(`Proxying ${req.parsed.href}`);
 
-      const [page] = await this.browser.pages();
-      const pageLocation = `/devtools/page/${this.getPageId(page)}`;
-
-      this.debug(`Proxying ${req.parsed.href} to page "${pageLocation}"`);
-
-      const target = new URL(pageLocation, this.browserWSEndpoint).href;
-
+      const shouldMakePage = req.parsed.pathname.includes(
+        BLESS_PAGE_IDENTIFIER,
+      );
+      const page = shouldMakePage ? await this.browser.newPage() : null;
+      const pathname = page
+        ? path.join('/devtools', '/page', this.getPageId(page))
+        : req.parsed.pathname;
+      const target = new URL(pathname, this.browserWSEndpoint).href;
       req.url = '';
 
+      // Delete headers known to cause issues
+      delete req.headers.origin;
+
       this.proxy.ws(
         req,
         socket,

package/src/browsers/{playwright-chromium.ts → chromium.playwright.ts}

@@ -10,7 +10,7 @@ import { Duplex } from 'stream';
 import { EventEmitter } from 'events';
 import httpProxy from 'http-proxy';
 
-export class PlaywrightChromium extends EventEmitter {
+export class ChromiumPlaywright extends EventEmitter {
   protected config: Config;
   protected userDataDir: string | null;
   protected record: boolean;
@@ -18,7 +18,8 @@ export class PlaywrightChromium extends EventEmitter {
   protected proxy = httpProxy.createProxyServer();
   protected browser: playwright.BrowserServer | null = null;
   protected browserWSEndpoint: string | null = null;
-  protected debug = createLogger('browsers:playwright:chromium');
+  protected debug = createLogger('browsers:chromium:playwright');
+  protected executablePath = playwright.chromium.executablePath();
 
   constructor({
     config,
@@ -27,7 +28,7 @@ export class PlaywrightChromium extends EventEmitter {
   }: {
     config: Config;
     record: boolean;
-    userDataDir: PlaywrightChromium['userDataDir'];
+    userDataDir: ChromiumPlaywright['userDataDir'];
   }) {
     super();
 
@@ -79,7 +80,7 @@ export class PlaywrightChromium extends EventEmitter {
   public launch = async (
     options: BrowserServerOptions = {},
   ): Promise<playwright.BrowserServer> => {
-    this.debug(`Launching Chrome Handler`);
+    this.debug(`Launching Playwright Handler`);
 
     if (this.record) {
       throw new ServerError(`Recording is not yet available with this browser`);
@@ -92,7 +93,7 @@ export class PlaywrightChromium extends EventEmitter {
         ...(options.args || []),
         this.userDataDir ? `--user-data-dir=${this.userDataDir}` : '',
       ],
-      executablePath: playwright.chromium.executablePath(),
+      executablePath: this.executablePath,
     });
 
     const browserWSEndpoint = this.browser.wsEndpoint();
@@ -113,7 +114,7 @@ export class PlaywrightChromium extends EventEmitter {
 
     const serverURL = new URL(this.config.getExternalWebSocketAddress());
     const wsURL = new URL(this.browserWSEndpoint);
-    wsURL.hostname = serverURL.hostname;
+    wsURL.host = serverURL.host;
     wsURL.port = serverURL.port;
 
     if (token) {

package/src/browsers/{playwright-firefox.ts → firefox.playwright.ts}

@@ -10,7 +10,7 @@ import { Duplex } from 'stream';
 import { EventEmitter } from 'events';
 import httpProxy from 'http-proxy';
 
-export class PlaywrightFirefox extends EventEmitter {
+export class FirefoxPlaywright extends EventEmitter {
   protected config: Config;
   protected userDataDir: string | null;
   protected record: boolean;
@@ -18,7 +18,7 @@ export class PlaywrightFirefox extends EventEmitter {
   protected proxy = httpProxy.createProxyServer();
   protected browser: playwright.BrowserServer | null = null;
   protected browserWSEndpoint: string | null = null;
-  protected debug = createLogger('browsers:playwright:firefox');
+  protected debug = createLogger('browsers:firefox:playwright');
 
   constructor({
     config,
@@ -27,7 +27,7 @@ export class PlaywrightFirefox extends EventEmitter {
   }: {
     config: Config;
     record: boolean;
-    userDataDir: PlaywrightFirefox['userDataDir'];
+    userDataDir: FirefoxPlaywright['userDataDir'];
   }) {
     super();
 
@@ -77,7 +77,7 @@ export class PlaywrightFirefox extends EventEmitter {
       throw new ServerError(`Recording is not yet available with this browser`);
     }
 
-    this.debug(`Launching Firefox Handler`);
+    this.debug(`Launching Playwright Handler`);
 
     this.browser = await playwright.firefox.launchServer({
       ...options,
@@ -106,7 +106,7 @@ export class PlaywrightFirefox extends EventEmitter {
 
     const serverURL = new URL(this.config.getExternalWebSocketAddress());
     const wsURL = new URL(this.browserWSEndpoint);
-    wsURL.hostname = serverURL.hostname;
+    wsURL.host = serverURL.host;
     wsURL.port = serverURL.port;
 
     if (token) {