@browserless.io/browserless 2.8.0 → 2.10.0

package/scripts/install-debugger.js

@@ -1,20 +1,60 @@
-import { cp } from 'fs/promises';
-import { fileURLToPath } from 'url';
-import path from 'path';
+/* global fetch, console, process */
+import { Readable } from 'stream';
+import { existsSync } from 'fs';
+import { join } from 'path';
+import os from 'os';
 
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
+import { deleteAsync } from 'del';
+import gunzip from 'gunzip-maybe';
+import { moveFile } from 'move-file';
+import tar from 'tar-fs';
 
-const from = path.join(
-  __dirname,
-  '..',
-  'node_modules',
-  'browserless-debugger',
-  'static',
-);
-const to = path.join(__dirname, '..', 'static', 'debugger');
+const registryURL = 'https://registry.npmjs.org/@browserless.io/debugger/';
+const tmp = join(os.tmpdir(), 'browserless-debugger');
+const untarDir = join(tmp, 'package', 'static');
+const debuggerDir = join(process.cwd(), 'static', 'debugger');
+
+const lastFromArr = (arr) => arr[arr.length - 1];
+const dlAndExtract = (url) =>
+  fetch(url).then(
+    (response) =>
+      new Promise((resolve, reject) => {
+        // @ts-ignore
+        Readable.fromWeb(response.body)
+          .pipe(gunzip())
+          .pipe(tar.extract(tmp))
+          .on('error', reject)
+          .on('finish', resolve);
+      }),
+  );
+
+const getLatestVersion = async () => {
+  const response = await fetch(registryURL);
+  const json = await response.json();
+  const latest = lastFromArr(Object.keys(json.versions));
+  return json.versions[latest];
+};
 
 (async () => {
-  await cp(from, to, {
-    recursive: true,
+  if (existsSync(debuggerDir)) {
+    await deleteAsync(debuggerDir);
+  }
+
+  const dist = await getLatestVersion()
+    .then((version) => version.dist.tarball)
+    .catch((error) => {
+      console.error(`Couldn't fetch latest debugger version: ${error.message}`);
+      process.exit(1);
+    });
+
+  await dlAndExtract(dist).catch((error) => {
+    console.error(`Couldn't download debugger: ${error.message}`);
+    process.exit(1);
   });
-})();
+
+  await moveFile(untarDir, debuggerDir);
+  await deleteAsync(tmp, { force: true });
+})().catch((error) => {
+  console.error(`An error occurred: ${error.message}`);
+  process.exit(1);
+});

package/src/browserless.ts

@@ -23,7 +23,6 @@ import {
   WebSocketRoute,
   WebkitPlaywright,
   availableBrowsers,
-  createLogger,
   getRouteFiles,
   makeExternalURL,
   printLogo,
@@ -46,7 +45,7 @@ type routeInstances =
   | BrowserWebsocketRoute;
 
 export class Browserless extends EventEmitter {
-  protected debug: debug.Debugger = createLogger('index');
+  protected logger: BlessLogger;
   protected browserManager: BrowserManager;
   protected config: Config;
   protected fileSystem: FileSystem;
@@ -94,6 +93,7 @@ export class Browserless extends EventEmitter {
   } = {}) {
     super();
     this.Logger = LoggerOverride ?? BlessLogger;
+    this.logger = new this.Logger('index');
     this.config = config || new Config();
     this.metrics = metrics || new Metrics();
     this.token = token || new Token(this.config);
@@ -129,7 +129,7 @@ export class Browserless extends EventEmitter {
 
     this.metrics.reset();
 
-    this.debug(
+    this.logger.info(
       `Current period usage: ${JSON.stringify({
         date: aggregatedStats.date,
         error: aggregatedStats.error,
@@ -146,7 +146,7 @@ export class Browserless extends EventEmitter {
     );
 
     if (metricsPath) {
-      this.debug(`Saving metrics to "${metricsPath}"`);
+      this.logger.info(`Saving metrics to "${metricsPath}"`);
       this.fileSystem.append(
         metricsPath,
         JSON.stringify(aggregatedStats),
@@ -170,7 +170,7 @@ export class Browserless extends EventEmitter {
     );
   };
 
-  private routeIsDisabled(route: routeInstances) {
+  protected routeIsDisabled(route: routeInstances) {
     return this.disabledRouteNames.some((name) => name === route.name);
   }
 
@@ -230,11 +230,18 @@ export class Browserless extends EventEmitter {
     const [[internalHttpRouteFiles, internalWsRouteFiles], installedBrowsers] =
       await Promise.all([getRouteFiles(this.config), availableBrowsers]);
 
+    const hasDebugger = await this.config.hasDebugger();
+    const debuggerURL =
+      hasDebugger &&
+      makeExternalURL(
+        this.config.getExternalAddress(),
+        `/debugger/?token=${this.config.getToken()}`,
+      );
     const docsLink = makeExternalURL(this.config.getExternalAddress(), '/docs');
 
-    this.debug(printLogo(docsLink));
-    this.debug(`Running as user "${userInfo().username}"`);
-    this.debug('Starting import of HTTP Routes');
+    this.logger.info(printLogo(docsLink, debuggerURL));
+    this.logger.info(`Running as user "${userInfo().username}"`);
+    this.logger.info('Starting import of HTTP Routes');
 
     for (const httpRoute of [
       ...this.httpRouteFiles,
@@ -265,12 +272,14 @@ export class Browserless extends EventEmitter {
           this.metrics,
           this.monitoring,
           this.staticSDKDir,
+          this.limiter,
         );
 
         if (!this.routeIsDisabled(route)) {
           route.bodySchema = safeParse(bodySchema);
           route.querySchema = safeParse(querySchema);
           route.config = () => this.config;
+          route.limiter = () => this.limiter;
           route.metrics = () => this.metrics;
           route.monitoring = () => this.monitoring;
           route.fileSystem = () => this.fileSystem;
@@ -281,7 +290,7 @@ export class Browserless extends EventEmitter {
       }
     }
 
-    this.debug('Starting import of WebSocket Routes');
+    this.logger.info('Starting import of WebSocket Routes');
     for (const wsRoute of [
       ...this.webSocketRouteFiles,
       ...internalWsRouteFiles,
@@ -314,11 +323,13 @@ export class Browserless extends EventEmitter {
           this.metrics,
           this.monitoring,
           this.staticSDKDir,
+          this.limiter,
         );
 
         if (!this.routeIsDisabled(route)) {
           route.querySchema = safeParse(querySchema);
           route.config = () => this.config;
+          route.limiter = () => this.limiter;
           route.metrics = () => this.metrics;
           route.monitoring = () => this.monitoring;
           route.fileSystem = () => this.fileSystem;
@@ -349,7 +360,7 @@ export class Browserless extends EventEmitter {
       .map((r) => r.name);
 
     if (duplicateNamedRoutes.length) {
-      this.debug(
+      this.logger.warn(
         `Found duplicate routing names. Route names must be unique:`,
         duplicateNamedRoutes,
       );
@@ -358,7 +369,9 @@ export class Browserless extends EventEmitter {
     httpRoutes.forEach((r) => this.router.registerHTTPRoute(r));
     wsRoutes.forEach((r) => this.router.registerWebSocketRoute(r));
 
-    this.debug(`Imported and validated all route files, starting up server.`);
+    this.logger.info(
+      `Imported and validated all route files, starting up server.`,
+    );
 
     this.server = new HTTPServer(
       this.config,
@@ -370,7 +383,7 @@ export class Browserless extends EventEmitter {
     );
 
     await this.server.start();
-    this.debug(`Starting metrics collection.`);
+    this.logger.info(`Starting metrics collection.`);
     this.metricsSaveIntervalID = setInterval(
       () => this.saveMetrics(),
       this.metricsSaveInterval,

package/src/browsers/chrome.cdp.ts

@@ -1,10 +1,7 @@
-import {
-  chromeExecutablePath,
-  createLogger,
-} from '@browserless.io/browserless';
+import { Logger, chromeExecutablePath } from '@browserless.io/browserless';
 import { ChromiumCDP } from './chromium.cdp.js';
 
 export class ChromeCDP extends ChromiumCDP {
   protected executablePath = chromeExecutablePath();
-  protected debug = createLogger('browsers:chrome:cdp');
+  protected logger = new Logger('browsers:chrome:cdp');
 }

package/src/browsers/chrome.playwright.ts

@@ -1,10 +1,7 @@
-import {
-  chromeExecutablePath,
-  createLogger,
-} from '@browserless.io/browserless';
+import { Logger, chromeExecutablePath } from '@browserless.io/browserless';
 import { ChromiumPlaywright } from './chromium.playwright.js';
 
 export class ChromePlaywright extends ChromiumPlaywright {
   protected executablePath = chromeExecutablePath();
-  protected debug = createLogger('browsers:chrome:playwright');
+  protected logger = new Logger('browsers:chrome:playwright');
 }

package/src/browsers/chromium.cdp.ts

@@ -2,9 +2,9 @@ import {
   BLESS_PAGE_IDENTIFIER,
   CDPLaunchOptions,
   Config,
+  Logger,
   Request,
   ServerError,
-  createLogger,
   noop,
   once,
 } from '@browserless.io/browserless';
@@ -29,7 +29,7 @@ export class ChromiumCDP extends EventEmitter {
   protected browser: Browser | null = null;
   protected browserWSEndpoint: string | null = null;
   protected port?: number;
-  protected debug = createLogger('browsers:chromium:cdp');
+  protected logger: Logger;
   protected proxy = httpProxy.createProxyServer();
   protected executablePath = playwright.chromium.executablePath();
 
@@ -37,9 +37,11 @@ export class ChromiumCDP extends EventEmitter {
     blockAds,
     config,
     userDataDir,
+    logger,
   }: {
     blockAds: boolean;
     config: Config;
+    logger: Logger;
     userDataDir: ChromiumCDP['userDataDir'];
   }) {
     super();
@@ -47,7 +49,9 @@ export class ChromiumCDP extends EventEmitter {
     this.userDataDir = userDataDir;
     this.config = config;
     this.blockAds = blockAds;
-    this.debug(`Starting new browser instance`);
+    this.logger = logger;
+
+    this.logger.info(`Starting new ${this.constructor.name} instance`);
   }
 
   protected cleanListeners() {
@@ -63,31 +67,62 @@ export class ChromiumCDP extends EventEmitter {
   protected onTargetCreated = async (target: Target) => {
     if (target.type() === 'page') {
       const page = await target.page().catch((e) => {
-        this.debug(`Error in new page ${e}`);
+        this.logger.error(`Error in ${this.constructor.name} new page ${e}`);
         return null;
       });
 
       if (page) {
-        if (!this.config.getAllowFileProtocol()) {
-          this.debug(`Setting up file:// protocol request rejection`);
-          page.on('request', async (request) => {
-            if (request.url().startsWith('file://')) {
-              this.debug(`File protocol request found in request, terminating`);
-              page.close().catch(noop);
-              this.close();
-            }
-          });
-
-          page.on('response', async (response) => {
-            if (response.url().startsWith('file://')) {
-              this.debug(
-                `File protocol request found in response, terminating`,
-              );
-              page.close().catch(noop);
-              this.close();
-            }
-          });
-        }
+        this.logger.trace(`Setting up file:// protocol request rejection`);
+
+        page.on('error', (err) => {
+          this.logger.error(err);
+        });
+
+        page.on('pageerror', (err) => {
+          this.logger.warn(err);
+        });
+
+        page.on('framenavigated', (frame) => {
+          this.logger.trace(`Navigation to ${frame.url()}`);
+        });
+
+        page.on('console', (message) => {
+          this.logger.trace(`${message.type()}: ${message.text()}`);
+        });
+
+        page.on('requestfailed', (req) => {
+          this.logger.warn(`"${req.failure()?.errorText}": ${req.url()}`);
+        });
+
+        page.on('request', async (request) => {
+          this.logger.trace(`${request.method()}: ${request.url()}`);
+          if (
+            !this.config.getAllowFileProtocol() &&
+            request.url().startsWith('file://')
+          ) {
+            this.logger.error(
+              `File protocol request found in request to ${this.constructor.name}, terminating`,
+            );
+            page.close().catch(noop);
+            this.close();
+          }
+        });
+
+        page.on('response', async (response) => {
+          this.logger.trace(`${response.status()}: ${response.url()}`);
+
+          if (
+            !this.config.getAllowFileProtocol() &&
+            response.url().startsWith('file://')
+          ) {
+            this.logger.error(
+              `File protocol request found in response to ${this.constructor.name}, terminating`,
+            );
+            page.close().catch(noop);
+            this.close();
+          }
+        });
+
         this.emit('newPage', page);
       }
     }
@@ -97,7 +132,9 @@ export class ChromiumCDP extends EventEmitter {
 
   public newPage = async (): Promise<Page> => {
     if (!this.browser) {
-      throw new ServerError(`Browser hasn't been launched yet!`);
+      throw new ServerError(
+        `${this.constructor.name} hasn't been launched yet!`,
+      );
     }
 
     return this.browser.newPage();
@@ -105,7 +142,9 @@ export class ChromiumCDP extends EventEmitter {
 
   public close = async (): Promise<void> => {
     if (this.browser) {
-      this.debug(`Closing browser process and all listeners`);
+      this.logger.info(
+        `Closing ${this.constructor.name} process and all listeners`,
+      );
       this.emit('close');
       this.cleanListeners();
       this.browser.removeAllListeners();
@@ -122,7 +161,7 @@ export class ChromiumCDP extends EventEmitter {
 
   public launch = async (options: CDPLaunchOptions = {}): Promise<Browser> => {
     this.port = await getPort();
-    this.debug(`Got open port ${this.port}`);
+    this.logger.info(`${this.constructor.name} got open port ${this.port}`);
     const __dirname = path.dirname(fileURLToPath(import.meta.url));
     const finalOptions = {
       ...options,
@@ -157,13 +196,17 @@ export class ChromiumCDP extends EventEmitter {
       ? puppeteerStealth.launch.bind(puppeteerStealth)
       : puppeteer.launch.bind(puppeteer);
 
-    this.debug(finalOptions, `Launching CDP Handler`);
-    // @ts-ignore mis-matched types from stealth...
+    this.logger.info(
+      finalOptions,
+      `Launching ${this.constructor.name} Handler`,
+    );
     this.browser = (await launch(finalOptions)) as Browser;
     this.browser.on('targetcreated', this.onTargetCreated);
     this.running = true;
     this.browserWSEndpoint = this.browser.wsEndpoint();
-    this.debug(`Browser is running on ${this.browserWSEndpoint}`);
+    this.logger.info(
+      `${this.constructor.name} is running on ${this.browserWSEndpoint}`,
+    );
 
     return this.browser;
   };
@@ -199,7 +242,9 @@ export class ChromiumCDP extends EventEmitter {
         );
       }
       socket.once('close', resolve);
-      this.debug(`Proxying ${req.parsed.href}`);
+      this.logger.info(
+        `Proxying ${req.parsed.href} to ${this.constructor.name}`,
+      );
 
       const shouldMakePage = req.parsed.pathname.includes(
         BLESS_PAGE_IDENTIFIER,
@@ -223,7 +268,9 @@ export class ChromiumCDP extends EventEmitter {
           target,
         },
         (error) => {
-          this.debug(`Error proxying session: ${error}`);
+          this.logger.error(
+            `Error proxying session to ${this.constructor.name}: ${error}`,
+          );
           this.close();
           return reject(error);
         },
@@ -253,8 +300,8 @@ export class ChromiumCDP extends EventEmitter {
       this.browser?.process()?.once('close', close);
       socket.once('close', close);
 
-      this.debug(
-        `Proxying ${req.parsed.href} to browser ${this.browserWSEndpoint}`,
+      this.logger.info(
+        `Proxying ${req.parsed.href} to ${this.constructor.name} ${this.browserWSEndpoint}`,
       );
 
       req.url = '';
@@ -271,7 +318,9 @@ export class ChromiumCDP extends EventEmitter {
           target: this.browserWSEndpoint,
         },
         (error) => {
-          this.debug(`Error proxying session: ${error}`);
+          this.logger.error(
+            `Error proxying session to ${this.constructor.name}: ${error}`,
+          );
           this.close();
           return reject(error);
         },

package/src/browsers/chromium.playwright.ts

@@ -1,9 +1,9 @@
 import {
   BrowserServerOptions,
   Config,
+  Logger,
   Request,
   ServerError,
-  createLogger,
 } from '@browserless.io/browserless';
 import playwright, { Page } from 'playwright-core';
 import { Duplex } from 'stream';
@@ -15,25 +15,28 @@ export class ChromiumPlaywright extends EventEmitter {
   protected config: Config;
   protected userDataDir: string | null;
   protected running = false;
+  protected logger: Logger;
   protected proxy = httpProxy.createProxyServer();
   protected browser: playwright.BrowserServer | null = null;
   protected browserWSEndpoint: string | null = null;
-  protected debug = createLogger('browsers:chromium:playwright');
   protected executablePath = playwright.chromium.executablePath();
 
   constructor({
     config,
     userDataDir,
+    logger,
   }: {
     config: Config;
+    logger: Logger;
     userDataDir: ChromiumPlaywright['userDataDir'];
   }) {
     super();
 
     this.userDataDir = userDataDir;
     this.config = config;
+    this.logger = logger;
 
-    this.debug(`Starting new browser instance`);
+    this.logger.info(`Starting new ${this.constructor.name} instance`);
   }
 
   protected cleanListeners() {
@@ -44,7 +47,9 @@ export class ChromiumPlaywright extends EventEmitter {
 
   public close = async (): Promise<void> => {
     if (this.browser) {
-      this.debug(`Closing browser process and all listeners`);
+      this.logger.info(
+        `Closing ${this.constructor.name} process and all listeners`,
+      );
       this.emit('close');
       this.cleanListeners();
       this.browser.close();
@@ -57,18 +62,22 @@ export class ChromiumPlaywright extends EventEmitter {
   public pages = async (): Promise<[]> => [];
 
   public getPageId = (): string => {
-    throw new ServerError(`#getPageId is not yet supported with this browser.`);
+    throw new ServerError(
+      `#getPageId is not yet supported with ${this.constructor.name}.`,
+    );
   };
 
   public makeLiveURL = (): void => {
     throw new ServerError(
-      `Live URLs are not yet supported with this browser. In the future this will be at "${this.config.getExternalAddress()}"`,
+      `Live URLs are not yet supported with ${this.constructor.name}. In the future this will be at "${this.config.getExternalAddress()}"`,
     );
   };
 
   public newPage = async (): Promise<Page> => {
     if (!this.browser || !this.browserWSEndpoint) {
-      throw new ServerError(`Browser hasn't been launched yet!`);
+      throw new ServerError(
+        `${this.constructor.name} hasn't been launched yet!`,
+      );
     }
     const browser = await playwright.chromium.connect(this.browserWSEndpoint);
     return await browser.newPage();
@@ -77,7 +86,7 @@ export class ChromiumPlaywright extends EventEmitter {
   public launch = async (
     options: BrowserServerOptions = {},
   ): Promise<playwright.BrowserServer> => {
-    this.debug(`Launching Playwright Handler`);
+    this.logger.info(`Launching ${this.constructor.name} Handler`);
 
     this.browser = await playwright.chromium.launchServer({
       ...options,
@@ -91,7 +100,9 @@ export class ChromiumPlaywright extends EventEmitter {
 
     const browserWSEndpoint = this.browser.wsEndpoint();
 
-    this.debug(`Browser is running on ${browserWSEndpoint}`);
+    this.logger.info(
+      `${this.constructor.name} is running on ${browserWSEndpoint}`,
+    );
     this.running = true;
     this.browserWSEndpoint = browserWSEndpoint;
 
@@ -118,7 +129,7 @@ export class ChromiumPlaywright extends EventEmitter {
   };
 
   public proxyPageWebSocket = async () => {
-    console.warn(`Not yet implemented`);
+    this.logger.warn(`${this.constructor.name} Not yet implemented`);
   };
 
   public proxyWebSocket = async (
@@ -134,8 +145,8 @@ export class ChromiumPlaywright extends EventEmitter {
       }
       socket.once('close', resolve);
 
-      this.debug(
-        `Proxying ${req.parsed.href} to browser ${this.browserWSEndpoint}`,
+      this.logger.info(
+        `Proxying ${req.parsed.href} to ${this.constructor.name} ${this.browserWSEndpoint}`,
       );
 
       // Delete headers known to cause issues
@@ -152,7 +163,9 @@ export class ChromiumPlaywright extends EventEmitter {
           target: this.browserWSEndpoint,
         },
         (error) => {
-          this.debug(`Error proxying session: ${error}`);
+          this.logger.error(
+            `Error proxying session to ${this.constructor.name}: ${error}`,
+          );
           this.close();
           return reject(error);
         },