@browserless.io/browserless 2.0.0-beta-1

package/src/browserless.ts

@@ -0,0 +1,253 @@
+import * as path from 'path';
+import {
+  BrowserHTTPRoute,
+  BrowserManager,
+  BrowserWebsocketRoute,
+  Config,
+  FileSystem,
+  HTTPRoute,
+  HTTPServer,
+  IBrowserlessStats,
+  Limiter,
+  Metrics,
+  Monitoring,
+  WebHooks,
+  WebSocketRoute,
+  availableBrowsers,
+  createLogger,
+  getRouteFiles,
+  makeExternalURL,
+  printLogo,
+  safeParse,
+} from '@browserless.io/browserless';
+import { readFile } from 'fs/promises';
+import { userInfo } from 'os';
+
+const routeSchemas = ['body', 'query'];
+
+export class Browserless {
+  private config: Config;
+  private monitoring: Monitoring;
+  private metrics: Metrics;
+  private fileSystem: FileSystem;
+  private browserManager: BrowserManager;
+  private limiter: Limiter;
+  private webhooks: WebHooks;
+  private debug: debug.Debugger = createLogger('index');
+
+  webSocketRouteFiles: string[] = [];
+  httpRouteFiles: string[] = [];
+  server?: HTTPServer;
+  metricsSaveInterval: number = 5 * 60 * 1000;
+  metricsSaveIntervalID?: NodeJS.Timer;
+
+  constructor({
+    browserManager,
+    config,
+    monitoring,
+    limiter,
+    metrics,
+    fileSystem,
+    webhooks,
+  }: {
+    browserManager?: BrowserManager;
+    config?: Config;
+    fileSystem?: FileSystem;
+    limiter?: Limiter;
+    metrics?: Metrics;
+    monitoring?: Monitoring;
+    webhooks?: WebHooks;
+  } = {}) {
+    this.config = config || new Config();
+    this.metrics = metrics || new Metrics();
+    this.webhooks = webhooks || new WebHooks(this.config);
+    this.browserManager = browserManager || new BrowserManager(this.config);
+    this.monitoring = monitoring || new Monitoring(this.config);
+    this.fileSystem = fileSystem || new FileSystem(this.config);
+    this.limiter =
+      limiter ||
+      new Limiter(this.config, this.metrics, this.monitoring, this.webhooks);
+  }
+
+  private saveMetrics = async (): Promise<void> => {
+    const metricsPath = this.config.getMetricsJSONPath();
+    const { cpu, memory } = await this.monitoring.getMachineStats();
+    const metrics = await this.metrics.get();
+    const aggregatedStats: IBrowserlessStats = {
+      ...metrics,
+      cpu,
+      memory,
+    };
+
+    this.metrics.reset();
+
+    this.debug(
+      `Current period usage: ${JSON.stringify({
+        date: aggregatedStats.date,
+        error: aggregatedStats.error,
+        maxConcurrent: aggregatedStats.maxConcurrent,
+        maxTime: aggregatedStats.maxTime,
+        meanTime: aggregatedStats.meanTime,
+        minTime: aggregatedStats.minTime,
+        rejected: aggregatedStats.rejected,
+        successful: aggregatedStats.successful,
+        timedout: aggregatedStats.timedout,
+        totalTime: aggregatedStats.totalTime,
+        units: aggregatedStats.units,
+      })}`,
+    );
+
+    if (metricsPath) {
+      this.debug(`Saving metrics to "${metricsPath}"`);
+      this.fileSystem.append(metricsPath, JSON.stringify(aggregatedStats));
+    }
+  };
+
+  public setMetricsSaveInterval = (interval: number) => {
+    if (interval <= 0) {
+      return console.warn(
+        `Interval value of "${interval}" must be greater than 1. Ignoring`,
+      );
+    }
+
+    clearInterval(this.metricsSaveInterval);
+    this.metricsSaveInterval = interval;
+    this.metricsSaveIntervalID = setInterval(
+      this.saveMetrics,
+      this.metricsSaveInterval,
+    );
+  };
+
+  public addHTTPRoute(httpRouteFilePath: string) {
+    this.httpRouteFiles.push(httpRouteFilePath);
+  }
+
+  public addWebSocketRoute(webSocketRouteFilePath: string) {
+    this.webSocketRouteFiles.push(webSocketRouteFilePath);
+  }
+
+  public setPort(port: number) {
+    if (this.server) {
+      throw new Error(
+        `Server is already instantiated and bound to port ${this.config.getPort()}`,
+      );
+    }
+    this.config.setPort(port);
+  }
+
+  public async stop() {
+    clearInterval(this.metricsSaveIntervalID as unknown as number);
+    return Promise.all([this.server?.stop()]);
+  }
+
+  public async start() {
+    const httpRoutes: Array<HTTPRoute | BrowserHTTPRoute> = [];
+    const wsRoutes: Array<WebSocketRoute | BrowserWebsocketRoute> = [];
+
+    const [[httpRouteFiles, wsRouteFiles], installedBrowsers] =
+      await Promise.all([getRouteFiles(this.config), availableBrowsers]);
+
+    const docsLink = makeExternalURL(this.config.getExternalAddress(), '/docs');
+
+    this.debug(printLogo(docsLink));
+    this.debug(`Running as user "${userInfo().username}"`);
+    this.debug('Starting import of HTTP Routes');
+
+    for (const httpRoute of [...httpRouteFiles, ...this.httpRouteFiles]) {
+      if (httpRoute.endsWith('js')) {
+        const { name } = path.parse(httpRoute);
+        const [bodySchema, querySchema] = await Promise.all(
+          routeSchemas.map(async (schemaType) => {
+            const schemaPath = path.parse(httpRoute);
+            schemaPath.base = `${schemaPath.name}.${schemaType}.json`;
+            return await readFile(path.format(schemaPath), 'utf-8').catch(
+              () => '',
+            );
+          }),
+        );
+
+        const routeImport = `${
+          this.config.getIsWin() ? 'file:///' : ''
+        }${httpRoute}`;
+        const logger = createLogger(`http:${name}`);
+        const { default: route }: { default: HTTPRoute | BrowserHTTPRoute } =
+          await import(routeImport + `?cb=${Date.now()}`);
+
+        route.bodySchema = safeParse(bodySchema);
+        route.querySchema = safeParse(querySchema);
+        route._config = () => this.config;
+        route._metrics = () => this.metrics;
+        route._monitor = () => this.monitoring;
+        route._fileSystem = () => this.fileSystem;
+        route._debug = () => logger;
+
+        httpRoutes.push(route);
+      }
+    }
+
+    this.debug('Starting import of WebSocket Routes');
+    for (const wsRoute of [...wsRouteFiles, ...this.webSocketRouteFiles]) {
+      if (wsRoute.endsWith('js')) {
+        const { name } = path.parse(wsRoute);
+        const [, querySchema] = await Promise.all(
+          routeSchemas.map(async (schemaType) => {
+            const schemaPath = path.parse(wsRoute);
+            schemaPath.base = `${schemaPath.name}.${schemaType}.json`;
+            return await readFile(path.format(schemaPath), 'utf-8').catch(
+              () => '',
+            );
+          }),
+        );
+
+        const wsImport = `${
+          this.config.getIsWin() ? 'file:///' : ''
+        }${wsRoute}`;
+        const logger = createLogger(`ws:${name}`);
+        const {
+          default: route,
+        }: { default: WebSocketRoute | BrowserWebsocketRoute } = await import(
+          wsImport + `?cb=${Date.now()}`
+        );
+
+        route.querySchema = safeParse(querySchema);
+        route._config = () => this.config;
+        route._metrics = () => this.metrics;
+        route._monitor = () => this.monitoring;
+        route._fileSystem = () => this.fileSystem;
+        route._debug = () => logger;
+
+        wsRoutes.push(route);
+      }
+    }
+
+    // Validate that browsers are installed and route paths are unique
+    [...httpRoutes, ...wsRoutes].forEach((route) => {
+      if (
+        route.browser &&
+        !installedBrowsers.some((b) => b.name === route.browser.name)
+      ) {
+        throw new Error(
+          `Couldn't load route "${route.path}" due to missing browser of "${route.browser.name}"`,
+        );
+      }
+    });
+
+    this.debug(`Imported and validated all route files, starting up server.`);
+
+    this.server = new HTTPServer(
+      this.config,
+      this.metrics,
+      this.browserManager,
+      this.limiter,
+      httpRoutes,
+      wsRoutes,
+    );
+
+    await this.server.start();
+    this.debug(`Starting metrics collection.`);
+    this.metricsSaveIntervalID = setInterval(
+      () => this.saveMetrics(),
+      this.metricsSaveInterval,
+    );
+  }
+}

package/src/browsers/cdp-chromium.ts

@@ -0,0 +1,422 @@
+import {
+  CDPLaunchOptions,
+  Config,
+  Request,
+  ServerError,
+  createLogger,
+  encrypt,
+  liveURLSep,
+  makeExternalURL,
+  noop,
+  once,
+} from '@browserless.io/browserless';
+import puppeteer, { Browser, Page, Target } from 'puppeteer-core';
+import { Duplex } from 'stream';
+import { EventEmitter } from 'events';
+import StealthPlugin from 'puppeteer-extra-plugin-stealth';
+import { fileURLToPath } from 'url';
+import getPort from 'get-port';
+import httpProxy from 'http-proxy';
+import path from 'path';
+import playwright from 'playwright-core';
+import puppeteerStealth from 'puppeteer-extra';
+
+puppeteerStealth.use(StealthPlugin());
+
+export class CDPChromium extends EventEmitter {
+  private config: Config;
+  private userDataDir: string | null;
+  private record: boolean;
+  private blockAds: boolean;
+  private running = false;
+  private browser: Browser | null = null;
+  private browserWSEndpoint: string | null = null;
+  private port?: number;
+  private debug = createLogger('browsers:cdp:chromium');
+  private proxy = httpProxy.createProxyServer();
+
+  constructor({
+    userDataDir,
+    config,
+    record,
+    blockAds,
+  }: {
+    blockAds: boolean;
+    config: Config;
+    record: boolean;
+    userDataDir: CDPChromium['userDataDir'];
+  }) {
+    super();
+
+    this.userDataDir = userDataDir;
+    this.config = config;
+    this.record = record;
+    this.blockAds = blockAds;
+    this.debug(`Starting new browser instance`);
+  }
+
+  private cleanListeners() {
+    this.browser?.removeAllListeners();
+    this.removeAllListeners();
+  }
+
+  private setUpEmbeddedAPI = async (
+    page: Page,
+    id: string,
+    record: boolean,
+  ): Promise<void> => {
+    const pageId = this.getPageId(page);
+    const liveUrl = this.makeLiveURL(id, pageId);
+    const embeddedAPI = (pageId: string, liveUrl: string, record: boolean) => {
+      Object.defineProperty(window, 'browserless', {
+        configurable: false,
+        enumerable: false,
+        value: {},
+        writable: false,
+      });
+
+      Object.defineProperties(window.browserless, {
+        liveUrl: {
+          configurable: false,
+          enumerable: false,
+          value: () => liveUrl,
+          writable: false,
+        },
+        pageId: {
+          configurable: false,
+          enumerable: false,
+          value: () => pageId,
+          writable: false,
+        },
+        startRecording: {
+          configurable: false,
+          enumerable: false,
+          value: (params: object) =>
+            new Promise((resolve, reject) => {
+              if (!record) {
+                throw new Error(
+                  `Must connect with a record query-param set to "true" in order to use recording`,
+                );
+              }
+              const start = () =>
+                window.postMessage(
+                  { ...params, id: pageId, type: 'REC_START' },
+                  '*',
+                );
+              const onStart = (event: MessageEvent) => {
+                if (event.data.id !== pageId) return;
+                if (event.data.message === 'REC_STARTED') {
+                  window.removeEventListener('message', onStart);
+                  return resolve(undefined);
+                }
+                if (event.data.message === 'REC_START_FAIL') {
+                  window.removeEventListener('message', onStart);
+                  return reject(new Error(event.data.error));
+                }
+              };
+
+              window.addEventListener('message', onStart);
+
+              return document.readyState == 'complete'
+                ? start()
+                : window.addEventListener('load', start);
+            }),
+          writable: false,
+        },
+        stopRecording: {
+          configurable: false,
+          enumerable: false,
+          value: () =>
+            new Promise((resolve, reject) => {
+              if (!record) {
+                return reject(
+                  new Error(
+                    `Must connect with a record query-param set to "true" in order to use recording`,
+                  ),
+                );
+              }
+              const onStop = (event: MessageEvent) => {
+                if (event.data.id !== pageId) return;
+                if (event.data.message === 'REC_FILE') {
+                  window.removeEventListener('message', onStop);
+                  return resolve(event.data.file);
+                }
+
+                if (event.data.message === 'REC_STOP_FAIL') {
+                  window.removeEventListener('message', onStop);
+                  return reject(new Error(event.data.error));
+                }
+
+                if (event.data.message === 'REC_NOT_STARTED') {
+                  window.removeEventListener('message', onStop);
+                  return reject(
+                    new Error(
+                      `Recording hasn't started, did you forget to start it?`,
+                    ),
+                  );
+                }
+              };
+
+              window.addEventListener('message', onStop);
+              return window.postMessage({ id: pageId, type: 'REC_STOP' }, '*');
+            }),
+          writable: false,
+        },
+      });
+    };
+
+    // Setup the browserless embedded API
+    await Promise.all([
+      page.evaluate(embeddedAPI, pageId, liveUrl, record),
+      page.evaluateOnNewDocument(embeddedAPI, pageId, liveUrl, record),
+    ]).catch((err) =>
+      this.debug(`Error setting up embedded API:`, err.message),
+    );
+  };
+
+  public getPageId = (page: Page): string => {
+    // @ts-ignore
+    return page.target()._targetId;
+  };
+
+  public makeLiveURL = (browserId: string, pageId: string) => {
+    const serverAddress = this.config.getExternalAddress();
+    const key = this.config.getAESKey();
+    const path = `${browserId}${liveURLSep}${pageId}`;
+    const encoded = encrypt(path, key);
+    const query = `?id=${encoded}`;
+
+    return makeExternalURL(serverAddress, 'live', query);
+  };
+
+  private onTargetCreated = async (target: Target) => {
+    if (target.type() === 'page') {
+      const page = await target.page().catch((e) => {
+        this.debug(`Error in new page ${e}`);
+        return null;
+      });
+
+      if (page) {
+        if (!this.config.getAllowFileProtocol()) {
+          this.debug(`Setting up file:// protocol request rejection`);
+          page.on('request', async (request) => {
+            if (request.url().startsWith('file://')) {
+              this.debug(`File protocol request found in request, terminating`);
+              page.close().catch(noop);
+              this.close();
+            }
+          });
+
+          page.on('response', async (response) => {
+            if (response.url().startsWith('file://')) {
+              this.debug(
+                `File protocol request found in response, terminating`,
+              );
+              page.close().catch(noop);
+              this.close();
+            }
+          });
+        }
+        const browserId = this.wsEndpoint()?.split('/').pop() as string;
+        await this.setUpEmbeddedAPI(page, browserId, this.record);
+        this.emit('newPage', page);
+      }
+    }
+  };
+
+  public isRunning = (): boolean => this.running;
+
+  public newPage = async (): Promise<Page> => {
+    if (!this.browser) {
+      throw new ServerError(`Browser hasn't been launched yet!`);
+    }
+
+    return this.browser.newPage();
+  };
+
+  public close = async (): Promise<void> => {
+    if (this.browser) {
+      this.debug(`Closing browser process and all listeners`);
+      this.emit('close');
+      this.cleanListeners();
+      this.browser.removeAllListeners();
+      this.browser.close();
+      this.running = false;
+      this.browser = null;
+      this.browserWSEndpoint = null;
+    }
+  };
+
+  public pages = async (): Promise<Page[]> => this.browser?.pages() || [];
+
+  public process = () => this.browser?.process() || null;
+
+  public launch = async (options: CDPLaunchOptions = {}): Promise<Browser> => {
+    this.port = await getPort();
+    this.debug(`Got open port ${this.port}`);
+    const __dirname = path.dirname(fileURLToPath(import.meta.url));
+    const finalOptions = {
+      ...options,
+      args: [
+        `--remote-debugging-port=${this.port}`,
+        `--no-sandbox`,
+        ...(options.args || []),
+        this.userDataDir ? `--user-data-dir=${this.userDataDir}` : '',
+      ].filter((_) => !!_),
+      executablePath: playwright.chromium.executablePath(),
+    };
+
+    if (this.record || this.blockAds) {
+      const requiredExtensionArgs: string[] = [];
+      // Necessary to load extensions
+      finalOptions.headless = false;
+
+      if (this.record) {
+        finalOptions.ignoreDefaultArgs = ['--enable-automation'];
+        requiredExtensionArgs.push(
+          '--enable-usermedia-screen-capturing',
+          '--enable-blink-features=GetUserMedia',
+          '--allow-http-screen-capture',
+          '--auto-select-desktop-capture-source=browserless-screencast',
+          '--disable-infobars',
+        );
+      }
+
+      const loadExtensionPaths: string = [
+        ...(this.record
+          ? [path.join(__dirname, '..', '..', 'extensions', 'screencast')]
+          : []),
+        ...(this.blockAds
+          ? [path.join(__dirname, '..', '..', 'extensions', 'ublock')]
+          : []),
+      ].join(',');
+
+      finalOptions.args.push(
+        ...requiredExtensionArgs,
+        '--load-extension=' + loadExtensionPaths,
+        '--disable-extensions-except=' + loadExtensionPaths,
+      );
+    }
+
+    const launch = options.stealth
+      ? puppeteerStealth.launch.bind(puppeteerStealth)
+      : puppeteer.launch.bind(puppeteer);
+
+    this.debug(finalOptions, `Launching CDP Chrome Handler`);
+    this.browser = (await launch(finalOptions)) as Browser;
+    this.browser.on('targetcreated', this.onTargetCreated);
+    this.running = true;
+    this.browserWSEndpoint = this.browser.wsEndpoint();
+    this.debug(`Browser is running on ${this.browserWSEndpoint}`);
+
+    return this.browser;
+  };
+
+  public wsEndpoint = (): string | null => this.browserWSEndpoint;
+
+  public publicWSEndpoint = (token: string): string | null => {
+    if (!this.browserWSEndpoint) {
+      return null;
+    }
+
+    const wsURL = new URL(this.browserWSEndpoint);
+    const serverURL = new URL(this.config.getExternalAddress());
+
+    wsURL.hostname = serverURL.hostname;
+    wsURL.port = serverURL.port;
+    wsURL.protocol = serverURL.protocol === 'https' ? 'wss' : 'ws';
+    if (token) {
+      wsURL.searchParams.set('token', token);
+    }
+
+    return wsURL.href;
+  };
+
+  public proxyPageWebSocket = async (
+    req: Request,
+    socket: Duplex,
+    head: Buffer,
+  ): Promise<void> =>
+    new Promise(async (resolve, reject) => {
+      if (!this.browserWSEndpoint || !this.browser) {
+        throw new ServerError(
+          `No browserWSEndpoint found, did you launch first?`,
+        );
+      }
+      socket.once('close', resolve);
+
+      this.debug(
+        `Proxying ${req.parsed.href} to browser ${this.browserWSEndpoint}`,
+      );
+
+      const [page] = await this.browser.pages();
+      const pageLocation = `/devtools/page/${this.getPageId(page)}`;
+
+      this.debug(`Proxying ${req.parsed.href} to page "${pageLocation}"`);
+
+      req.url = pageLocation;
+
+      this.proxy.ws(
+        req,
+        socket,
+        head,
+        {
+          changeOrigin: true,
+          target: this.browserWSEndpoint,
+        },
+        (error) => {
+          this.debug(`Error proxying session: ${error}`);
+          this.close();
+          return reject(error);
+        },
+      );
+    });
+
+  public proxyWebSocket = async (
+    req: Request,
+    socket: Duplex,
+    head: Buffer,
+  ): Promise<void> =>
+    new Promise((resolve, reject) => {
+      if (!this.browserWSEndpoint) {
+        throw new ServerError(
+          `No browserWSEndpoint found, did you launch first?`,
+        );
+      }
+
+      const close = once(() => {
+        this.browser?.off('close', close);
+        this.browser?.process()?.off('close', close);
+        socket.off('close', close);
+        return resolve();
+      });
+
+      this.browser?.once('close', close);
+      this.browser?.process()?.once('close', close);
+      socket.once('close', close);
+
+      this.debug(
+        `Proxying ${req.parsed.href} to browser ${this.browserWSEndpoint}`,
+      );
+
+      req.url = '';
+
+      // Delete headers known to cause issues
+      delete req.headers.origin;
+
+      this.proxy.ws(
+        req,
+        socket,
+        head,
+        {
+          changeOrigin: true,
+          target: this.browserWSEndpoint,
+        },
+        (error) => {
+          this.debug(`Error proxying session: ${error}`);
+          this.close();
+          return reject(error);
+        },
+      );
+    });
+}