@browserless.io/browserless 2.4.0-beta-1 → 2.4.0

package/src/browserless.ts

@@ -27,6 +27,7 @@ import {
   printLogo,
   safeParse,
 } from '@browserless.io/browserless';
+import { EventEmitter } from 'events';
 import { readFile } from 'fs/promises';
 import { userInfo } from 'os';
 
@@ -42,7 +43,7 @@ type routeInstances =
   | WebSocketRoute
   | BrowserWebsocketRoute;
 
-export class Browserless {
+export class Browserless extends EventEmitter {
   protected debug: debug.Debugger = createLogger('index');
   protected browserManager: BrowserManager;
   protected config: Config;
@@ -83,6 +84,7 @@ export class Browserless {
     token?: Browserless['token'];
     webhooks?: Browserless['webhooks'];
   } = {}) {
+    super();
     this.config = config || new Config();
     this.metrics = metrics || new Metrics();
     this.token = token || new Token(this.config);
@@ -177,7 +179,18 @@ export class Browserless {
 
   public async stop() {
     clearInterval(this.metricsSaveIntervalID as unknown as number);
-    return Promise.all([this.server?.stop()]);
+    return Promise.all([
+      this.server?.shutdown(),
+      this.browserManager.shutdown(),
+      this.config.shutdown(),
+      this.fileSystem.shutdown(),
+      this.limiter.shutdown(),
+      this.metrics.shutdown(),
+      this.monitoring.shutdown(),
+      this.router.shutdown(),
+      this.token.shutdown(),
+      this.webhooks.shutdown(),
+    ]);
   }
 
   public async start() {
@@ -201,8 +214,8 @@ export class Browserless {
     this.debug('Starting import of HTTP Routes');
 
     for (const httpRoute of [
-      ...internalHttpRouteFiles,
       ...this.httpRouteFiles,
+      ...internalHttpRouteFiles,
     ]) {
       if (httpRoute.endsWith('js')) {
         const { name } = path.parse(httpRoute);
@@ -251,8 +264,8 @@ export class Browserless {
 
     this.debug('Starting import of WebSocket Routes');
     for (const wsRoute of [
-      ...internalWsRouteFiles,
       ...this.webSocketRouteFiles,
+      ...internalWsRouteFiles,
     ]) {
       if (wsRoute.endsWith('js')) {
         const { name } = path.parse(wsRoute);

package/src/browsers/chromium.cdp.ts

@@ -5,9 +5,6 @@ import {
   Request,
   ServerError,
   createLogger,
-  encrypt,
-  liveURLSep,
-  makeExternalURL,
   noop,
   once,
 } from '@browserless.io/browserless';
@@ -27,7 +24,6 @@ puppeteerStealth.use(StealthPlugin());
 export class ChromiumCDP extends EventEmitter {
   protected config: Config;
   protected userDataDir: string | null;
-  protected record: boolean;
   protected blockAds: boolean;
   protected running = false;
   protected browser: Browser | null = null;
@@ -38,21 +34,18 @@ export class ChromiumCDP extends EventEmitter {
   protected executablePath = playwright.chromium.executablePath();
 
   constructor({
-    userDataDir,
-    config,
-    record,
     blockAds,
+    config,
+    userDataDir,
   }: {
     blockAds: boolean;
     config: Config;
-    record: boolean;
     userDataDir: ChromiumCDP['userDataDir'];
   }) {
     super();
 
     this.userDataDir = userDataDir;
     this.config = config;
-    this.record = record;
     this.blockAds = blockAds;
     this.debug(`Starting new browser instance`);
   }
@@ -62,135 +55,11 @@ export class ChromiumCDP extends EventEmitter {
     this.removeAllListeners();
   }
 
-  protected setUpEmbeddedAPI = async (
-    page: Page,
-    id: string,
-    record: boolean,
-  ): Promise<void> => {
-    const pageId = this.getPageId(page);
-    const liveUrl = this.makeLiveURL(id, pageId);
-    const embeddedAPI = (pageId: string, liveUrl: string, record: boolean) => {
-      Object.defineProperty(window, 'browserless', {
-        configurable: false,
-        enumerable: false,
-        value: {},
-        writable: false,
-      });
-
-      Object.defineProperties(window.browserless, {
-        liveUrl: {
-          configurable: false,
-          enumerable: false,
-          value: () => liveUrl,
-          writable: false,
-        },
-        pageId: {
-          configurable: false,
-          enumerable: false,
-          value: () => pageId,
-          writable: false,
-        },
-        startRecording: {
-          configurable: false,
-          enumerable: false,
-          value: (params: object) =>
-            new Promise((resolve, reject) => {
-              if (!record) {
-                throw new Error(
-                  `Must connect with a record query-param set to "true" in order to use recording`,
-                );
-              }
-              const start = () =>
-                window.postMessage(
-                  { ...params, id: pageId, type: 'REC_START' },
-                  '*',
-                );
-              const onStart = (event: MessageEvent) => {
-                if (event.data.id !== pageId) return;
-                if (event.data.message === 'REC_STARTED') {
-                  window.removeEventListener('message', onStart);
-                  return resolve(undefined);
-                }
-                if (event.data.message === 'REC_START_FAIL') {
-                  window.removeEventListener('message', onStart);
-                  return reject(new Error(event.data.error));
-                }
-              };
-
-              window.addEventListener('message', onStart);
-
-              return document.readyState == 'complete'
-                ? start()
-                : window.addEventListener('load', start);
-            }),
-          writable: false,
-        },
-        stopRecording: {
-          configurable: false,
-          enumerable: false,
-          value: () =>
-            new Promise((resolve, reject) => {
-              if (!record) {
-                return reject(
-                  new Error(
-                    `Must connect with a record query-param set to "true" in order to use recording`,
-                  ),
-                );
-              }
-              const onStop = (event: MessageEvent) => {
-                if (event.data.id !== pageId) return;
-                if (event.data.message === 'REC_FILE') {
-                  window.removeEventListener('message', onStop);
-                  return resolve(event.data.file);
-                }
-
-                if (event.data.message === 'REC_STOP_FAIL') {
-                  window.removeEventListener('message', onStop);
-                  return reject(new Error(event.data.error));
-                }
-
-                if (event.data.message === 'REC_NOT_STARTED') {
-                  window.removeEventListener('message', onStop);
-                  return reject(
-                    new Error(
-                      `Recording hasn't started, did you forget to start it?`,
-                    ),
-                  );
-                }
-              };
-
-              window.addEventListener('message', onStop);
-              return window.postMessage({ id: pageId, type: 'REC_STOP' }, '*');
-            }),
-          writable: false,
-        },
-      });
-    };
-
-    // Setup the browserless embedded API
-    await Promise.all([
-      page.evaluate(embeddedAPI, pageId, liveUrl, record),
-      page.evaluateOnNewDocument(embeddedAPI, pageId, liveUrl, record),
-    ]).catch((err) =>
-      this.debug(`Error setting up embedded API:`, err.message),
-    );
-  };
-
   public getPageId = (page: Page): string => {
     // @ts-ignore
     return page.target()._targetId;
   };
 
-  public makeLiveURL = (browserId: string, pageId: string) => {
-    const serverAddress = this.config.getExternalAddress();
-    const key = this.config.getAESKey();
-    const path = `${browserId}${liveURLSep}${pageId}`;
-    const encoded = encrypt(path, key);
-    const query = `?id=${encoded}`;
-
-    return makeExternalURL(serverAddress, 'live', query);
-  };
-
   protected onTargetCreated = async (target: Target) => {
     if (target.type() === 'page') {
       const page = await target.page().catch((e) => {
@@ -219,8 +88,6 @@ export class ChromiumCDP extends EventEmitter {
             }
           });
         }
-        const browserId = this.wsEndpoint()?.split('/').pop() as string;
-        await this.setUpEmbeddedAPI(page, browserId, this.record);
         this.emit('newPage', page);
       }
     }
@@ -268,33 +135,19 @@ export class ChromiumCDP extends EventEmitter {
       executablePath: this.executablePath,
     };
 
-    if (this.record || this.blockAds) {
-      const requiredExtensionArgs: string[] = [];
+    if (this.blockAds) {
       // Necessary to load extensions
       finalOptions.headless = false;
 
-      if (this.record) {
-        finalOptions.ignoreDefaultArgs = ['--enable-automation'];
-        requiredExtensionArgs.push(
-          '--enable-usermedia-screen-capturing',
-          '--enable-blink-features=GetUserMedia',
-          '--allow-http-screen-capture',
-          '--auto-select-desktop-capture-source=browserless-screencast',
-          '--disable-infobars',
-        );
-      }
-
-      const loadExtensionPaths: string = [
-        ...(this.record
-          ? [path.join(__dirname, '..', '..', 'extensions', 'screencast')]
-          : []),
-        ...(this.blockAds
-          ? [path.join(__dirname, '..', '..', 'extensions', 'ublock')]
-          : []),
-      ].join(',');
+      const loadExtensionPaths: string = path.join(
+        __dirname,
+        '..',
+        '..',
+        'extensions',
+        'ublock',
+      );
 
       finalOptions.args.push(
-        ...requiredExtensionArgs,
         '--load-extension=' + loadExtensionPaths,
         '--disable-extensions-except=' + loadExtensionPaths,
       );

package/src/browsers/chromium.playwright.ts

@@ -13,7 +13,6 @@ import httpProxy from 'http-proxy';
 export class ChromiumPlaywright extends EventEmitter {
   protected config: Config;
   protected userDataDir: string | null;
-  protected record: boolean;
   protected running = false;
   protected proxy = httpProxy.createProxyServer();
   protected browser: playwright.BrowserServer | null = null;
@@ -24,17 +23,14 @@ export class ChromiumPlaywright extends EventEmitter {
   constructor({
     config,
     userDataDir,
-    record,
   }: {
     config: Config;
-    record: boolean;
     userDataDir: ChromiumPlaywright['userDataDir'];
   }) {
     super();
 
     this.userDataDir = userDataDir;
     this.config = config;
-    this.record = record;
 
     this.debug(`Starting new browser instance`);
   }
@@ -82,10 +78,6 @@ export class ChromiumPlaywright extends EventEmitter {
   ): Promise<playwright.BrowserServer> => {
     this.debug(`Launching Playwright Handler`);
 
-    if (this.record) {
-      throw new ServerError(`Recording is not yet available with this browser`);
-    }
-
     this.browser = await playwright.chromium.launchServer({
       ...options,
       args: [

package/src/browsers/firefox.playwright.ts

@@ -13,7 +13,6 @@ import httpProxy from 'http-proxy';
 export class FirefoxPlaywright extends EventEmitter {
   protected config: Config;
   protected userDataDir: string | null;
-  protected record: boolean;
   protected running = false;
   protected proxy = httpProxy.createProxyServer();
   protected browser: playwright.BrowserServer | null = null;
@@ -23,17 +22,14 @@ export class FirefoxPlaywright extends EventEmitter {
   constructor({
     config,
     userDataDir,
-    record,
   }: {
     config: Config;
-    record: boolean;
     userDataDir: FirefoxPlaywright['userDataDir'];
   }) {
     super();
 
     this.userDataDir = userDataDir;
     this.config = config;
-    this.record = record;
 
     this.debug(`Starting new browser instance`);
   }
@@ -73,10 +69,6 @@ export class FirefoxPlaywright extends EventEmitter {
   public launch = async (
     options: BrowserServerOptions = {},
   ): Promise<playwright.BrowserServer> => {
-    if (this.record) {
-      throw new ServerError(`Recording is not yet available with this browser`);
-    }
-
     this.debug(`Launching Playwright Handler`);
 
     this.browser = await playwright.firefox.launchServer({

package/src/browsers/index.ts

@@ -84,7 +84,6 @@ export class BrowserManager {
     const browser = new Browser({
       blockAds: false,
       config: this.config,
-      record: false,
       userDataDir: null,
     });
     await browser.launch();
@@ -122,7 +121,6 @@ export class BrowserManager {
     const browser = new Browser({
       blockAds: false,
       config: this.config,
-      record: false,
       userDataDir: null,
     });
     await browser.launch();
@@ -322,7 +320,6 @@ export class BrowserManager {
     router: BrowserHTTPRoute | BrowserWebsocketRoute,
   ): Promise<BrowserInstance> => {
     const { browser: Browser } = router;
-    const record = parseBooleanParam(req.parsed.searchParams, 'record', false);
     const blockAds = parseBooleanParam(
       req.parsed.searchParams,
       'blockAds',
@@ -446,7 +443,6 @@ export class BrowserManager {
     const browser = new Browser({
       blockAds,
       config: this.config,
-      record,
       userDataDir,
     });
 
@@ -477,7 +473,7 @@ export class BrowserManager {
     return browser;
   };
 
-  public stop = async (): Promise<void> => {
+  public shutdown = async (): Promise<void> => {
     this.debug(`Closing down browser instances`);
     const sessions = Array.from(this.browsers);
     await Promise.all(sessions.map(([b]) => b.close()));
@@ -486,7 +482,12 @@ export class BrowserManager {
     this.timers.forEach((t) => clearTimeout(t));
     this.browsers = new Map();
     this.timers = new Map();
-
+    await this.stop();
     this.debug(`Shutdown complete`);
   };
+
+  /**
+   * Left blank for downstream SDK modules to optionally implement.
+   */
+  public stop = () => {};
 }

package/src/browsers/webkit.playwright.ts

@@ -13,7 +13,6 @@ import httpProxy from 'http-proxy';
 export class WebkitPlaywright extends EventEmitter {
   protected config: Config;
   protected userDataDir: string | null;
-  protected record: boolean;
   protected running = false;
   protected proxy = httpProxy.createProxyServer();
   protected browser: playwright.BrowserServer | null = null;
@@ -23,17 +22,14 @@ export class WebkitPlaywright extends EventEmitter {
   constructor({
     config,
     userDataDir,
-    record,
   }: {
     config: Config;
-    record: boolean;
     userDataDir: WebkitPlaywright['userDataDir'];
   }) {
     super();
 
     this.userDataDir = userDataDir;
     this.config = config;
-    this.record = record;
 
     this.debug(`Starting new browser instance`);
   }
@@ -73,10 +69,6 @@ export class WebkitPlaywright extends EventEmitter {
   public launch = async (
     options: BrowserServerOptions = {},
   ): Promise<playwright.BrowserServer> => {
-    if (this.record) {
-      throw new ServerError(`Recording is not yet available with this browser`);
-    }
-
     this.debug(`Launching Playwright Handler`);
 
     this.browser = await playwright.webkit.launchServer({

package/src/config.ts

@@ -441,4 +441,17 @@ export class Config extends EventEmitter {
     'Access-Control-Allow-Origin': this.corsOrigin,
     'Access-Control-Max-Age': this.corsMaxAge,
   });
+
+  /**
+   * Implement any browserless-core-specific shutdown logic here.
+   * Calls the empty-SDK stop method for downstream implementations.
+   */
+  public shutdown = async () => {
+    await this.stop();
+  };
+
+  /**
+   * Left blank for downstream SDK modules to optionally implement.
+   */
+  public stop = () => {};
 }

package/src/constants.ts

@@ -1,5 +1,4 @@
 export const encryptionAlgo = 'aes-192-cbc';
 export const encryptionSep = '.';
-export const liveURLSep = ':';
 export const keyLength = 24;
 export const BLESS_PAGE_IDENTIFIER = 'BLESS';

package/src/file-system.ts

@@ -5,13 +5,15 @@ import {
   encrypt,
 } from '@browserless.io/browserless';
 import { readFile, writeFile } from 'fs/promises';
+import { EventEmitter } from 'events';
 
-export class FileSystem {
+export class FileSystem extends EventEmitter {
   protected fsMap: Map<string, string[]> = new Map();
   protected currentAESKey: Buffer;
   protected log = createLogger('file-system');
 
   constructor(protected config: Config) {
+    super();
     this.currentAESKey = config.getAESKey();
     this.config.on('token', this.handleTokenChange);
   }
@@ -79,4 +81,17 @@ export class FileSystem {
 
     return splitContents;
   };
+
+  /**
+   * Implement any browserless-core-specific shutdown logic here.
+   * Calls the empty-SDK stop method for downstream implementations.
+   */
+  public shutdown = async () => {
+    await this.stop();
+  };
+
+  /**
+   * Left blank for downstream SDK modules to optionally implement.
+   */
+  public stop = () => {};
 }

package/src/http.ts

@@ -161,14 +161,6 @@ export interface SystemQueryParameters {
    */
   launch?: CDPLaunchOptions | BrowserServerOptions | string;
 
-  /**
-   * Whether or nor to record the session. The browser will run
-   * in "head-full" mode, and recording is started and closed
-   * via the embedded browserless API. Please refer to the "Recording"
-   * section in the live documentation site for more details.
-   */
-  record?: boolean;
-
   /**
    * Override the system-level timeout for this request.
    * Accepts a value in milliseconds.

package/src/limiter.ts

@@ -226,4 +226,17 @@ export class Limiter extends q {
         return bound;
       });
   };
+
+  /**
+   * Implement any browserless-core-specific shutdown logic here.
+   * Calls the empty-SDK stop method for downstream implementations.
+   */
+  public shutdown = async () => {
+    await this.stop();
+  };
+
+  /**
+   * Left blank for downstream SDK modules to optionally implement.
+   */
+  public stop = () => {};
 }

package/src/metrics.ts

@@ -1,6 +1,8 @@
 import { IBrowserlessStats } from '@browserless.io/browserless';
 
-export class Metrics {
+import { EventEmitter } from 'events';
+
+export class Metrics extends EventEmitter {
   protected sessionTimes: number[] = [];
   protected successful = 0;
   protected queued = 0;
@@ -105,4 +107,17 @@ export class Metrics {
       ),
     };
   }
+
+  /**
+   * Implement any browserless-core-specific shutdown logic here.
+   * Calls the empty-SDK stop method for downstream implementations.
+   */
+  public shutdown = async () => {
+    await this.stop();
+  };
+
+  /**
+   * Left blank for downstream SDK modules to optionally implement.
+   */
+  public stop = () => {};
 }

package/src/monitoring.ts

@@ -3,11 +3,14 @@ import {
   IResourceLoad,
   createLogger,
 } from '@browserless.io/browserless';
+import { EventEmitter } from 'events';
 import si from 'systeminformation';
 
-export class Monitoring {
+export class Monitoring extends EventEmitter {
   protected log = createLogger('hardware');
-  constructor(protected config: Config) {}
+  constructor(protected config: Config) {
+    super();
+  }
 
   public getMachineStats = async (): Promise<IResourceLoad> => {
     const [cpuLoad, memLoad] = await Promise.all([
@@ -50,4 +53,17 @@ export class Monitoring {
       memoryOverloaded,
     };
   };
+
+  /**
+   * Implement any browserless-core-specific shutdown logic here.
+   * Calls the empty-SDK stop method for downstream implementations.
+   */
+  public shutdown = async () => {
+    await this.stop();
+  };
+
+  /**
+   * Left blank for downstream SDK modules to optionally implement.
+   */
+  public stop = () => {};
 }

package/src/router.ts

@@ -16,10 +16,11 @@ import {
   isConnected,
   writeResponse,
 } from '@browserless.io/browserless';
+import { EventEmitter } from 'events';
 import micromatch from 'micromatch';
 import stream from 'stream';
 
-export class Router {
+export class Router extends EventEmitter {
   protected log = createLogger('router');
   protected verbose = createLogger('router:verbose');
   protected httpRoutes: Array<HTTPRoute | BrowserHTTPRoute> = [];
@@ -29,7 +30,9 @@ export class Router {
     protected config: Config,
     protected browserManager: BrowserManager,
     protected limiter: Limiter,
-  ) {}
+  ) {
+    super();
+  }
 
   protected getTimeout(req: Request) {
     const timer = req.parsed.searchParams.get('timeout');
@@ -207,13 +210,6 @@ export class Router {
     return route;
   }
 
-  public teardown() {
-    this.httpRoutes = [];
-    this.webSocketRoutes = [];
-
-    return this.browserManager.stop();
-  }
-
   public getStaticHandler() {
     return this.httpRoutes.find((route) =>
       route.path.includes(HTTPManagementRoutes.static),
@@ -253,4 +249,19 @@ export class Router {
       (r.path as Array<PathTypes>).some((p) => micromatch.isMatch(pathname, p)),
     );
   }
+
+  /**
+   * Implement any browserless-core-specific shutdown logic here.
+   * Calls the empty-SDK stop method for downstream implementations.
+   */
+  public shutdown = async () => {
+    this.httpRoutes = [];
+    this.webSocketRoutes = [];
+    await this.stop();
+  };
+
+  /**
+   * Left blank for downstream SDK modules to optionally implement.
+   */
+  public stop = () => {};
 }