@browserless.io/browserless 2.4.0 → 2.5.0-beta-1

package/CHANGELOG.md

@@ -1,4 +1,11 @@
 # [Latest](https://github.com/browserless/chrome/compare/v2.4.0...main)
+- Support for a new "Hooks" module for setting up SDK hooks in a more SDK-friendly manner.
+- Adds new exports for building downstream SDK projects more easily, versus using our CLI:
+  - `getArgSwitches`
+  - `getSourceFiles`
+  - `installDependencies`
+  - `buildDockerImage`
+  - `buildTypeScript`
 - Dependency updates.
 
 # [v2.4.0](https://github.com/browserless/chrome/compare/v2.3.0...v2.4.0)

package/bin/browserless.js

@@ -1,25 +1,30 @@
 #!/usr/bin/env node
 /* eslint-disable no-undef */
 'use strict';
+import {
+  Browserless,
+  buildTypeScript,
+  camelCase,
+  getArgSwitches,
+  getSourceFiles,
+  installDependencies,
+  prompt,
+} from '@browserless.io/browserless';
 import { readFile, writeFile } from 'fs/promises';
-import { Browserless } from '@browserless.io/browserless';
 import buildOpenAPI from '../scripts/build-open-api.js';
 import buildSchemas from '../scripts/build-schemas.js';
 
-import { createInterface } from 'readline';
 import debug from 'debug';
 import { dedent } from '../build/utils.js';
 import { fileURLToPath } from 'url';
 import fs from 'fs/promises';
 import path from 'path';
-import { spawn } from 'child_process';
 
 if (typeof process.env.DEBUG === 'undefined') {
   debug.enable('browserless*');
 }
 
 const log = debug('browserless.io:sdk:log');
-const promptLog = debug('browserless.io:prompt');
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const cmd = process.argv[2];
@@ -53,23 +58,6 @@ const browserlessPackageJSON = readFile(
   path.join(__dirname, '..', 'package.json'),
 ).then((r) => JSON.parse(r.toString()));
 
-const camelCase = (str) => str.replace(/-([a-z])/g, (_, w) => w.toUpperCase());
-
-const prompt = async (question) => {
-  const rl = createInterface({
-    input: process.stdin,
-    output: process.stdout,
-  });
-
-  return new Promise((resolve) => {
-    promptLog(question);
-    rl.question('  > ', (a) => {
-      rl.close();
-      resolve(a.trim());
-    });
-  });
-};
-
 const translateSrcToBuild = (directory) => {
   const srcToBuild = directory.replace(srcDir, '');
   const pathParsed = path.parse(srcToBuild);
@@ -115,109 +103,6 @@ const clean = async () =>
     recursive: true,
   });
 
-const installDependencies = async (workingDirectory) =>
-  new Promise((resolve, reject) => {
-    spawn('npm', ['i'], {
-      cwd: workingDirectory,
-      stdio: 'inherit',
-    }).once('close', (code) => {
-      if (code === 0) {
-        log(`Successfully installed Dependencies.`);
-        return resolve();
-      }
-      return reject(
-        `Error when installing dependencies, see output for more details`,
-      );
-    });
-  });
-
-const buildDockerImage = async (cmd) => {
-  new Promise((resolve, reject) => {
-    const [docker, ...args] = cmd.split(' ');
-    spawn(docker, args, {
-      cwd: projectDir,
-      stdio: 'inherit',
-    }).once('close', (code) => {
-      if (code === 0) {
-        log(`Successfully built the docker image.`);
-        return resolve();
-      }
-      return reject(
-        `Error when building Docker image, see output for more details`,
-      );
-    });
-  });
-};
-
-const buildTypeScript = async () =>
-  new Promise((resolve, reject) => {
-    spawn('npx', ['tsc', '--outDir', buildDir], {
-      cwd: projectDir,
-      stdio: 'inherit',
-    }).once('close', (code) => {
-      if (code === 0) {
-        return resolve();
-      }
-      return reject(
-        `Error in building TypeScript, see output for more details`,
-      );
-    });
-  });
-
-const getSourceFiles = async () => {
-  const files = await fs.readdir(compiledDir, { recursive: true });
-  const [httpRoutes, webSocketRoutes] = files.reduce(
-    ([httpRoutes, webSocketRoutes], file) => {
-      const parsed = path.parse(file);
-      if (parsed.name.endsWith('http')) {
-        httpRoutes.push(path.join(compiledDir, file));
-      }
-
-      if (parsed.name.endsWith('ws')) {
-        webSocketRoutes.push(path.join(compiledDir, file));
-      }
-
-      return [httpRoutes, webSocketRoutes];
-    },
-    [[], []],
-  );
-
-  return {
-    files,
-    httpRoutes,
-    webSocketRoutes,
-  };
-};
-
-const getArgSwitches = () => {
-  return process.argv.reduce((accum, arg, idx) => {
-    if (!arg.startsWith('--')) {
-      return accum;
-    }
-
-    if (arg.includes('=')) {
-      const [parameter, value] = arg.split('=');
-      accum[parameter.replace(/-/gi, '')] = value || true;
-      return accum;
-    }
-
-    const nextSwitchOrParameter = process.argv[idx + 1];
-    const param = arg.replace(/-/gi, '');
-
-    if (
-      typeof nextSwitchOrParameter === 'undefined' ||
-      nextSwitchOrParameter?.startsWith('--')
-    ) {
-      accum[param] = true;
-      return accum;
-    }
-
-    accum[param] = nextSwitchOrParameter;
-
-    return accum;
-  }, {});
-};
-
 /**
  * Build
  * Responsible for compiling TypeScript, generating routes meta-data
@@ -228,10 +113,11 @@ const build = async () => {
   await clean();
 
   log(`Compiling TypeScript`);
-  await buildTypeScript();
+  await buildTypeScript(buildDir, projectDir);
 
   log(`Building custom routes`);
-  const { files, httpRoutes, webSocketRoutes } = await getSourceFiles();
+  const { files, httpRoutes, webSocketRoutes } =
+    await getSourceFiles(projectDir);
 
   log(`Building route runtime schema validation`);
   await buildSchemas(
@@ -272,6 +158,7 @@ const start = async (dev = false) => {
     Token,
     Webhooks,
     disabledRoutes,
+    Hooks,
   ] = await Promise.all([
     importDefault(files, 'browser-manager'),
     importDefault(files, 'config'),
@@ -283,6 +170,7 @@ const start = async (dev = false) => {
     importDefault(files, 'token'),
     importDefault(files, 'webhooks'),
     importDefault(files, 'disabled-routes'),
+    importDefault(files,' hooks'),
   ]);
 
   log(`Starting Browserless`);
@@ -291,6 +179,7 @@ const start = async (dev = false) => {
   const metrics = isConstructor(Metrics) ? new Metrics() : Metrics;
   const token = isConstructor(Token) ? new Token(config) : Token;
   const webhooks = isConstructor(Webhooks) ? new Webhooks(config) : Webhooks;
+  const hooks = isConstructor(Hooks) ? new Hooks() : Hooks;
   const browserManager = isConstructor(BrowserManager)
     ? new BrowserManager(config)
     : BrowserManager;
@@ -311,6 +200,7 @@ const start = async (dev = false) => {
     browserManager,
     config,
     fileSystem,
+    hooks,
     limiter,
     metrics,
     monitoring,
@@ -433,7 +323,7 @@ const buildDocker = async () => {
 
   if (proceed || !proceed.includes('n')) {
     log(`Starting docker build`);
-    await buildDockerImage(cmd);
+    await buildDockerImage(cmd, projectDir);
     process.exit(0);
   }
 };

package/bin/scaffold/README.md

@@ -7,6 +7,7 @@ Please note that, as of right now, breaking changes aren't yet reflected in our
 Finally, this SDK and Browserless.io are built to support businesses and enterprise clients. If you're looking to use our code and modules in a production environment, [please contact us to get appropriately licensed](https://www.browserless.io/contact).
 
 ## Table of Contents
+
 - [Quick Start](#quick-start)
 - [About](#about)
 - [The CLI](#the-cli)
@@ -15,6 +16,7 @@ Finally, this SDK and Browserless.io are built to support businesses and enterpr
 - [Extending Modules](#extending-modules)
 - [Disabling Routes](#disabling-routes)
 - [Serving Static Files](#serving-static-files)
+- [Implementing Hooks](#implementing-hooks)
 - [Running in Development](#running-in-development)
 - [Building for Production](#building-for-production)
 - [Running without Building](#running-without-building)
@@ -35,7 +37,7 @@ browserless will install a scaffolded project, install dependencies, and establi
 
 The Browserless.io SDK and accompanying CLI were written with intention that developers can add and enhance functionality into Browserless for your needs. This way you can get results into a database, third-party uploads, work within your enterprises requirements, all while using your favorite modern libraries. The Browserless platform simply ensure system stability, authorization, and the best developer experience.
 
-When creating a new project, the scaffold will ask a series of questions and generate the project for you. Once complete,  a list of files it created for you. Here's the list so far:
+When creating a new project, the scaffold will ask a series of questions and generate the project for you. Once complete, a list of files it created for you. Here's the list so far:
 
 ```
 ├── node_modules
@@ -67,6 +69,7 @@ Aside form scaffolding the project, Browserless also looks for the following:
 - `*.websocket.ts` Files with this naming convention are treated as WebSocket routes.
 - `config.ts` Loads the default export here as a config override.
 - `file-system.ts` Loads the default export here as a file-system override.
+- `hooks.ts` Loads the default export as a Hooks override.
 - `limiter.ts` Loads the default export here as a limiter override (concurrency).
 - `metrics.ts` Loads the default export as a metrics override.
 - `monitoring.ts` Loads the default export as a monitoring override.
@@ -120,6 +123,7 @@ Browserless has 4 different types of primitive routes:
 Internally, we use this same class-based system, so feel free to see how those work in our open-source repositories. All routes are TypeScript-based and all our modules are documented, so you should be able to effectively write routes and modules with your code editor and not necessarily need these examples open. Below are a few examples:
 
 ### Basic HTTP Route
+
 ```ts
 import {
   APITags,
@@ -178,6 +182,7 @@ export default class HelloWorldRoute extends HTTPRoute {
 ```
 
 ### Chromium WebSocket Route
+
 ```ts
 import {
   APITags,
@@ -221,12 +226,8 @@ export default class ChromiumWebSocketRoute extends BrowserWebsocketRoute {
   // Routes with a browser type get a browser argument of the Browser instance, otherwise
   // request, socket, and head are the other 3 arguments. Here we pass them through
   // and proxy the request into Chromium to handle.
-  handler = async (
-    req,
-    socket,
-    head,
-    chromium,
-  ): Promise<void> => chromium.proxyWebSocket(req, socket, head);
+  handler = async (req, socket, head, chromium): Promise<void> =>
+    chromium.proxyWebSocket(req, socket, head);
 }
 ```
 
@@ -256,6 +257,7 @@ Browserless comes out-of-the-box with many utilities and functions to help with
 - `untildify(path)` Remove `~` characters from a path and returns the full filepath.
 
 ### Error helpers:
+
 - `BadRequest` An error that will cause browserless to return a `400` response with the error text being the message.
 - `TooManyRequests` When thrown causes browserless to return a `429` with the error as the message.
 - `ServerError` Returns a `500` code and shows the corresponding message.
@@ -284,7 +286,7 @@ export default class MyConfig extends Config {
     // Load from environment variables or default to some other named bucket.
     return process.env.S3_BUCKET ?? 'my-fun-s3-bucket';
   };
-};
+}
 ```
 
 Then, later, in your route you can define some functionality and load the config object. Let's make a PDF route that generates a PDF from a URL and then saves the result to this S3 bucket.
@@ -340,11 +342,7 @@ export default class PDFToS3Route extends BrowserHTTPRoute {
   tags = [APITags.browserAPI];
 
   // Handler's are where we embed the logic that facilitates this route.
-  handler = async (
-    req,
-    res,
-    browser,
-  ): Promise<void> => {
+  handler = async (req, res, browser): Promise<void> => {
     // Modules like Config are injected via this internal methods.
     // Use them to load core modules within the platform.
     const config = this.config() as MyConfig;
@@ -422,6 +420,89 @@ Will be available to be served under:
 
 Which prevents this route from colliding with our internal `/docs` route.
 
+## Implementing Hooks
+
+Browserless support writing a custom hooks module, where you can run or do custom checks during lifecycle events in Browserless. There are 4 events you can write custom functions for, which are detailed below. By default these hooks are benign and do nothing, so implementing them will alter how Browserless functions. Here's the default class written out with types:
+
+```ts
+// src/hooks.ts file
+import {
+  Request,
+  Response,
+  ChromiumCDP,
+  FirefoxPlaywright,
+  ChromiumPlaywright,
+  WebkitPlaywright,
+} from '@browserless.io/browserless';
+import * as stream from 'stream';
+import puppeteer from 'puppeteer-core';
+
+export class Hooks extends EventEmitter {
+  constructor() {
+    super();
+  }
+
+  // Called in src/server.ts for incoming HTTP and WebSocket requests, which
+  // is why certain arguments might not be present -- only the Request is
+  // guaranteed to be present as it is shared in both WS and HTTP requests.
+  // MUST return a true/false indicating if Browserless should continue
+  // handling the request or not.
+  before({
+    req,
+    res,
+    socket,
+    head,
+  }: {
+    req: Request;
+    res?: Response;
+    socket?: stream.Duplex;
+    head?: Buffer;
+  }): Promise<boolean> {
+    return Promise.resolve(true);
+  }
+
+  // Called in src/limiter.ts and provides details regarding the result of the
+  // session and a "start" time (Date.now()) of when the session started to run.
+  // No return value or type required.
+  after(args: {
+    status: 'successful' | 'error' | 'timedout';
+    start: number;
+    req: Request;
+  }): Promise<void> {
+    return Promise.resolve(undefined);
+  }
+
+  // Called in src/browsers/index.ts
+  // Called for every new CDP or Puppeteer-like "Page" creation in a browser.
+  // Can be used to inject behaviors or add events to a page's lifecycle.
+  // "meta" property is a parsed URL of the original incoming request.
+  // No return value or type required.
+  page(args: { meta: URL, page: puppeteer.Page }): Promise<void> {
+    return Promise.resolve(undefined);
+  }
+
+  // Called in src/browsers/index.ts
+  // Called for every new Browser creation in browserless, regardless of type.
+  // Can be used to inject behaviors or add events to a browser's lifecycle.
+  // "meta" property is a parsed URL of the original incoming request.
+  // No return value or type required.
+  browser(args: {
+    browser:
+      | ChromiumCDP
+      | FirefoxPlaywright
+      | ChromiumPlaywright
+      | WebkitPlaywright;
+    meta: URL;
+  }): Promise<unknown> {
+    return Promise.resolve(undefined);
+  }
+}
+```
+
+Of these hooks only `before` needs to return a true/false condition on whether or not to allow the request to proceed. If `false` Browserless does NOT write a response and simply closes the connection. Your `before` function should take care of any writing, closing, or streaming responses back before returning a `boolean`.
+
+Otherwise all other hooks are there for injecting side-effect like behaviors and don't necessarily need to return any kind of value. If a certain value is returned, Browserless simply ignores it.
+
 ## Running in Development
 
 After the project has been set up, you can use npm commands to build and run your code. The most important of these is the `npm run dev` command, which will do the following:
@@ -443,7 +524,7 @@ While the end-goal is a docker image being built, you can simply do a complete b
 $ npm run build
 ```
 
-Similar to development builds, this will compile all assets, generate OpenAPI JSON, and build out the runtime validation files, but *won't start the http server*.
+Similar to development builds, this will compile all assets, generate OpenAPI JSON, and build out the runtime validation files, but _won't start the http server_.
 
 If you wish to simply run the server without having to rebuild assets, then read more below.
 

package/build/browserless.d.ts

@@ -1,13 +1,14 @@
 /// <reference types="debug" />
 /// <reference types="node" />
 /// <reference types="node" />
-import { BrowserManager, Config, FileSystem, HTTPServer, Limiter, Metrics, Monitoring, Router, Token, WebHooks } from '@browserless.io/browserless';
+import { BrowserManager, Config, FileSystem, HTTPServer, Hooks, Limiter, Metrics, Monitoring, Router, Token, WebHooks } from '@browserless.io/browserless';
 import { EventEmitter } from 'events';
 export declare class Browserless extends EventEmitter {
     protected debug: debug.Debugger;
     protected browserManager: BrowserManager;
     protected config: Config;
     protected fileSystem: FileSystem;
+    protected hooks: Hooks;
     protected limiter: Limiter;
     protected metrics: Metrics;
     protected monitoring: Monitoring;
@@ -21,10 +22,11 @@ export declare class Browserless extends EventEmitter {
     server?: HTTPServer;
     metricsSaveInterval: number;
     metricsSaveIntervalID?: NodeJS.Timer;
-    constructor({ browserManager, config, fileSystem, limiter, metrics, monitoring, router, token, webhooks, }?: {
+    constructor({ browserManager, config, fileSystem, hooks, limiter, metrics, monitoring, router, token, webhooks, }?: {
         browserManager?: Browserless['browserManager'];
         config?: Browserless['config'];
         fileSystem?: Browserless['fileSystem'];
+        hooks?: Browserless['hooks'];
         limiter?: Browserless['limiter'];
         metrics?: Browserless['metrics'];
         monitoring?: Browserless['monitoring'];

package/build/browserless.js

@@ -1,5 +1,5 @@
 import * as path from 'path';
-import { BrowserManager, ChromeCDP, ChromiumCDP, ChromiumPlaywright, Config, FileSystem, FirefoxPlaywright, HTTPServer, Limiter, Metrics, Monitoring, Router, Token, WebHooks, WebkitPlaywright, availableBrowsers, createLogger, getRouteFiles, makeExternalURL, printLogo, safeParse, } from '@browserless.io/browserless';
+import { BrowserManager, ChromeCDP, ChromiumCDP, ChromiumPlaywright, Config, FileSystem, FirefoxPlaywright, HTTPServer, Hooks, Limiter, Metrics, Monitoring, Router, Token, WebHooks, WebkitPlaywright, availableBrowsers, createLogger, getRouteFiles, makeExternalURL, printLogo, safeParse, } from '@browserless.io/browserless';
 import { EventEmitter } from 'events';
 import { readFile } from 'fs/promises';
 import { userInfo } from 'os';
@@ -9,6 +9,7 @@ export class Browserless extends EventEmitter {
     browserManager;
     config;
     fileSystem;
+    hooks;
     limiter;
     metrics;
     monitoring;
@@ -22,18 +23,20 @@ export class Browserless extends EventEmitter {
     server;
     metricsSaveInterval = 5 * 60 * 1000;
     metricsSaveIntervalID;
-    constructor({ browserManager, config, fileSystem, limiter, metrics, monitoring, router, token, webhooks, } = {}) {
+    constructor({ browserManager, config, fileSystem, hooks, limiter, metrics, monitoring, router, token, webhooks, } = {}) {
         super();
         this.config = config || new Config();
         this.metrics = metrics || new Metrics();
         this.token = token || new Token(this.config);
+        this.hooks = hooks || new Hooks();
         this.webhooks = webhooks || new WebHooks(this.config);
-        this.browserManager = browserManager || new BrowserManager(this.config);
+        this.browserManager =
+            browserManager || new BrowserManager(this.config, this.hooks);
         this.monitoring = monitoring || new Monitoring(this.config);
         this.fileSystem = fileSystem || new FileSystem(this.config);
         this.limiter =
             limiter ||
-                new Limiter(this.config, this.metrics, this.monitoring, this.webhooks);
+                new Limiter(this.config, this.metrics, this.monitoring, this.webhooks, this.hooks);
         this.router =
             router || new Router(this.config, this.browserManager, this.limiter);
     }
@@ -62,7 +65,7 @@ export class Browserless extends EventEmitter {
         })}`);
         if (metricsPath) {
             this.debug(`Saving metrics to "${metricsPath}"`);
-            this.fileSystem.append(metricsPath, JSON.stringify(aggregatedStats));
+            this.fileSystem.append(metricsPath, JSON.stringify(aggregatedStats), false);
         }
     };
     setMetricsSaveInterval = (interval) => {
@@ -199,7 +202,7 @@ export class Browserless extends EventEmitter {
         httpRoutes.forEach((r) => this.router.registerHTTPRoute(r));
         wsRoutes.forEach((r) => this.router.registerWebSocketRoute(r));
         this.debug(`Imported and validated all route files, starting up server.`);
-        this.server = new HTTPServer(this.config, this.metrics, this.token, this.router);
+        this.server = new HTTPServer(this.config, this.metrics, this.token, this.router, this.hooks);
         await this.server.start();
         this.debug(`Starting metrics collection.`);
         this.metricsSaveIntervalID = setInterval(() => this.saveMetrics(), this.metricsSaveInterval);

package/build/browsers/index.d.ts

@@ -1,14 +1,15 @@
 /// <reference types="debug" />
-import { BrowserHTTPRoute, BrowserInstance, BrowserWebsocketRoute, BrowserlessSession, BrowserlessSessionJSON, CDPJSONPayload, ChromiumCDP, Config, Request } from '@browserless.io/browserless';
+import { BrowserHTTPRoute, BrowserInstance, BrowserWebsocketRoute, BrowserlessSession, BrowserlessSessionJSON, CDPJSONPayload, ChromiumCDP, Config, Hooks, Request } from '@browserless.io/browserless';
 export declare class BrowserManager {
     protected config: Config;
+    protected hooks: Hooks;
     protected browsers: Map<BrowserInstance, BrowserlessSession>;
     protected launching: Map<string, Promise<unknown>>;
     protected timers: Map<string, number>;
     protected debug: import("debug").Debugger;
     protected chromeBrowsers: (typeof ChromiumCDP)[];
     protected playwrightBrowserNames: string[];
-    constructor(config: Config);
+    constructor(config: Config, hooks: Hooks);
     private browserIsChrome;
     protected removeUserDataDir: (userDataDir: string | null) => Promise<void>;
     protected onNewPage: (req: Request, page: unknown) => Promise<void>;

package/build/browsers/index.js

@@ -1,8 +1,9 @@
-import { BLESS_PAGE_IDENTIFIER, BadRequest, ChromeCDP, ChromePlaywright, ChromiumCDP, ChromiumPlaywright, FirefoxPlaywright, HTTPManagementRoutes, NotFound, ServerError, WebkitPlaywright, availableBrowsers, browserHook, convertIfBase64, createLogger, exists, generateDataDir, makeExternalURL, noop, pageHook, parseBooleanParam, } from '@browserless.io/browserless';
+import { BLESS_PAGE_IDENTIFIER, BadRequest, ChromeCDP, ChromePlaywright, ChromiumCDP, ChromiumPlaywright, FirefoxPlaywright, HTTPManagementRoutes, NotFound, ServerError, WebkitPlaywright, availableBrowsers, convertIfBase64, createLogger, exists, generateDataDir, makeExternalURL, noop, parseBooleanParam, } from '@browserless.io/browserless';
 import { deleteAsync } from 'del';
 import path from 'path';
 export class BrowserManager {
     config;
+    hooks;
     browsers = new Map();
     launching = new Map();
     timers = new Map();
@@ -14,8 +15,9 @@ export class BrowserManager {
         FirefoxPlaywright.name,
         WebkitPlaywright.name,
     ];
-    constructor(config) {
+    constructor(config, hooks) {
         this.config = config;
+        this.hooks = hooks;
     }
     browserIsChrome = (b) => this.chromeBrowsers.some((chromeBrowser) => b instanceof chromeBrowser);
     removeUserDataDir = async (userDataDir) => {
@@ -27,7 +29,7 @@ export class BrowserManager {
         }
     };
     onNewPage = async (req, page) => {
-        await pageHook({ meta: req.parsed, page });
+        await this.hooks.page({ meta: req.parsed, page });
     };
     /**
      * Returns the /json/protocol API contents from Chromium or Chrome, whichever is installed,
@@ -313,7 +315,7 @@ export class BrowserManager {
         };
         this.browsers.set(browser, connectionMeta);
         await browser.launch(launchOptions);
-        await browserHook({ browser, meta: req.parsed });
+        await this.hooks.browser({ browser, meta: req.parsed });
         browser.on('newPage', async (page) => {
             await this.onNewPage(req, page);
             (router.onNewPage || noop)(req.parsed || '', page);