@browserless.io/browserless 2.2.0 → 2.3.0-beta-2

package/README.md

@@ -23,7 +23,7 @@ If you've been struggling to deploy headless browsers without running into issue
 4. [Hosting](#hosting-providers)
 5. [Puppeteer](#puppeteer)
 6. [Playwright](#playwright)
-7. [Extending with NodeJS SDK](#extending-nodejs-skd)
+7. [Extending with NodeJS SDK](#extending-nodejs-sdk)
 8. [Licensing](#licensing)
 9. [Changelog](https://github.com/browserless/chrome/blob/master/CHANGELOG.md)
 
@@ -102,7 +102,7 @@ const browser = await pw.chromium.connectOverCDP('ws://localhost:3000');
 
 After that, the rest of your code remains the same with no other changes required.
 
-# Extending (NodeJS SKD)
+# Extending (NodeJS SDK)
 
 Browserless comes with built-in extension capabilities, and allows for extending nearly any aspect of the system (for Version 2+). For more details on how to write your own routes, build docker images, and more, [see our SDK README.md](/bin/scaffold/README.md) or simply run "npx @browserless.io/browserless create" in a terminal and follow the onscreen prompts.
 

package/bin/browserless.js

@@ -33,11 +33,6 @@ const allowedCMDs = [
   'help',
   'clean',
 ];
-const projectDir = process.cwd();
-const compiledDir = path.join(projectDir, 'build');
-const packageJSON = readFile(path.join(__dirname, '..', 'package.json')).then(
-  (r) => JSON.parse(r.toString()),
-);
 
 if (!allowedCMDs.includes(cmd)) {
   throw new Error(
@@ -45,6 +40,21 @@ if (!allowedCMDs.includes(cmd)) {
   );
 }
 
+const projectDir = process.cwd();
+const buildDir = 'build';
+const srcDir = 'src';
+const compiledDir = path.join(projectDir, buildDir);
+
+const projectPackageJSON = readFile(path.join(projectDir, 'package.json'))
+  .then((r) => JSON.parse(r.toString()))
+  .catch(() => null);
+
+const browserlessPackageJSON = readFile(
+  path.join(__dirname, '..', 'package.json'),
+).then((r) => JSON.parse(r.toString()));
+
+const camelCase = (str) => str.replace(/-([a-z])/g, (_, w) => w.toUpperCase());
+
 const prompt = async (question) => {
   const rl = createInterface({
     input: process.stdin,
@@ -60,22 +70,43 @@ const prompt = async (question) => {
   });
 };
 
-const importClassOverride = async (files, className) => {
+const translateSrcToBuild = (directory) => {
+  const srcToBuild = directory.replace(srcDir, '');
+  const pathParsed = path.parse(srcToBuild);
+  return path.format({ ...pathParsed, base: '', ext: '.js' });
+};
+
+const importDefault = async (files, fileName) => {
+  const pJSON = await projectPackageJSON;
+  // Check first if overrides are manually specified in the project's package.json
+  if (pJSON && pJSON.browserless && typeof pJSON.browserless === 'object') {
+    const camelCaseFileName = camelCase(fileName);
+    const relativePath = pJSON.browserless[camelCaseFileName];
+    if (relativePath) {
+      const fullFilePath = path.join(
+        compiledDir,
+        translateSrcToBuild(relativePath),
+      );
+      log(`Importing module from package.json: "${fullFilePath}"`);
+      return (await import(fullFilePath)).default;
+    }
+  }
+
   const classModuleFile = files.find((f) =>
-    path.parse(f).name.endsWith(className),
+    path.parse(f).name.endsWith(fileName),
   );
 
   if (!classModuleFile) {
     return;
   }
 
-  const classModuleFullFilePath = path.join(compiledDir, classModuleFile);
+  const fullFilePath = path.join(compiledDir, classModuleFile);
 
   if (!classModuleFile) {
     return;
   }
-  log(`Importing module override "${classModuleFullFilePath}"`);
-  return (await import(classModuleFullFilePath)).default;
+  log(`Importing module from found files: "${fullFilePath}"`);
+  return (await import(fullFilePath)).default;
 };
 
 const clean = async () =>
@@ -120,7 +151,7 @@ const buildDockerImage = async (cmd) => {
 
 const buildTypeScript = async () =>
   new Promise((resolve, reject) => {
-    spawn('npx', ['tsc', '--outDir', 'build'], {
+    spawn('npx', ['tsc', '--outDir', buildDir], {
       cwd: projectDir,
       stdio: 'inherit',
     }).once('close', (code) => {
@@ -209,7 +240,8 @@ const build = async () => {
   );
 
   log(`Generating OpenAPI JSON file`);
-  await buildOpenAPI(httpRoutes, webSocketRoutes);
+  const disabledRoutes = await importDefault(files, 'disabled-routes');
+  await buildOpenAPI(httpRoutes, webSocketRoutes, disabledRoutes);
 
   log(`All built assets complete`);
 
@@ -239,16 +271,18 @@ const start = async (dev = false) => {
     Router,
     Token,
     Webhooks,
+    disabledRoutes,
   ] = await Promise.all([
-    importClassOverride(files, 'browser-manager'),
-    importClassOverride(files, 'config'),
-    importClassOverride(files, 'file-system'),
-    importClassOverride(files, 'limiter'),
-    importClassOverride(files, 'metrics'),
-    importClassOverride(files, 'monitoring'),
-    importClassOverride(files, 'router'),
-    importClassOverride(files, 'token'),
-    importClassOverride(files, 'webhooks'),
+    importDefault(files, 'browser-manager'),
+    importDefault(files, 'config'),
+    importDefault(files, 'file-system'),
+    importDefault(files, 'limiter'),
+    importDefault(files, 'metrics'),
+    importDefault(files, 'monitoring'),
+    importDefault(files, 'router'),
+    importDefault(files, 'token'),
+    importDefault(files, 'webhooks'),
+    importDefault(files, 'disabled-routes'),
   ]);
 
   log(`Starting Browserless`);
@@ -285,6 +319,15 @@ const start = async (dev = false) => {
     webhooks,
   });
 
+  if (disabledRoutes !== undefined) {
+    if (!Array.isArray(disabledRoutes)) {
+      throw new Error(
+        `The "disabled-routes.ts" default export should be an array of Route classes.`,
+      );
+    }
+    browserless.disableRoutes(...disabledRoutes);
+  }
+
   httpRoutes.forEach((r) => browserless.addHTTPRoute(r));
   webSocketRoutes.forEach((r) => browserless.addWebSocketRoute(r));
 
@@ -328,7 +371,7 @@ const start = async (dev = false) => {
 };
 
 const buildDocker = async () => {
-  const finalDockerPath = path.join(projectDir, 'build', 'Dockerfile');
+  const finalDockerPath = path.join(compiledDir, 'Dockerfile');
   const argSwitches = getArgSwitches();
 
   await build();
@@ -339,10 +382,7 @@ const buildDocker = async () => {
 
   log(`Generating Dockerfile at "${finalDockerPath}"`);
 
-  await fs.writeFile(
-    path.join(projectDir, 'build', 'Dockerfile'),
-    dockerContents,
-  );
+  await fs.writeFile(compiledDir, dockerContents);
 
   const from =
     argSwitches.from ||
@@ -424,7 +464,7 @@ const create = async () => {
     const to = path.join(installPath, sdkFile);
     if (sdkFile === 'package.json') {
       const sdkPackageJSONTemplate = (await readFile(from)).toString();
-      const { version } = await packageJSON;
+      const { version } = await browserlessPackageJSON;
       const sdkPackageJSON = sdkPackageJSONTemplate.replace(
         '${BROWSERLESS_VERSION}',
         version,

package/bin/scaffold/README.md

@@ -13,6 +13,7 @@ Finally, this SDK and Browserless.io are built to support businesses and enterpr
 - [Routing](#routing)
 - [Utilities](#utilities)
 - [Extending Modules](#extending-modules)
+- [Disabling Routes](#dis)
 - [Running in Development](#running-in-development)
 - [Building for Production](#building-for-production)
 - [Running without Building](#running-without-building)
@@ -133,6 +134,9 @@ export type ResponseSchema = string;
 
 // Similar to React and other ecosystems, extend our basic HTTPRoute
 export default class HelloWorldRoute extends HTTPRoute {
+  // Must have a unique name for things like disabling to work if desired
+  name = 'PDFToS3Route';
+
   // Detail any content-types that this route should except. "contentTypes.any" here means any content-type.
   // If the content-type does not match then a 404 will be sent back
   accepts = [contentTypes.any];
@@ -192,6 +196,9 @@ export interface QuerySchema extends SystemQueryParameters {
 }
 
 export default class ChromiumWebSocketRoute extends BrowserWebsocketRoute {
+  // Must have a unique name for things like disabling to work if desired
+  name = 'ChromiumWebSocketRoute';
+
   // This route requires a valid authorization token.
   auth = true;
 
@@ -297,6 +304,9 @@ export interface BodySchema {
 }
 
 export default class PDFToS3Route extends BrowserHTTPRoute {
+  // Must have a unique name for things like disabling to work if desired
+  name = 'PDFToS3Route';
+
   // Our route only accepts JSON content-types, and the rest 404
   accepts = [contentTypes.json];
 
@@ -347,6 +357,46 @@ export default class PDFToS3Route extends BrowserHTTPRoute {
 
 With this approach you can effectively write, extend and author your own workflows within browserless!
 
+## Disabling Routes
+
+You can disable access to core routes by specifying the route names you want to disable in a file named `disabled-routes.ts`. Browserless will scan all directories for a file named as such, and disable the named classes exported by this file. The alternative is to create a `browserless` property in your package.json file that contains a `disabledRoutes` string pointing to the relative path of your disabled routes file.
+
+For example, if you want to disable all metrics, config, and session information your `src/disabled-routes.ts` file would look like this:
+
+```ts
+import { BrowserlessRoutes } from '@browserless.io/browserless';
+
+export default [
+  BrowserlessRoutes.ConfigGetRoute,
+  BrowserlessRoutes.SessionsGetGetRoute,
+  BrowserlessRoutes.MetricsGetRoute,
+  BrowserlessRoutes.MetricsTotalGetRoute,
+];
+```
+
+And in the package.json file, it'd look like this:
+
+```json
+{
+  // ... lots of package.json stuff
+  "browserless": {
+    "disabledRoutes": "./src/disabled-routes.ts"
+  }
+}
+```
+
+In order for route-disabling to work, you must have a `default` export that's an array of names. Browserless exports every route name it builds and runs internally, meaning you simply need to pass them through this `disabled-routes.ts` file after importing them.
+
+Disabling a route will do several things:
+
+- Return a `404` HTTP response when trying to call any of these routes.
+- Remove them from the embedded documentation site that is auto-generated.
+- Removes them from the OpenAPI JSON Schematic.
+- Prevents their type information from being converted from TypeScript to runtime validation.
+- It doesn't, however, remove them from Node's Module cache.
+
+All of Browserless' internal routes are side-effect free, meaning their largely state-less and don't do exhibit kind of behavior aside from route handling and metrics reporting. Having them in Node's module cache is fine since they're never mounted in the router and set up as a potential route.
+
 ## Running in Development
 
 After the project has been set up, you can use npm commands to build and run your code. The most important of these is the `npm run dev` command, which will do the following:

package/bin/scaffold/src/hello-world.http.ts

@@ -10,7 +10,8 @@ import {
 
 export type ResponseSchema = string;
 
-export default class HelloWorldRoute extends HTTPRoute {
+export default class HelloWorldHTTPRoute extends HTTPRoute {
+  name = 'HelloWorldHTTPRoute';
   accepts = [contentTypes.any];
   auth = true;
   browser = null;

package/build/browserless.d.ts

@@ -12,6 +12,7 @@ export declare class Browserless {
     protected router: Router;
     protected token: Token;
     protected webhooks: WebHooks;
+    disabledRouteNames: string[];
     webSocketRouteFiles: string[];
     httpRouteFiles: string[];
     server?: HTTPServer;
@@ -30,6 +31,8 @@ export declare class Browserless {
     });
     protected saveMetrics: () => Promise<void>;
     setMetricsSaveInterval: (interval: number) => void;
+    private routeIsDisabled;
+    disableRoutes(...routeNames: string[]): void;
     addHTTPRoute(httpRouteFilePath: string): void;
     addWebSocketRoute(webSocketRouteFilePath: string): void;
     setPort(port: number): void;

package/build/browserless.js

@@ -14,6 +14,7 @@ export class Browserless {
     router;
     token;
     webhooks;
+    disabledRouteNames = [];
     webSocketRouteFiles = [];
     httpRouteFiles = [];
     server;
@@ -69,6 +70,12 @@ export class Browserless {
         this.metricsSaveInterval = interval;
         this.metricsSaveIntervalID = setInterval(this.saveMetrics, this.metricsSaveInterval);
     };
+    routeIsDisabled(route) {
+        return this.disabledRouteNames.some((name) => name === route.name);
+    }
+    disableRoutes(...routeNames) {
+        this.disabledRouteNames.push(...routeNames);
+    }
     addHTTPRoute(httpRouteFilePath) {
         this.httpRouteFiles.push(httpRouteFilePath);
     }
@@ -95,12 +102,15 @@ export class Browserless {
             ChromiumPlaywright,
             WebkitPlaywright,
         ];
-        const [[httpRouteFiles, wsRouteFiles], installedBrowsers] = await Promise.all([getRouteFiles(this.config), availableBrowsers]);
+        const [[internalHttpRouteFiles, internalWsRouteFiles], installedBrowsers] = await Promise.all([getRouteFiles(this.config), availableBrowsers]);
         const docsLink = makeExternalURL(this.config.getExternalAddress(), '/docs');
         this.debug(printLogo(docsLink));
         this.debug(`Running as user "${userInfo().username}"`);
         this.debug('Starting import of HTTP Routes');
-        for (const httpRoute of [...httpRouteFiles, ...this.httpRouteFiles]) {
+        for (const httpRoute of [
+            ...internalHttpRouteFiles,
+            ...this.httpRouteFiles,
+        ]) {
             if (httpRoute.endsWith('js')) {
                 const { name } = path.parse(httpRoute);
                 const [bodySchema, querySchema] = await Promise.all(routeSchemas.map(async (schemaType) => {
@@ -112,18 +122,23 @@ export class Browserless {
                 const logger = createLogger(`http:${name}`);
                 const { default: Route, } = await import(routeImport + `?cb=${Date.now()}`);
                 const route = new Route(this.browserManager, this.config, this.fileSystem, logger, this.metrics, this.monitoring);
-                route.bodySchema = safeParse(bodySchema);
-                route.querySchema = safeParse(querySchema);
-                route.config = () => this.config;
-                route.metrics = () => this.metrics;
-                route.monitoring = () => this.monitoring;
-                route.fileSystem = () => this.fileSystem;
-                route.debug = () => logger;
-                httpRoutes.push(route);
+                if (!this.routeIsDisabled(route)) {
+                    route.bodySchema = safeParse(bodySchema);
+                    route.querySchema = safeParse(querySchema);
+                    route.config = () => this.config;
+                    route.metrics = () => this.metrics;
+                    route.monitoring = () => this.monitoring;
+                    route.fileSystem = () => this.fileSystem;
+                    route.debug = () => logger;
+                    httpRoutes.push(route);
+                }
             }
         }
         this.debug('Starting import of WebSocket Routes');
-        for (const wsRoute of [...wsRouteFiles, ...this.webSocketRouteFiles]) {
+        for (const wsRoute of [
+            ...internalWsRouteFiles,
+            ...this.webSocketRouteFiles,
+        ]) {
             if (wsRoute.endsWith('js')) {
                 const { name } = path.parse(wsRoute);
                 const [, querySchema] = await Promise.all(routeSchemas.map(async (schemaType) => {
@@ -135,17 +150,20 @@ export class Browserless {
                 const logger = createLogger(`ws:${name}`);
                 const { default: Route, } = await import(wsImport + `?cb=${Date.now()}`);
                 const route = new Route(this.browserManager, this.config, this.fileSystem, logger, this.metrics, this.monitoring);
-                route.querySchema = safeParse(querySchema);
-                route.config = () => this.config;
-                route.metrics = () => this.metrics;
-                route.monitoring = () => this.monitoring;
-                route.fileSystem = () => this.fileSystem;
-                route.debug = () => logger;
-                wsRoutes.push(route);
+                if (!this.routeIsDisabled(route)) {
+                    route.querySchema = safeParse(querySchema);
+                    route.config = () => this.config;
+                    route.metrics = () => this.metrics;
+                    route.monitoring = () => this.monitoring;
+                    route.fileSystem = () => this.fileSystem;
+                    route.debug = () => logger;
+                    wsRoutes.push(route);
+                }
             }
         }
+        const allRoutes = [...httpRoutes, ...wsRoutes];
         // Validate that we have the browsers they are asking for
-        [...httpRoutes, ...wsRoutes].forEach((route) => {
+        allRoutes.forEach((route) => {
             if ('browser' in route &&
                 route.browser &&
                 internalBrowsers.includes(route.browser) &&
@@ -153,6 +171,12 @@ export class Browserless {
                 throw new Error(`Couldn't load route "${route.path}" due to missing browser binary for "${route.browser?.name}"`);
             }
         });
+        const duplicateNamedRoutes = allRoutes
+            .filter((e, i, a) => a.findIndex((r) => r.name === e.name) !== i)
+            .map((r) => r.name);
+        if (duplicateNamedRoutes.length) {
+            this.debug(`Found duplicate routing names. Route names must be unique:`, duplicateNamedRoutes);
+        }
         httpRoutes.forEach((r) => this.router.registerHTTPRoute(r));
         wsRoutes.forEach((r) => this.router.registerWebSocketRoute(r));
         this.debug(`Imported and validated all route files, starting up server.`);

package/build/browsers/chrome.cdp.js

@@ -1,6 +1,6 @@
 import { chromeExecutablePath, createLogger, } from '@browserless.io/browserless';
 import { ChromiumCDP } from './chromium.cdp.js';
 export class ChromeCDP extends ChromiumCDP {
-    executablePath = chromeExecutablePath;
+    executablePath = chromeExecutablePath();
     debug = createLogger('browsers:chrome:cdp');
 }

package/build/browsers/chrome.playwright.js

@@ -1,6 +1,6 @@
 import { chromeExecutablePath, createLogger, } from '@browserless.io/browserless';
 import { ChromiumPlaywright } from './chromium.playwright.js';
 export class ChromePlaywright extends ChromiumPlaywright {
-    executablePath = chromeExecutablePath;
+    executablePath = chromeExecutablePath();
     debug = createLogger('browsers:chrome:playwright');
 }

package/build/browsers/index.d.ts

@@ -11,6 +11,7 @@ export declare class BrowserManager {
     constructor(config: Config);
     private browserIsChrome;
     protected removeUserDataDir: (userDataDir: string | null) => Promise<void>;
+    protected onNewPage: (req: Request, page: unknown) => Promise<void>;
     /**
      * Returns the /json/protocol API contents from Chromium or Chrome, whichever is installed,
      * and modifies URLs to set them to the appropriate addresses configured.

package/build/browsers/index.js

@@ -26,6 +26,9 @@ export class BrowserManager {
             });
         }
     };
+    onNewPage = async (req, page) => {
+        await pageHook({ meta: req.parsed, page });
+    };
     /**
      * Returns the /json/protocol API contents from Chromium or Chrome, whichever is installed,
      * and modifies URLs to set them to the appropriate addresses configured.
@@ -316,7 +319,7 @@ export class BrowserManager {
         await browser.launch(launchOptions);
         await browserHook({ browser, meta: req.parsed });
         browser.on('newPage', async (page) => {
-            await pageHook({ meta: req.parsed, page });
+            await this.onNewPage(req, page);
             (router.onNewPage || noop)(req.parsed || '', page);
         });
         return browser;

package/build/data/classes.json

@@ -1 +1 @@
-["b-modal-banner--disabled","blocked","bodyBlocked","c24-cc-visible","cbar","ccm-blocked","cmp-open","consent-modal-open","consent-modal-overflow","CookiePolicy--show","cookies","cu_k_cookie_consent_modal_open","gcdc-locked","gdpr-infobar-visible","gdprbanner_consent_gdpr_consent","gdprCookieBanner-acceptedAll","hasCookieBanner","ibeugdpr-disabled","idxrcookies-block-user-nav","modal--is-open","modal-open","no_scroll","no-scroll","noScroll","popin-gdpr-no-scroll","shopee-no-scroll","show-cookie-policy-info","sp-message-open","start-cookies","ta-cc-modal-open","touchevents-false","wt-cli-eu-country","wt-cli-geoip-on"]
+["appconsent_noscroll","b-modal-banner--disabled","blocked","bodyBlocked","c24-cc-visible","cbar","ccm-blocked","cmp-open","consent-modal-open","consent-modal-overflow","cookie_consent","CookiePolicy--show","cookies","cu_k_cookie_consent_modal_open","didomi-popup-open","gcdc-locked","gdpr-infobar-visible","gdprbanner_consent_gdpr_consent","gdprCookieBanner-acceptedAll","hasCookieBanner","ibeugdpr-disabled","idxrcookies-block-user-nav","modal--is-open","modal-open","no_scroll","no-scroll","noScroll","popin-gdpr-no-scroll","shopee-no-scroll","show-cookie-policy-info","sp-message-open","start-cookies","ta-cc-modal-open","touchevents-false","wt-cli-eu-country","wt-cli-geoip-on"]