@browserless.io/browserless 2.2.0 → 2.3.0-beta-1

package/docker/firefox/Dockerfile

@@ -4,6 +4,8 @@ LABEL org.opencontainers.image.source https://github.com/browserless/browserless
 
 COPY fonts/* /usr/share/fonts/truetype/
 COPY src src/
+# Only use core as exports since routes are lazy loaded and not always present
+COPY src/exports.core.ts src/exports.ts
 RUN rm -r src/routes/
 COPY src/routes/management src/routes/management/
 COPY src/routes/firefox src/routes/firefox/

package/docker/multi/Dockerfile

@@ -7,6 +7,8 @@ ENV TARGETPLATFORM=${TARGETPLATFORM}
 
 COPY fonts/* /usr/share/fonts/truetype/
 COPY src src/
+# Only use core as exports since routes are lazy loaded and not always present
+COPY src/exports.core.ts src/exports.ts
 
 RUN echo "ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true" | debconf-set-selections &&\
   apt-get -y -qq install software-properties-common &&\

package/docker/webkit/Dockerfile

@@ -5,6 +5,8 @@ LABEL org.opencontainers.image.source https://github.com/browserless/browserless
 # Install Fonts
 COPY fonts/* /usr/share/fonts/truetype/
 COPY src src/
+# Only use core as exports since routes are lazy loaded and not always present
+COPY src/exports.core.ts src/exports.ts
 RUN rm -r src/routes/
 COPY src/routes/management src/routes/management/
 COPY src/routes/webkit src/routes/webkit/

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@browserless.io/browserless",
-  "version": "2.2.0",
+  "version": "2.3.0-beta-1",
   "license": "SSPL",
   "description": "The browserless platform",
   "author": "browserless.io",
@@ -81,7 +81,6 @@
     "esbuild": "^0.20.1",
     "esbuild-plugin-polyfill-node": "^0.3.0",
     "eslint": "^8.56.0",
-    "eslint-plugin-import": "^2.29.1",
     "eslint-plugin-typescript-sort-keys": "^3.1.0",
     "extract-zip": "^2.0.1",
     "marked": "^12.0.0",

package/scripts/build-open-api.js

@@ -3,9 +3,12 @@
 'use strict';
 
 import { join, parse } from 'path';
+import { Config } from '../build/config.js';
 import { dirname } from 'path';
+import { errorCodes } from '../build/http.js';
 import { fileURLToPath } from 'url';
 import fs from 'fs/promises';
+import { getRouteFiles } from '../build/utils.js';
 import { marked } from 'marked';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
@@ -39,17 +42,17 @@ const sortSwaggerRequiredAlpha = (prop, otherProp) => {
   return Number(otherProp.required) - Number(prop.required);
 };
 
+const routeIsDisabled = (Route, DisabledRoutes) => {
+  const stringified = Route.toString();
+  return DisabledRoutes.some((r) => r.toString() === stringified);
+};
+
 const buildOpenAPI = async (
   externalHTTPRoutes = [],
   externalWebSocketRoutes = [],
+  DisabledRoutes = [],
 ) => {
-  const [{ getRouteFiles }, { Config }, { errorCodes }, packageJSON] =
-    await Promise.all([
-      import('../build/utils.js'),
-      import('../build/config.js'),
-      import('../build/http.js'),
-      fs.readFile(packageJSONPath),
-    ]);
+  const packageJSON = await fs.readFile(packageJSONPath);
 
   const isWin = process.platform === 'win32';
   const readme = (await fs.readFile('README.md').catch(() => '')).toString();
@@ -90,7 +93,12 @@ const buildOpenAPI = async (
         const routeImport = `${isWin ? 'file:///' : ''}${routeModule}`;
         const { default: Route } = await import(routeImport);
         if (!Route) {
-          throw new Error(`Invalid route file to import docs ${routeModule}`);
+          throw new Error(
+            `Invalid route file found while building OpenAPI JSON: "${routeModule}"`,
+          );
+        }
+        if (routeIsDisabled(Route, DisabledRoutes)) {
+          return null;
         }
         const route = new Route();
         const { name } = parse(routeModule);
@@ -127,7 +135,8 @@ const buildOpenAPI = async (
           tags,
           title,
         };
-      }),
+      })
+      .filter((r) => r !== null),
   );
 
   const paths = routeMetaData.reduce((accum, r) => {

package/scripts/build-schemas.js

@@ -2,6 +2,8 @@
 /* global console, process */
 'use strict';
 
+import { getRouteFiles, tsExtension } from '../build/utils.js';
+import { Config } from '../build/config.js';
 import TJS from 'typescript-json-schema';
 import fs from 'fs/promises';
 import path from 'path';
@@ -12,8 +14,6 @@ const buildSchemas = async (
   externalHTTPRoutes = [],
   externalWebSocketRoutes = [],
 ) => {
-  const { getRouteFiles, tsExtension } = await import('../build/utils.js');
-
   const schemas = ['BodySchema', 'QuerySchema', 'ResponseSchema'];
   const settings = {
     ignoreErrors: true,
@@ -25,7 +25,6 @@ const buildSchemas = async (
     await fs.readFile('tsconfig.json', 'utf-8'),
   );
 
-  const { Config } = await import('../build/config.js');
   const [httpRoutes, wsRoutes] = await getRouteFiles(new Config());
 
   // Depending on if we're parsing an external projects routes,

package/src/browserless.ts

@@ -36,6 +36,12 @@ type Implements<T> = {
   new (...args: unknown[]): T;
 };
 
+type RouteTypes =
+  | typeof HTTPRoute
+  | typeof BrowserHTTPRoute
+  | typeof WebSocketRoute
+  | typeof BrowserWebsocketRoute;
+
 export class Browserless {
   protected debug: debug.Debugger = createLogger('index');
   protected browserManager: BrowserManager;
@@ -48,6 +54,7 @@ export class Browserless {
   protected token: Token;
   protected webhooks: WebHooks;
 
+  DisabledRoutes: RouteTypes[] = [];
   webSocketRouteFiles: string[] = [];
   httpRouteFiles: string[] = [];
   server?: HTTPServer;
@@ -138,6 +145,15 @@ export class Browserless {
     );
   };
 
+  private routeIsDisabled(Route: RouteTypes) {
+    const Stringified = Route.toString();
+    return this.DisabledRoutes.some((r) => r.toString() === Stringified);
+  }
+
+  public disableRoute(Route: RouteTypes) {
+    this.DisabledRoutes.push(Route);
+  }
+
   public addHTTPRoute(httpRouteFilePath: string) {
     this.httpRouteFiles.push(httpRouteFilePath);
   }
@@ -171,7 +187,7 @@ export class Browserless {
       WebkitPlaywright,
     ];
 
-    const [[httpRouteFiles, wsRouteFiles], installedBrowsers] =
+    const [[internalHttpRouteFiles, internalWsRouteFiles], installedBrowsers] =
       await Promise.all([getRouteFiles(this.config), availableBrowsers]);
 
     const docsLink = makeExternalURL(this.config.getExternalAddress(), '/docs');
@@ -180,7 +196,10 @@ export class Browserless {
     this.debug(`Running as user "${userInfo().username}"`);
     this.debug('Starting import of HTTP Routes');
 
-    for (const httpRoute of [...httpRouteFiles, ...this.httpRouteFiles]) {
+    for (const httpRoute of [
+      ...internalHttpRouteFiles,
+      ...this.httpRouteFiles,
+    ]) {
       if (httpRoute.endsWith('js')) {
         const { name } = path.parse(httpRoute);
         const [bodySchema, querySchema] = await Promise.all(
@@ -201,28 +220,33 @@ export class Browserless {
           default: Route,
         }: { default: Implements<HTTPRoute> | Implements<BrowserHTTPRoute> } =
           await import(routeImport + `?cb=${Date.now()}`);
-        const route = new Route(
-          this.browserManager,
-          this.config,
-          this.fileSystem,
-          logger,
-          this.metrics,
-          this.monitoring,
-        );
-        route.bodySchema = safeParse(bodySchema);
-        route.querySchema = safeParse(querySchema);
-        route.config = () => this.config;
-        route.metrics = () => this.metrics;
-        route.monitoring = () => this.monitoring;
-        route.fileSystem = () => this.fileSystem;
-        route.debug = () => logger;
-
-        httpRoutes.push(route);
+        if (!this.routeIsDisabled(Route)) {
+          const route = new Route(
+            this.browserManager,
+            this.config,
+            this.fileSystem,
+            logger,
+            this.metrics,
+            this.monitoring,
+          );
+          route.bodySchema = safeParse(bodySchema);
+          route.querySchema = safeParse(querySchema);
+          route.config = () => this.config;
+          route.metrics = () => this.metrics;
+          route.monitoring = () => this.monitoring;
+          route.fileSystem = () => this.fileSystem;
+          route.debug = () => logger;
+
+          httpRoutes.push(route);
+        }
       }
     }
 
     this.debug('Starting import of WebSocket Routes');
-    for (const wsRoute of [...wsRouteFiles, ...this.webSocketRouteFiles]) {
+    for (const wsRoute of [
+      ...internalWsRouteFiles,
+      ...this.webSocketRouteFiles,
+    ]) {
       if (wsRoute.endsWith('js')) {
         const { name } = path.parse(wsRoute);
         const [, querySchema] = await Promise.all(
@@ -246,23 +270,24 @@ export class Browserless {
             | Implements<WebSocketRoute>
             | Implements<BrowserWebsocketRoute>;
         } = await import(wsImport + `?cb=${Date.now()}`);
+        if (!this.routeIsDisabled(Route)) {
+          const route = new Route(
+            this.browserManager,
+            this.config,
+            this.fileSystem,
+            logger,
+            this.metrics,
+            this.monitoring,
+          );
+          route.querySchema = safeParse(querySchema);
+          route.config = () => this.config;
+          route.metrics = () => this.metrics;
+          route.monitoring = () => this.monitoring;
+          route.fileSystem = () => this.fileSystem;
+          route.debug = () => logger;
 
-        const route = new Route(
-          this.browserManager,
-          this.config,
-          this.fileSystem,
-          logger,
-          this.metrics,
-          this.monitoring,
-        );
-        route.querySchema = safeParse(querySchema);
-        route.config = () => this.config;
-        route.metrics = () => this.metrics;
-        route.monitoring = () => this.monitoring;
-        route.fileSystem = () => this.fileSystem;
-        route.debug = () => logger;
-
-        wsRoutes.push(route);
+          wsRoutes.push(route);
+        }
       }
     }
 

package/src/exports.core.ts

@@ -0,0 +1,26 @@
+// This file takes the place of 'exports.ts' when doing our docker builds
+// since docker images are missing some/most of the browser routes depending.
+export * from './browserless.js';
+export * from './config.js';
+export * from './constants.js';
+export * from './file-system.js';
+export * from './hooks.js';
+export * from './http.js';
+export * from './limiter.js';
+export * from './metrics.js';
+export * from './mime-types.js';
+export * from './monitoring.js';
+export * from './router.js';
+export * from './server.js';
+export * from './shim.js';
+export * from './token.js';
+export * from './types.js';
+export * from './utils.js';
+export * from './webhooks.js';
+export * from './browsers/index.js';
+export * from './browsers/chrome.cdp.js';
+export * from './browsers/chrome.playwright.js';
+export * from './browsers/chromium.cdp.js';
+export * from './browsers/chromium.playwright.js';
+export * from './browsers/firefox.playwright.js';
+export * from './browsers/webkit.playwright.js';

package/src/exports.ts

@@ -1,27 +1,49 @@
-// Main Files
-export * from './browserless.js';
-export * from './config.js';
-export * from './constants.js';
-export * from './file-system.js';
-export * from './hooks.js';
-export * from './http.js';
-export * from './limiter.js';
-export * from './metrics.js';
-export * from './mime-types.js';
-export * from './monitoring.js';
-export * from './router.js';
-export * from './server.js';
-export * from './shim.js';
-export * from './token.js';
-export * from './types.js';
-export * from './utils.js';
-export * from './webhooks.js';
+// Core
+export * from './exports.core.js';
 
-// Browsers and Helpers
-export * from './browsers/index.js';
-export * from './browsers/chrome.cdp.js';
-export * from './browsers/chrome.playwright.js';
-export * from './browsers/chromium.cdp.js';
-export * from './browsers/chromium.playwright.js';
-export * from './browsers/firefox.playwright.js';
-export * from './browsers/webkit.playwright.js';
+// Chrome
+export { default as ChromeContentPostRoute } from './routes/chrome/http/content.post.js';
+export { default as ChromeDownloadPostRoute } from './routes/chrome/http/download.post.js';
+export { default as ChromeFunctionPostRoute } from './routes/chrome/http/function.post.js';
+export { default as ChromeJSONListGetRoute } from './routes/chrome/http/json-list.get.js';
+export { default as ChromeJSONNewPutRoute } from './routes/chrome/http/json-new.put.js';
+export { default as ChromeJSONProtocolGetRoute } from './routes/chrome/http/json-protocol.get.js';
+export { default as ChromeJSONVersionGetRoute } from './routes/chrome/http/json-version.get.js';
+export { default as ChromePDFPostRoute } from './routes/chrome/http/pdf.post.js';
+export { default as ChromePerformancePostRoute } from './routes/chrome/http/performance.post.js';
+export { default as ChromeScrapePostRoute } from './routes/chrome/http/scrape.post.js';
+export { default as ChromeScreenshotPostRoute } from './routes/chrome/http/screenshot.post.js';
+export { default as ChromeBrowserWebSocketRoute } from './routes/chrome/ws/browser.js';
+export { default as ChromeCDPWebSocketRoute } from './routes/chrome/ws/cdp.js';
+export { default as ChromePageWebSocketRoute } from './routes/chrome/ws/page.js';
+export { default as ChromePlaywrightWebSocketRoute } from './routes/chrome/ws/playwright.js';
+
+// Chromium
+export { default as ChromiumContentPostRoute } from './routes/chromium/http/content.post.js';
+export { default as ChromiumDownloadPostRoute } from './routes/chromium/http/download.post.js';
+export { default as ChromiumFunctionPostRoute } from './routes/chromium/http/function.post.js';
+export { default as ChromiumJSONListGetRoute } from './routes/chromium/http/json-list.get.js';
+export { default as ChromiumJSONNewPutRoute } from './routes/chromium/http/json-new.put.js';
+export { default as ChromiumJSONProtocolGetRoute } from './routes/chromium/http/json-protocol.get.js';
+export { default as ChromiumJSONVersionGetRoute } from './routes/chromium/http/json-version.get.js';
+export { default as ChromiumPDFPostRoute } from './routes/chromium/http/pdf.post.js';
+export { default as ChromiumPerformancePostRoute } from './routes/chromium/http/performance.post.js';
+export { default as ChromiumScrapePostRoute } from './routes/chromium/http/scrape.post.js';
+export { default as ChromiumScreenshotPostRoute } from './routes/chromium/http/screenshot.post.js';
+export { default as ChromiumBrowserWebSocketRoute } from './routes/chromium/ws/browser.js';
+export { default as ChromiumCDPWebSocketRoute } from './routes/chromium/ws/cdp.js';
+export { default as ChromiumPageWebSocketRoute } from './routes/chromium/ws/page.js';
+export { default as ChromiumPlaywrightWebSocketRoute } from './routes/chromium/ws/playwright.js';
+
+// Firefox
+export { default as FirefoxPlaywrightWebSocketRoute } from './routes/firefox/ws/playwright.js';
+
+// WebKit
+export { default as WebKitPlaywrightWebSocketRoute } from './routes/webkit/ws/playwright.js';
+
+// Management
+export { default as ConfigGetRoute } from './routes/management/http/config.get.js';
+export { default as MetricsTotalGetRoute } from './routes/management/http/metrics-total.get.js';
+export { default as MetricsGetRoute } from './routes/management/http/metrics.get.js';
+export { default as SessionsGetGetRoute } from './routes/management/http/sessions.get.js';
+export { default as StaticGetRoute } from './routes/management/http/static.get.js';

package/src/router.ts

@@ -166,7 +166,9 @@ export class Router {
       : wrapped;
     route.path = Array.isArray(route.path) ? route.path : [route.path];
     const registeredPaths = this.httpRoutes.map((r) => r.path).flat();
-    const duplicatePaths = registeredPaths.filter((path) => route.path.includes(path));
+    const duplicatePaths = registeredPaths.filter((path) =>
+      route.path.includes(path),
+    );
 
     if (duplicatePaths.length) {
       this.log(`Found duplicate routes: ${duplicatePaths.join(', ')}`);
@@ -194,7 +196,9 @@ export class Router {
       : wrapped;
     route.path = Array.isArray(route.path) ? route.path : [route.path];
     const registeredPaths = this.webSocketRoutes.map((r) => r.path).flat();
-    const duplicatePaths = registeredPaths.filter((path) => route.path.includes(path));
+    const duplicatePaths = registeredPaths.filter((path) =>
+      route.path.includes(path),
+    );
 
     if (duplicatePaths.length) {
       this.log(`Found duplicate routes: ${duplicatePaths.join(', ')}`);

package/src/routes/firefox/ws/playwright.ts

@@ -11,7 +11,9 @@ import {
 import { Duplex } from 'stream';
 
 export interface QuerySchema extends SystemQueryParameters {
-  launch?: BrowserServerOptions | string;
+  launch?: BrowserServerOptions & {
+    firefoxUserPrefs?: { [key: string]: string | number | boolean };
+  };
 }
 
 export default class FirefoxPlayWrightRoute extends BrowserWebsocketRoute {

package/src/routes/management/http/static.get.ts

@@ -6,7 +6,6 @@ import {
   NotFound,
   Request,
   contentTypes,
-  createLogger,
   fileExists,
   mimeTypes,
 } from '@browserless.io/browserless';
@@ -14,9 +13,6 @@ import { ServerResponse } from 'http';
 import { createReadStream } from 'fs';
 import path from 'path';
 
-const debug = createLogger('http:static');
-const verbose = createLogger('http:static:verbose');
-
 const pathMap: Map<
   string,
   {
@@ -25,10 +21,15 @@ const pathMap: Map<
   }
 > = new Map();
 
-const streamFile = (res: ServerResponse, file: string, contentType?: string) =>
+const streamFile = (
+  debug: debug.Debugger,
+  res: ServerResponse,
+  file: string,
+  contentType?: string,
+) =>
   new Promise((resolve, reject) => {
     if (contentType) {
-      verbose(`Setting content-type ${contentType}`);
+      debug(`Setting content-type ${contentType}`);
       res.setHeader('Content-Type', contentType);
     }
 
@@ -59,9 +60,11 @@ export default class StaticGetRoute extends HTTPRoute {
   handler = async (req: Request, res: ServerResponse): Promise<unknown> => {
     const { pathname } = req.parsed;
     const fileCache = pathMap.get(pathname);
+    const debug = this.debug();
+    const verbose = debug.extend('verbose');
 
     if (fileCache) {
-      return streamFile(res, fileCache.path, fileCache.contentType);
+      return streamFile(verbose, res, fileCache.path, fileCache.contentType);
     }
 
     const config = this.config();
@@ -78,7 +81,9 @@ export default class StaticGetRoute extends HTTPRoute {
     ).find((_) => !!_);
 
     if (!filePath) {
-      throw new NotFound(`No route or file found for resource ${req.method}: ${pathname}`);
+      throw new NotFound(
+        `No route or file found for resource ${req.method}: ${pathname}`,
+      );
     }
 
     verbose(`Found new file "${filePath}", caching path and serving`);
@@ -96,6 +101,6 @@ export default class StaticGetRoute extends HTTPRoute {
       path: filePath,
     });
 
-    return streamFile(res, filePath, contentType);
+    return streamFile(verbose, res, filePath, contentType);
   };
 }

package/src/shared/utils/performance/main.ts

@@ -7,14 +7,13 @@ const DEFAULT_AUDIT_CONFIG = {
   extends: 'lighthouse:default',
 };
 
-const debug = createLogger('http:performance:main');
-
 export default async ({
   browser,
   context,
   timeout,
 }: mainOptions): Promise<unknown> => {
   return new Promise((resolve, reject) => {
+    const debug = createLogger('http:performance:main');
     const childPath = path.join(
       './',
       'build',