@browserless.io/browserless 2.2.0 → 2.3.0-beta-2

package/scripts/build-open-api.js

@@ -11,6 +11,13 @@ import { marked } from 'marked';
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const moduleMain = import.meta.url.endsWith(process.argv[1]);
 const swaggerJSONPath = join(__dirname, '..', 'static', 'docs', 'swagger.json');
+const swaggerJSONMinimal = join(
+  __dirname,
+  '..',
+  'static',
+  'docs',
+  'swagger.min.json',
+);
 const packageJSONPath = join(__dirname, '..', 'package.json');
 
 const readFileOrNull = async (path) => {
@@ -42,6 +49,7 @@ const sortSwaggerRequiredAlpha = (prop, otherProp) => {
 const buildOpenAPI = async (
   externalHTTPRoutes = [],
   externalWebSocketRoutes = [],
+  disabledRoutes = [],
 ) => {
   const [{ getRouteFiles }, { Config }, { errorCodes }, packageJSON] =
     await Promise.all([
@@ -62,8 +70,6 @@ const buildOpenAPI = async (
     customSiteTitle: 'Browserless Documentation',
     definitions: {},
     info: {
-      // Concatenation necessary for Changelog to show up in sidebar
-      description: readme + `\n# Changelog\n` + changelog,
       title: 'Browserless',
       version: JSON.parse(packageJSON.toString()).version,
       'x-logo': {
@@ -93,6 +99,11 @@ const buildOpenAPI = async (
           throw new Error(`Invalid route file to import docs ${routeModule}`);
         }
         const route = new Route();
+
+        if (disabledRoutes.includes(route.name)) {
+          return null;
+        }
+
         const { name } = parse(routeModule);
         const body = routeModule.replace('.js', '.body.json');
         const query = routeModule.replace('.js', '.query.json');
@@ -100,7 +111,7 @@ const buildOpenAPI = async (
         const isWebSocket = routeModule.includes('/ws/') || name.endsWith('ws');
         const path = Array.isArray(route.path)
           ? route.path.join(' ')
-          : [route.path];
+          : route.path;
         const {
           tags,
           description,
@@ -130,145 +141,152 @@ const buildOpenAPI = async (
       }),
   );
 
-  const paths = routeMetaData.reduce((accum, r) => {
-    const swaggerRoute = {
-      definitions: {},
-      description: r.description,
-      parameters: [],
-      requestBody: {
-        content: {},
-      },
-      responses: {
-        ...errorCodes,
-      },
-      summary: r.path,
-      tags: r.tags,
-    };
+  const paths = routeMetaData
+    .filter((_) => !!_)
+    .reduce((accum, r) => {
+      const swaggerRoute = {
+        definitions: {},
+        description: r.description,
+        parameters: [],
+        requestBody: {
+          content: {},
+        },
+        responses: {
+          ...errorCodes,
+        },
+        summary: r.path,
+        tags: r.tags,
+      };
 
-    r.method = r.isWebSocket ? 'get' : r.method;
+      r.method = r.isWebSocket ? 'get' : r.method;
 
-    // Find all the swagger definitions and merge them into the
-    // definitions object
-    const allDefs = {
-      ...(r?.body?.definitions || {}),
-      ...(r?.query?.definitions || {}),
-      ...(r?.response?.definitions || {}),
-    };
+      // Find all the swagger definitions and merge them into the
+      // definitions object
+      const allDefs = {
+        ...(r?.body?.definitions || {}),
+        ...(r?.query?.definitions || {}),
+        ...(r?.response?.definitions || {}),
+      };
 
-    Object.entries(allDefs).forEach(([defName, definition]) => {
-      // @ts-ignore
-      swaggerJSON.definitions[defName] =
+      Object.entries(allDefs).forEach(([defName, definition]) => {
         // @ts-ignore
-        swaggerJSON.definitions[defName] ?? definition;
-    });
-
-    if (r.isWebSocket) {
-      swaggerRoute.responses['101'] = {
-        description: 'Indicates successful WebSocket upgrade.',
-      };
-    }
+        swaggerJSON.definitions[defName] =
+          // @ts-ignore
+          swaggerJSON.definitions[defName] ?? definition;
+      });
 
-    // Does a best-attempt at configuring multiple response types
-    // Won't figure out APIs that return mixed response types like
-    // JSON and binary blobs
-    if (r.response) {
-      if (r.contentTypes.length === 1) {
-        const [type] = r.contentTypes;
-        swaggerRoute.responses['200'] = {
-          content: {
-            [type]: {
-              schema: r.response,
-            },
-          },
-          description: r.response.description,
+      if (r.isWebSocket) {
+        swaggerRoute.responses['101'] = {
+          description: 'Indicates successful WebSocket upgrade.',
         };
-      } else {
-        const okResponses = r.contentTypes.reduce(
-          (accum, c) => {
-            // @ts-ignore
-            accum.content[c] = {
-              schema: {
-                type: 'text',
+      }
+
+      // Does a best-attempt at configuring multiple response types
+      // Won't figure out APIs that return mixed response types like
+      // JSON and binary blobs
+      if (r.response) {
+        if (r.contentTypes.length === 1) {
+          const [type] = r.contentTypes;
+          swaggerRoute.responses['200'] = {
+            content: {
+              [type]: {
+                schema: r.response,
               },
-            };
-            return accum;
-          },
-          {
-            content: {},
+            },
             description: r.response.description,
-          },
-        );
-        swaggerRoute.responses['200'] = okResponses;
+          };
+        } else {
+          const okResponses = r.contentTypes.reduce(
+            (accum, c) => {
+              // @ts-ignore
+              accum.content[c] = {
+                schema: {
+                  type: 'text',
+                },
+              };
+              return accum;
+            },
+            {
+              content: {},
+              description: r.response.description,
+            },
+          );
+          swaggerRoute.responses['200'] = okResponses;
+        }
       }
-    }
 
-    // Does a best-attempt at configuring multiple body types and
-    // ignores the "accepts" properties on routes since we can't
-    // yet correlate the accepted types to the proper body
-    if (r.body) {
-      const { properties, type, anyOf } = r.body;
-      if (anyOf) {
-        // @ts-ignore
-        anyOf.forEach((anyType) => {
-          if (anyType.type === 'string') {
-            const type = r.accepts.filter(
-              // @ts-ignore
-              (accept) => accept !== 'application/json',
-            );
-            swaggerRoute.requestBody.content[type] = {
-              schema: {
-                type: 'string',
-              },
-            };
-          }
+      // Does a best-attempt at configuring multiple body types and
+      // ignores the "accepts" properties on routes since we can't
+      // yet correlate the accepted types to the proper body
+      if (r.body) {
+        const { properties, type, anyOf } = r.body;
+        if (anyOf) {
+          // @ts-ignore
+          anyOf.forEach((anyType) => {
+            if (anyType.type === 'string') {
+              const type = r.accepts.filter(
+                // @ts-ignore
+                (accept) => accept !== 'application/json',
+              );
+              swaggerRoute.requestBody.content[type] = {
+                schema: {
+                  type: 'string',
+                },
+              };
+            }
 
-          if (anyType['$ref']) {
-            swaggerRoute.requestBody.content['application/json'] = {
-              schema: {
-                $ref: anyType['$ref'],
-              },
-            };
-          }
-        });
-      }
+            if (anyType['$ref']) {
+              swaggerRoute.requestBody.content['application/json'] = {
+                schema: {
+                  $ref: anyType['$ref'],
+                },
+              };
+            }
+          });
+        }
 
-      // Handle JSON
-      if (type === 'object') {
-        swaggerRoute.requestBody.content['application/json'] = {
-          schema: {
-            properties,
-            type: 'object',
-          },
-        };
+        // Handle JSON
+        if (type === 'object') {
+          swaggerRoute.requestBody.content['application/json'] = {
+            schema: {
+              properties,
+              type: 'object',
+            },
+          };
+        }
       }
-    }
 
-    // Queries are easy in comparison, but still have to be iterated
-    // over and made open-api-able
-    if (r.query) {
-      const { properties, required } = r.query;
-      const props = Object.keys(properties || {});
-      if (props.length) {
-        swaggerRoute.parameters = props
-          .map((prop) => ({
-            in: 'query',
-            name: prop,
-            required: required?.includes(prop),
-            schema: properties[prop],
-          }))
-          .sort(sortSwaggerRequiredAlpha);
+      // Queries are easy in comparison, but still have to be iterated
+      // over and made open-api-able
+      if (r.query) {
+        const { properties, required } = r.query;
+        const props = Object.keys(properties || {});
+        if (props.length) {
+          swaggerRoute.parameters = props
+            .map((prop) => ({
+              in: 'query',
+              name: prop,
+              required: required?.includes(prop),
+              schema: properties[prop],
+            }))
+            .sort(sortSwaggerRequiredAlpha);
+        }
       }
-    }
 
-    // @ts-ignore
-    accum[r.path] = accum[r.path] || {};
-    // @ts-ignore
-    accum[r.path][r.method] = swaggerRoute;
+      // @ts-ignore
+      accum[r.path] = accum[r.path] || {};
+      // @ts-ignore
+      accum[r.path][r.method] = swaggerRoute;
 
-    return accum;
-  }, {});
+      return accum;
+    }, {});
   swaggerJSON.paths = paths;
-  fs.writeFile(swaggerJSONPath, JSON.stringify(swaggerJSON, null, '  '));
+  await fs.writeFile(
+    swaggerJSONMinimal,
+    JSON.stringify(swaggerJSON, null, '  '),
+  );
+  swaggerJSON.info.description = readme + `\n# Changelog\n` + changelog;
+  await fs.writeFile(swaggerJSONPath, JSON.stringify(swaggerJSON, null, '  '));
 };
 
 export default buildOpenAPI;

package/src/browserless.ts

@@ -36,6 +36,12 @@ type Implements<T> = {
   new (...args: unknown[]): T;
 };
 
+type routeInstances =
+  | HTTPRoute
+  | BrowserHTTPRoute
+  | WebSocketRoute
+  | BrowserWebsocketRoute;
+
 export class Browserless {
   protected debug: debug.Debugger = createLogger('index');
   protected browserManager: BrowserManager;
@@ -48,6 +54,7 @@ export class Browserless {
   protected token: Token;
   protected webhooks: WebHooks;
 
+  disabledRouteNames: string[] = [];
   webSocketRouteFiles: string[] = [];
   httpRouteFiles: string[] = [];
   server?: HTTPServer;
@@ -138,6 +145,14 @@ export class Browserless {
     );
   };
 
+  private routeIsDisabled(route: routeInstances) {
+    return this.disabledRouteNames.some((name) => name === route.name);
+  }
+
+  public disableRoutes(...routeNames: string[]) {
+    this.disabledRouteNames.push(...routeNames);
+  }
+
   public addHTTPRoute(httpRouteFilePath: string) {
     this.httpRouteFiles.push(httpRouteFilePath);
   }
@@ -171,7 +186,7 @@ export class Browserless {
       WebkitPlaywright,
     ];
 
-    const [[httpRouteFiles, wsRouteFiles], installedBrowsers] =
+    const [[internalHttpRouteFiles, internalWsRouteFiles], installedBrowsers] =
       await Promise.all([getRouteFiles(this.config), availableBrowsers]);
 
     const docsLink = makeExternalURL(this.config.getExternalAddress(), '/docs');
@@ -180,7 +195,10 @@ export class Browserless {
     this.debug(`Running as user "${userInfo().username}"`);
     this.debug('Starting import of HTTP Routes');
 
-    for (const httpRoute of [...httpRouteFiles, ...this.httpRouteFiles]) {
+    for (const httpRoute of [
+      ...internalHttpRouteFiles,
+      ...this.httpRouteFiles,
+    ]) {
       if (httpRoute.endsWith('js')) {
         const { name } = path.parse(httpRoute);
         const [bodySchema, querySchema] = await Promise.all(
@@ -209,20 +227,26 @@ export class Browserless {
           this.metrics,
           this.monitoring,
         );
-        route.bodySchema = safeParse(bodySchema);
-        route.querySchema = safeParse(querySchema);
-        route.config = () => this.config;
-        route.metrics = () => this.metrics;
-        route.monitoring = () => this.monitoring;
-        route.fileSystem = () => this.fileSystem;
-        route.debug = () => logger;
-
-        httpRoutes.push(route);
+
+        if (!this.routeIsDisabled(route)) {
+          route.bodySchema = safeParse(bodySchema);
+          route.querySchema = safeParse(querySchema);
+          route.config = () => this.config;
+          route.metrics = () => this.metrics;
+          route.monitoring = () => this.monitoring;
+          route.fileSystem = () => this.fileSystem;
+          route.debug = () => logger;
+
+          httpRoutes.push(route);
+        }
       }
     }
 
     this.debug('Starting import of WebSocket Routes');
-    for (const wsRoute of [...wsRouteFiles, ...this.webSocketRouteFiles]) {
+    for (const wsRoute of [
+      ...internalWsRouteFiles,
+      ...this.webSocketRouteFiles,
+    ]) {
       if (wsRoute.endsWith('js')) {
         const { name } = path.parse(wsRoute);
         const [, querySchema] = await Promise.all(
@@ -246,7 +270,6 @@ export class Browserless {
             | Implements<WebSocketRoute>
             | Implements<BrowserWebsocketRoute>;
         } = await import(wsImport + `?cb=${Date.now()}`);
-
         const route = new Route(
           this.browserManager,
           this.config,
@@ -255,19 +278,23 @@ export class Browserless {
           this.metrics,
           this.monitoring,
         );
-        route.querySchema = safeParse(querySchema);
-        route.config = () => this.config;
-        route.metrics = () => this.metrics;
-        route.monitoring = () => this.monitoring;
-        route.fileSystem = () => this.fileSystem;
-        route.debug = () => logger;
-
-        wsRoutes.push(route);
+
+        if (!this.routeIsDisabled(route)) {
+          route.querySchema = safeParse(querySchema);
+          route.config = () => this.config;
+          route.metrics = () => this.metrics;
+          route.monitoring = () => this.monitoring;
+          route.fileSystem = () => this.fileSystem;
+          route.debug = () => logger;
+
+          wsRoutes.push(route);
+        }
       }
     }
 
+    const allRoutes = [...httpRoutes, ...wsRoutes];
     // Validate that we have the browsers they are asking for
-    [...httpRoutes, ...wsRoutes].forEach((route) => {
+    allRoutes.forEach((route) => {
       if (
         'browser' in route &&
         route.browser &&
@@ -280,6 +307,17 @@ export class Browserless {
       }
     });
 
+    const duplicateNamedRoutes = allRoutes
+      .filter((e, i, a) => a.findIndex((r) => r.name === e.name) !== i)
+      .map((r) => r.name);
+
+    if (duplicateNamedRoutes.length) {
+      this.debug(
+        `Found duplicate routing names. Route names must be unique:`,
+        duplicateNamedRoutes,
+      );
+    }
+
     httpRoutes.forEach((r) => this.router.registerHTTPRoute(r));
     wsRoutes.forEach((r) => this.router.registerWebSocketRoute(r));
 

package/src/browsers/chrome.cdp.ts

@@ -5,6 +5,6 @@ import {
 import { ChromiumCDP } from './chromium.cdp.js';
 
 export class ChromeCDP extends ChromiumCDP {
-  protected executablePath = chromeExecutablePath;
+  protected executablePath = chromeExecutablePath();
   protected debug = createLogger('browsers:chrome:cdp');
 }

package/src/browsers/chrome.playwright.ts

@@ -5,6 +5,6 @@ import {
 import { ChromiumPlaywright } from './chromium.playwright.js';
 
 export class ChromePlaywright extends ChromiumPlaywright {
-  protected executablePath = chromeExecutablePath;
+  protected executablePath = chromeExecutablePath();
   protected debug = createLogger('browsers:chrome:playwright');
 }

package/src/browsers/index.ts

@@ -63,6 +63,10 @@ export class BrowserManager {
     }
   };
 
+  protected onNewPage = async (req: Request, page: unknown) => {
+    await pageHook({ meta: req.parsed, page });
+  };
+
   /**
    * Returns the /json/protocol API contents from Chromium or Chrome, whichever is installed,
    * and modifies URLs to set them to the appropriate addresses configured.
@@ -466,7 +470,7 @@ export class BrowserManager {
     await browserHook({ browser, meta: req.parsed });
 
     browser.on('newPage', async (page) => {
-      await pageHook({ meta: req.parsed, page });
+      await this.onNewPage(req, page);
       (router.onNewPage || noop)(req.parsed || '', page);
     });
 

package/src/exports.ts

@@ -1,4 +1,4 @@
-// Main Files
+// Export all them goods...
 export * from './browserless.js';
 export * from './config.js';
 export * from './constants.js';
@@ -16,8 +16,6 @@ export * from './token.js';
 export * from './types.js';
 export * from './utils.js';
 export * from './webhooks.js';
-
-// Browsers and Helpers
 export * from './browsers/index.js';
 export * from './browsers/chrome.cdp.js';
 export * from './browsers/chrome.playwright.js';

package/src/limiter.ts

@@ -77,6 +77,10 @@ export class Limiter extends q {
     this.logQueue('All jobs complete.');
   }
 
+  protected jobEnd(jobInfo: AfterResponse) {
+    afterRequest(jobInfo);
+  }
+
   protected handleSuccess({ detail: { job } }: { detail: { job: Job } }) {
     const timeUsed = Date.now() - job.start;
     this.debug(
@@ -84,7 +88,7 @@ export class Limiter extends q {
     );
     this.metrics.addSuccessful(Date.now() - job.start);
     // @TODO Figure out a better argument handling for jobs
-    afterRequest({
+    this.jobEnd({
       req: job.args[0],
       start: job.start,
       status: 'successful',
@@ -104,7 +108,7 @@ export class Limiter extends q {
     this.webhooks.callTimeoutAlertURL();
     this.debug(`Calling timeout handler`);
     job?.onTimeoutFn(job);
-    afterRequest({
+    this.jobEnd({
       req: job.args[0],
       start: job.start,
       status: 'timedout',
@@ -121,7 +125,7 @@ export class Limiter extends q {
     this.debug(`Recording failed stat, cleaning up: "${error?.toString()}"`);
     this.metrics.addError(Date.now() - job.start);
     this.webhooks.callErrorAlertURL(error?.toString() ?? 'Unknown Error');
-    afterRequest({
+    this.jobEnd({
       req: job.args[0],
       start: job.start,
       status: 'error',

package/src/router.ts

@@ -166,7 +166,9 @@ export class Router {
       : wrapped;
     route.path = Array.isArray(route.path) ? route.path : [route.path];
     const registeredPaths = this.httpRoutes.map((r) => r.path).flat();
-    const duplicatePaths = registeredPaths.filter((path) => route.path.includes(path));
+    const duplicatePaths = registeredPaths.filter((path) =>
+      route.path.includes(path),
+    );
 
     if (duplicatePaths.length) {
       this.log(`Found duplicate routes: ${duplicatePaths.join(', ')}`);
@@ -194,7 +196,9 @@ export class Router {
       : wrapped;
     route.path = Array.isArray(route.path) ? route.path : [route.path];
     const registeredPaths = this.webSocketRoutes.map((r) => r.path).flat();
-    const duplicatePaths = registeredPaths.filter((path) => route.path.includes(path));
+    const duplicatePaths = registeredPaths.filter((path) =>
+      route.path.includes(path),
+    );
 
     if (duplicatePaths.length) {
       this.log(`Found duplicate routes: ${duplicatePaths.join(', ')}`);

package/src/routes/chrome/http/content.post.ts

@@ -4,9 +4,14 @@ import {
   QuerySchema,
   ResponseSchema,
 } from '../../../shared/content.http.js';
-import { ChromeCDP, HTTPRoutes } from '@browserless.io/browserless';
+import {
+  BrowserlessRoutes,
+  ChromeCDP,
+  HTTPRoutes,
+} from '@browserless.io/browserless';
 
-export default class ChromeContentRoute extends Content {
+export default class ChromeContentPostRoute extends Content {
+  name = BrowserlessRoutes.ChromeContentPostRoute;
   browser = ChromeCDP;
   path = [HTTPRoutes.chromeContent];
 }

package/src/routes/chrome/http/download.post.ts

@@ -4,9 +4,14 @@ import {
   QuerySchema,
   ResponseSchema,
 } from '../../../shared/download.http.js';
-import { ChromeCDP, HTTPRoutes } from '@browserless.io/browserless';
+import {
+  BrowserlessRoutes,
+  ChromeCDP,
+  HTTPRoutes,
+} from '@browserless.io/browserless';
 
-export default class ChromeDownloadRoute extends Download {
+export default class ChromeDownloadPostRoute extends Download {
+  name = BrowserlessRoutes.ChromeDownloadPostRoute;
   browser = ChromeCDP;
   path = [HTTPRoutes.chromeDownload];
 }

package/src/routes/chrome/http/function.post.ts

@@ -4,9 +4,14 @@ import {
   QuerySchema,
   ResponseSchema,
 } from '../../../shared/function.http.js';
-import { ChromeCDP, HTTPRoutes } from '@browserless.io/browserless';
+import {
+  BrowserlessRoutes,
+  ChromeCDP,
+  HTTPRoutes,
+} from '@browserless.io/browserless';
 
-export default class ChromeFunctionRoute extends Function {
+export default class ChromeFunctionPostRoute extends Function {
+  name = BrowserlessRoutes.ChromeFunctionPostRoute;
   browser = ChromeCDP;
   path = [HTTPRoutes.chromeFunction];
 }

package/src/routes/chrome/http/json-list.get.ts

@@ -1 +1,7 @@
-export { default, ResponseSchema } from '../../../shared/json-list.http.js';
+import { BrowserlessRoutes } from '@browserless.io/browserless';
+export { ResponseSchema } from '../../../shared/json-list.http.js';
+import { default as ChromiumJSONListGetRoute } from '../../../shared/json-list.http.js';
+
+export default class ChromeJSONListGetRoute extends ChromiumJSONListGetRoute {
+  name = BrowserlessRoutes.ChromeJSONListGetRoute;
+}

package/src/routes/chrome/http/json-new.put.ts

@@ -1 +1,7 @@
-export { default, ResponseSchema } from '../../../shared/json-new.http.js';
+import { BrowserlessRoutes } from '@browserless.io/browserless';
+export { ResponseSchema } from '../../../shared/json-new.http.js';
+import { default as ChromiumJSONNewPutRoute } from '../../../shared/json-new.http.js';
+
+export default class ChromeJSONNewPutRoute extends ChromiumJSONNewPutRoute {
+  name = BrowserlessRoutes.ChromeJSONNewPutRoute;
+}