@browserless.io/browserless 2.0.0-beta-1 → 2.0.0-beta-4

package/src/server.ts

@@ -2,19 +2,15 @@ import * as http from 'http';
 import * as stream from 'stream';
 import {
   BadRequest,
-  BrowserHTTPRoute,
-  BrowserManager,
-  BrowserWebsocketRoute,
   Config,
-  HTTPManagementRoutes,
   HTTPRoute,
-  Limiter,
-  Methods,
   Metrics,
   NotFound,
   Request,
   Response,
+  Router,
   Timeout,
+  Token,
   TooManyRequests,
   Unauthorized,
   WebSocketRoute,
@@ -22,8 +18,6 @@ import {
   contentTypes,
   convertPathToURL,
   createLogger,
-  isAuthorized,
-  isConnected,
   queryParamsToObject,
   readBody,
   shimLegacyRequests,
@@ -32,7 +26,6 @@ import {
 
 // @ts-ignore
 import Enjoi from 'enjoi';
-import micromatch from 'micromatch';
 
 export interface HTTPServerOptions {
   concurrent: number;
@@ -43,26 +36,20 @@ export interface HTTPServerOptions {
 }
 
 export class HTTPServer {
-  private server: http.Server = http.createServer();
-  private port: number;
-  private host?: string;
-  private log = createLogger('server');
-  private verbose = createLogger('server:verbose');
+  protected server: http.Server = http.createServer();
+  protected port: number;
+  protected host?: string;
+  protected log = createLogger('server');
+  protected verbose = createLogger('server:verbose');
 
   constructor(
-    private config: Config,
-    private metrics: Metrics,
-    private browserManager: BrowserManager,
-    private limiter: Limiter,
-    private httpRoutes: Array<HTTPRoute | BrowserHTTPRoute>,
-    private webSocketRoutes: Array<WebSocketRoute | BrowserWebsocketRoute>,
+    protected config: Config,
+    protected metrics: Metrics,
+    protected token: Token,
+    protected router: Router,
   ) {
     this.host = config.getHost();
     this.port = config.getPort();
-    this.httpRoutes = httpRoutes.map((r) => this.registerHTTPRoute(r));
-    this.webSocketRoutes = webSocketRoutes.map((r) =>
-      this.registerWebSocketRoute(r),
-    );
 
     this.log(
       `Server instantiated with host "${this.host}" on port "${
@@ -71,33 +58,16 @@ export class HTTPServer {
     );
   }
 
-  private onQueueFullHTTP = (_req: Request, res: Response) => {
-    this.log(`Queue is full, sending 429 response`);
-    return writeResponse(res, 429, 'Too many requests');
-  };
-
-  private onQueueFullWebSocket = (_req: Request, socket: stream.Duplex) => {
-    this.log(`Queue is full, sending 429 response`);
-    return writeResponse(socket, 429, 'Too many requests');
-  };
-
-  private onHTTPTimeout = (_req: Request, res: Response) => {
-    this.log(`HTTP job has timedout, sending 429 response`);
-    return writeResponse(res, 408, 'Request has timed out');
-  };
-
-  private onWebsocketTimeout = (_req: Request, socket: stream.Duplex) => {
-    this.log(`Websocket job has timedout, sending 429 response`);
-    return writeResponse(socket, 408, 'Request has timed out');
-  };
-
-  private onHTTPUnauthorized = (_req: Request, res: Response) => {
+  protected onHTTPUnauthorized = (_req: Request, res: Response) => {
     this.log(`HTTP request is not properly authorized, responding with 401`);
     this.metrics.addUnauthorized();
     return writeResponse(res, 401, 'Bad or missing authentication.');
   };
 
-  private onWebsocketUnauthorized = (_req: Request, socket: stream.Duplex) => {
+  protected onWebsocketUnauthorized = (
+    _req: Request,
+    socket: stream.Duplex,
+  ) => {
     this.log(
       `Websocket request is not properly authorized, responding with 401`,
     );
@@ -105,141 +75,6 @@ export class HTTPServer {
     return writeResponse(socket, 401, 'Bad or missing authentication.');
   };
 
-  private wrapHTTPHandler =
-    (
-      route: HTTPRoute | BrowserHTTPRoute,
-      handler: HTTPRoute['handler'] | BrowserHTTPRoute['handler'],
-    ) =>
-    async (req: Request, res: Response) => {
-      if (!isConnected(res)) {
-        this.log(`HTTP Request has closed prior to running`);
-        return Promise.resolve();
-      }
-
-      if (route.browser) {
-        const browser = await this.browserManager.getBrowserForRequest(
-          req,
-          route,
-        );
-
-        if (!isConnected(res)) {
-          this.log(`HTTP Request has closed prior to running`);
-          this.browserManager.complete(browser);
-          return Promise.resolve();
-        }
-
-        if (!browser) {
-          return writeResponse(res, 500, `Error loading the browser.`);
-        }
-
-        if (!isConnected(res)) {
-          this.log(`HTTP Request has closed prior to running`);
-          return Promise.resolve();
-        }
-
-        try {
-          this.verbose(`Running found HTTP handler.`);
-          return await handler(req, res, browser);
-        } finally {
-          this.verbose(`HTTP Request handler has finished.`);
-          this.browserManager.complete(browser);
-        }
-      }
-
-      return (handler as HTTPRoute['handler'])(req, res);
-    };
-
-  private wrapWebSocketHandler =
-    (
-      route: WebSocketRoute | BrowserWebsocketRoute,
-      handler: WebSocketRoute['handler'] | BrowserWebsocketRoute['handler'],
-    ) =>
-    async (req: Request, socket: stream.Duplex, head: Buffer) => {
-      if (!isConnected(socket)) {
-        this.log(`WebSocket Request has closed prior to running`);
-        return Promise.resolve();
-      }
-
-      if (route.browser) {
-        const browser = await this.browserManager.getBrowserForRequest(
-          req,
-          route,
-        );
-
-        if (!isConnected(socket)) {
-          this.log(`WebSocket Request has closed prior to running`);
-          this.browserManager.complete(browser);
-          return Promise.resolve();
-        }
-
-        if (!browser) {
-          return writeResponse(socket, 500, `Error loading the browser.`);
-        }
-
-        try {
-          this.verbose(`Running found WebSocket handler.`);
-          await handler(req, socket, head, browser);
-        } finally {
-          this.verbose(`WebSocket Request handler has finished.`);
-          this.browserManager.complete(browser);
-        }
-        return;
-      }
-      return (handler as WebSocketRoute['handler'])(req, socket, head);
-    };
-
-  private getTimeout(req: Request) {
-    const timer = req.parsed.searchParams.get('timeout');
-
-    return timer ? +timer : undefined;
-  }
-
-  private registerHTTPRoute(
-    route: HTTPRoute | BrowserHTTPRoute,
-  ): HTTPRoute | BrowserHTTPRoute {
-    this.verbose(
-      `Registering HTTP ${route.method.toUpperCase()} ${route.path}`,
-    );
-
-    route._browserManager = () => this.browserManager;
-
-    const bound = route.handler.bind(route);
-    const wrapped = this.wrapHTTPHandler(route, bound);
-
-    route.handler = route.concurrency
-      ? this.limiter.limit(
-          wrapped,
-          this.onQueueFullHTTP,
-          this.onHTTPTimeout,
-          this.getTimeout,
-        )
-      : wrapped;
-
-    return route;
-  }
-
-  private registerWebSocketRoute(
-    route: WebSocketRoute | BrowserWebsocketRoute,
-  ): WebSocketRoute | BrowserWebsocketRoute {
-    this.verbose(`Registering WebSocket "${route.path}"`);
-
-    route._browserManager = () => this.browserManager;
-
-    const bound = route.handler.bind(route);
-    const wrapped = this.wrapWebSocketHandler(route, bound);
-
-    route.handler = route.concurrency
-      ? this.limiter.limit(
-          wrapped,
-          this.onQueueFullWebSocket,
-          this.onWebsocketTimeout,
-          this.getTimeout,
-        )
-      : wrapped;
-
-    return route;
-  }
-
   public async start(): Promise<void> {
     this.log(`HTTP Server is starting`);
 
@@ -268,21 +103,19 @@ export class HTTPServer {
   public async stop(): Promise<void> {
     this.log(`HTTP Server is shutting down`);
     await new Promise((r) => this.server.close(r));
-    await Promise.all([this.tearDown(), this.browserManager.stop()]);
+    await Promise.all([this.tearDown(), this.router.teardown()]);
     this.log(`HTTP Server shutdown complete`);
   }
 
-  private tearDown() {
+  protected tearDown() {
     this.log(`Tearing down all listeners and internal routes`);
     this.server && this.server.removeAllListeners();
-    this.httpRoutes = [];
-    this.webSocketRoutes = [];
 
     // @ts-ignore garbage collect this reference
     this.server = null;
   }
 
-  private handleRequest = async (
+  protected handleRequest = async (
     request: http.IncomingMessage,
     res: http.ServerResponse,
   ) => {
@@ -297,10 +130,6 @@ export class HTTPServer {
 
     if (!proceed) return;
 
-    const staticHandler = this.httpRoutes.find(
-      (route) => route.path === HTTPManagementRoutes.static,
-    ) as HTTPRoute;
-
     if (this.config.getAllowCORS()) {
       Object.entries(this.config.getCORSHeaders()).forEach(([header, value]) =>
         res.setHeader(header, value),
@@ -323,36 +152,19 @@ export class HTTPServer {
       req.parsed.searchParams.delete('body');
     }
 
-    const accepts = (req.headers['accept']?.toLowerCase() || '*/*').split(',');
-    const contentType = req.headers['content-type']?.toLowerCase() as
-      | contentTypes
-      | undefined;
-
-    const found =
-      this.httpRoutes.find(
-        (r) =>
-          micromatch.isMatch(req.parsed.pathname, r.path) &&
-          r.method === (req.method?.toLocaleLowerCase() as Methods) &&
-          (accepts.some((a) => a.startsWith('*/*')) ||
-            r.contentTypes.some((contentType) =>
-              accepts.includes(contentType),
-            )) &&
-          ((!contentType && r.accepts.includes(contentTypes.any)) ||
-            r.accepts.includes(contentType as contentTypes)),
-      ) || (req.method?.toLowerCase() === 'get' ? staticHandler : null);
-
-    if (!found) {
+    const route = await this.router.getRouteForHTTPRequest(req);
+
+    if (!route) {
       this.log(`No matching WebSocket route handler for "${req.parsed.href}"`);
       writeResponse(res, 404, 'Not Found');
       return Promise.resolve();
     }
 
-    this.verbose(`Found matching HTTP route handler "${found.path}"`);
+    this.verbose(`Found matching HTTP route handler "${route.path}"`);
 
-    if (found?.auth) {
+    if (route?.auth) {
       this.verbose(`Authorizing HTTP request to "${request.url}"`);
-      const tokens = this.config.getToken();
-      const isPermitted = isAuthorized(req, found, tokens);
+      const isPermitted = await this.token.isAuthorized(req, route);
 
       if (!isPermitted) {
         return this.onHTTPUnauthorized(req, res);
@@ -365,8 +177,8 @@ export class HTTPServer {
 
     if (
       ((req.headers['content-type']?.includes(contentTypes.json) ||
-        (found.accepts.length === 1 &&
-          found.accepts.includes(contentTypes.json))) &&
+        (route.accepts.length === 1 &&
+          route.accepts.includes(contentTypes.json))) &&
         typeof body !== 'object') ||
       body === null
     ) {
@@ -374,10 +186,10 @@ export class HTTPServer {
       return Promise.resolve();
     }
 
-    if (found.querySchema) {
+    if (route.querySchema) {
       this.verbose(`Validating route query-params with QUERY schema`);
       try {
-        const schema = Enjoi.schema(found.querySchema);
+        const schema = Enjoi.schema(route.querySchema);
         const valid = schema.validate(req.queryParams, {
           abortEarly: false,
         });
@@ -419,10 +231,10 @@ export class HTTPServer {
       }
     }
 
-    if (found.bodySchema) {
+    if (route.bodySchema) {
       this.verbose(`Validating route payload with BODY schema`);
       try {
-        const schema = Enjoi.schema(found.bodySchema);
+        const schema = Enjoi.schema(route.bodySchema);
         const valid = schema.validate(body, { abortEarly: false });
 
         if (valid.error) {
@@ -460,9 +272,7 @@ export class HTTPServer {
       }
     }
 
-    // #wrapHTTPHandler will take care of applying the extra browser
-    // argument for this to to work properly
-    return (found as HTTPRoute)
+    return (route as HTTPRoute)
       .handler(req, res)
       .then(() => {
         this.verbose('HTTP connection complete');
@@ -488,12 +298,12 @@ export class HTTPServer {
           return writeResponse(res, 408, e.message);
         }
 
-        this.log(`Error handling request at "${found.path}": ${e}`);
+        this.log(`Error handling request at "${route.path}": ${e}`);
         return writeResponse(res, 500, e.toString());
       });
   };
 
-  private handleWebSocket = async (
+  protected handleWebSocket = async (
     request: http.IncomingMessage,
     socket: stream.Duplex,
     head: Buffer,
@@ -507,29 +317,26 @@ export class HTTPServer {
 
     if (!proceed) return;
 
-    const { pathname } = req.parsed;
     req.queryParams = queryParamsToObject(req.parsed.searchParams);
 
-    const found = this.webSocketRoutes.find((r) =>
-      micromatch.isMatch(pathname, r.path),
-    );
+    const route = await this.router.getRouteForWebSocketRequest(req);
 
-    if (found) {
-      this.verbose(`Found matching WebSocket route handler "${found.path}"`);
+    if (route) {
+      this.verbose(`Found matching WebSocket route handler "${route.path}"`);
 
-      if (found?.auth) {
+      if (route?.auth) {
         this.verbose(`Authorizing WebSocket request to "${req.parsed.href}"`);
-        const isPermitted = isAuthorized(req, found, this.config.getToken());
+        const isPermitted = await this.token.isAuthorized(req, route);
 
         if (!isPermitted) {
           return this.onWebsocketUnauthorized(req, socket);
         }
       }
 
-      if (found.querySchema) {
+      if (route.querySchema) {
         this.verbose(`Validating route query-params with QUERY schema`);
         try {
-          const schema = Enjoi.schema(found.querySchema);
+          const schema = Enjoi.schema(route.querySchema);
           const valid = schema.validate(req.queryParams, {
             abortEarly: false,
           });
@@ -571,9 +378,7 @@ export class HTTPServer {
         }
       }
 
-      // #wrapWebSocketHandler will take care of applying the extra browser
-      // argument for this to to work properly
-      return (found as WebSocketRoute)
+      return (route as WebSocketRoute)
         .handler(req, socket, head)
         .then(() => {
           this.verbose('Websocket connection complete');
@@ -596,7 +401,7 @@ export class HTTPServer {
           }
 
           this.log(
-            `Error handling request at "${found.path}": ${e}\n${e.stack}`,
+            `Error handling request at "${route.path}": ${e}\n${e.stack}`,
           );
 
           return writeResponse(socket, 500, e.message);

package/src/token.ts

@@ -0,0 +1,40 @@
+import {
+  BrowserHTTPRoute,
+  BrowserWebsocketRoute,
+  Config,
+  HTTPRoute,
+  Request,
+  WebSocketRoute,
+  getTokenFromRequest,
+} from '@browserless.io/browserless';
+
+export class Token {
+  constructor(protected config: Config) {}
+
+  public isAuthorized = async (
+    req: Request,
+    route:
+      | BrowserHTTPRoute
+      | BrowserWebsocketRoute
+      | HTTPRoute
+      | WebSocketRoute,
+  ): Promise<boolean> => {
+    const token = this.config.getToken();
+
+    if (token === null) {
+      return true;
+    }
+
+    if (route.auth !== true) {
+      return true;
+    }
+
+    const requestToken = getTokenFromRequest(req);
+
+    if (!requestToken) {
+      return false;
+    }
+
+    return (Array.isArray(token) ? token : [token]).includes(requestToken);
+  };
+}

package/src/types.ts

@@ -2,15 +2,13 @@ import * as http from 'http';
 import * as stream from 'stream';
 import {
   APITags,
-  BrowserManager,
+  Browserless,
   CDPChromium,
   Config,
-  FileSystem,
   HTTPManagementRoutes,
   HTTPRoutes,
   Methods,
   Metrics,
-  Monitoring,
   PlaywrightChromium,
   PlaywrightFirefox,
   PlaywrightWebkit,
@@ -81,54 +79,103 @@ export interface BrowserJSON {
   'WebKit-Version': string;
 }
 
+/**
+ * The default launch options or a function, accepting
+ * the request object, that produces the launch options.
+ */
 type defaultLaunchOptions =
   | CDPLaunchOptions
   | BrowserlessLaunch
   | ((req: Request) => CDPLaunchOptions | BrowserlessLaunch);
 
 interface Route {
-  _browserManager?: () => BrowserManager;
-  _config?: () => Config;
-  _debug?: () => debug.Debugger;
-  _fileSystem?: () => FileSystem;
-  _metrics?: () => Metrics;
-  _monitor?: () => Monitoring;
-
   /**
-   * Whether the route requires a token to access.
+   * A boolean, or a function that returns a boolean, on
+   * whether the route requires an API token to access.
    */
   auth: boolean | ((req: Request) => Promise<boolean>);
 
   /**
    * The schematic of the submitted BODY (typically)
-   * an object when the route is json-based.
+   * an object when the route is json-based. This is generated
+   * automatically if your route defines a BodySchema type.
    */
   bodySchema?: unknown;
 
   /**
    * Whether the route should be bound by the global
-   * concurrency limit
+   * concurrency limit defined in your configuration.
    */
   concurrency: boolean;
 
   /**
-   * Description of the route and what it does
+   * Description of the route and what it does. This description
+   * is then used in the embedded documentation site.
    */
   description: string;
 
   /**
-   * The HTTP path that this route handles
+   * Helper function to load the browser-manager instance. Defined
+   * and injected by browserless after initialization.
+   * @returns BrowserManager
+   */
+  getBrowserManager?: () => Browserless['browserManager'];
+
+  /**
+   * Helper function that loads the config module. Defined and injected by
+   * browserless after initialization.
+   * @returns Config
+   */
+  getConfig?: () => Browserless['config'];
+
+  /**
+   * Helper function that loads the debug module, useful
+   * for logging messages scoped to the routes path. Defined
+   * and injected by browserless after initialization.
+   * @returns Debug
+   */
+  getDebug?: () => debug.Debugger;
+
+  /**
+   * Helper function that loads the file-system module
+   * for interacting with file-systems. Defined and injected by
+   * browserless after initialization.
+   * @returns FileSystem
+   */
+  getFileSystem?: () => Browserless['fileSystem'];
+
+  /**
+   * Helper function that loads the metrics module for
+   * collecting and aggregating statistics. Defined and injected by
+   * browserless after initialization.
+   * @returns Metrics
+   */
+  getMetrics?: () => Browserless['metrics'];
+
+  /**
+   * Helper function that loads the monitoring module useful
+   * for monitoring system health. Defined and injected by
+   * browserless after initialization.
+   * @returns Monitor
+   */
+  getMonitoring?: () => Browserless['monitoring'];
+
+  /**
+   * The HTTP path that this route handles, eg '/my-route'
    */
   path: HTTPRoutes | WebsocketRoutes | HTTPManagementRoutes | string;
 
   /**
    * The query parameters accepted by the route, defined in
-   * an object format.
+   * an object format. This is auto-generated for you if your
+   * route defines and exports a QuerySchema type.
    */
   querySchema?: unknown;
 
   /**
-   * The structure of the routes response when successful
+   * The structure of the routes response when successful. This
+   * is auto-generated for you if your route defines a ResponseSchema
+   * type and exports it in your route.
    */
   responseSchema?: unknown;
 
@@ -139,6 +186,11 @@ interface Route {
   tags: APITags[];
 }
 
+/**
+ * A primitive HTTP-based route that doesn't require a
+ * browser in order to fulfill requests. Used by downstream HTTPRoute
+ * and WebSocketRoute
+ */
 interface BasicHTTPRoute extends Route {
   /**
    * The allowed Content-Types that this route can read and handle.
@@ -160,6 +212,10 @@ interface BasicHTTPRoute extends Route {
   method: Methods;
 }
 
+/**
+ * A HTTP-based route, with a handler, that can fulfill requests without
+ * a browser required.
+ */
 export interface HTTPRoute extends BasicHTTPRoute {
   browser: null;
 
@@ -169,6 +225,11 @@ export interface HTTPRoute extends BasicHTTPRoute {
   handler: (req: Request, res: http.ServerResponse) => Promise<unknown>;
 }
 
+/**
+ * A HTTP-based route, with a handler, that can fulfill requests but
+ * requires a browser in order to do so. Handler will then be called
+ * with a 3rd argument of the browser class specified.
+ */
 export interface BrowserHTTPRoute extends BasicHTTPRoute {
   browser: BrowserClasses;
 
@@ -187,6 +248,10 @@ export interface BrowserHTTPRoute extends BasicHTTPRoute {
   onNewPage?: undefined;
 }
 
+/**
+ * A WebSocket-based route, with a handler, that can fulfill requests
+ * that do not require a browser in order to operate.
+ */
 export interface WebSocketRoute extends Route {
   browser: null;
 
@@ -200,6 +265,11 @@ export interface WebSocketRoute extends Route {
   ) => Promise<unknown>;
 }
 
+/**
+ * A WebSocket-based route, with a handler, that can fulfill requests
+ * that need a browser. Handler is called with an additional argument of
+ * browser (the browser class required to run the route).
+ */
 export interface BrowserWebsocketRoute extends Route {
   browser: BrowserClasses;
 
@@ -216,6 +286,10 @@ export interface BrowserWebsocketRoute extends Route {
     browser: BrowserInstance,
   ): Promise<unknown>;
 
+  /**
+   * An optional function to automatically set up or handle new page
+   * creation. Useful for injecting behaviors or other functionality.
+   */
   onNewPage?: (url: URL, page: Page) => Promise<void>;
 }