@browserless.io/browserless 2.8.0 → 2.10.0

package/src/monitoring.ts

@@ -1,13 +1,9 @@
-import {
-  Config,
-  IResourceLoad,
-  createLogger,
-} from '@browserless.io/browserless';
+import { Config, IResourceLoad, Logger } from '@browserless.io/browserless';
 import { EventEmitter } from 'events';
 import si from 'systeminformation';
 
 export class Monitoring extends EventEmitter {
-  protected log = createLogger('hardware');
+  protected log = new Logger('hardware');
   constructor(protected config: Config) {
     super();
   }
@@ -17,7 +13,7 @@ export class Monitoring extends EventEmitter {
       si.currentLoad(),
       si.mem(),
     ]).catch((err) => {
-      this.log(`Error checking machine stats`, err);
+      this.log.error(`Error checking machine stats`, err);
       return [null, null];
     });
 
@@ -40,7 +36,9 @@ export class Monitoring extends EventEmitter {
     const cpuInt = cpu && Math.ceil(cpu * 100);
     const memoryInt = memory && Math.ceil(memory * 100);
 
-    this.log(`Checking overload status: CPU ${cpuInt}% Memory ${memoryInt}%`);
+    this.log.info(
+      `Checking overload status: CPU ${cpuInt}% Memory ${memoryInt}%`,
+    );
 
     const cpuOverloaded = !!(cpuInt && cpuInt >= this.config.getCPULimit());
     const memoryOverloaded = !!(

package/src/router.ts

@@ -13,7 +13,6 @@ import {
   Response,
   WebSocketRoute,
   contentTypes,
-  createLogger,
   isConnected,
   writeResponse,
 } from '@browserless.io/browserless';
@@ -22,8 +21,7 @@ import micromatch from 'micromatch';
 import stream from 'stream';
 
 export class Router extends EventEmitter {
-  protected log = createLogger('router');
-  protected verbose = createLogger('router:verbose');
+  protected log = new Logger('router');
   protected httpRoutes: Array<HTTPRoute | BrowserHTTPRoute> = [];
   protected webSocketRoutes: Array<WebSocketRoute | BrowserWebsocketRoute> = [];
 
@@ -43,22 +41,22 @@ export class Router extends EventEmitter {
   }
 
   protected onQueueFullHTTP = (_req: Request, res: Response) => {
-    this.log(`Queue is full, sending 429 response`);
+    this.log.warn(`Queue is full, sending 429 response`);
     return writeResponse(res, 429, 'Too many requests');
   };
 
   protected onQueueFullWebSocket = (_req: Request, socket: stream.Duplex) => {
-    this.log(`Queue is full, sending 429 response`);
+    this.log.warn(`Queue is full, sending 429 response`);
     return writeResponse(socket, 429, 'Too many requests');
   };
 
   protected onHTTPTimeout = (_req: Request, res: Response) => {
-    this.log(`HTTP job has timedout, sending 429 response`);
+    this.log.error(`HTTP job has timedout, sending 429 response`);
     return writeResponse(res, 408, 'Request has timed out');
   };
 
   protected onWebsocketTimeout = (_req: Request, socket: stream.Duplex) => {
-    this.log(`Websocket job has timedout, sending 429 response`);
+    this.log.error(`Websocket job has timedout, sending 429 response`);
     return writeResponse(socket, 408, 'Request has timed out');
   };
 
@@ -69,7 +67,7 @@ export class Router extends EventEmitter {
     ) =>
     async (req: Request, res: Response) => {
       if (!isConnected(res)) {
-        this.log(`HTTP Request has closed prior to running`);
+        this.log.warn(`HTTP Request has closed prior to running`);
         return Promise.resolve();
       }
       const logger = new this.logger(route.name, req);
@@ -77,10 +75,11 @@ export class Router extends EventEmitter {
         const browser = await this.browserManager.getBrowserForRequest(
           req,
           route,
+          logger,
         );
 
         if (!isConnected(res)) {
-          this.log(`HTTP Request has closed prior to running`);
+          this.log.warn(`HTTP Request has closed prior to running`);
           this.browserManager.complete(browser);
           return Promise.resolve();
         }
@@ -90,7 +89,7 @@ export class Router extends EventEmitter {
         }
 
         try {
-          this.verbose(`Running found HTTP handler.`);
+          this.log.trace(`Running found HTTP handler.`);
           return await Promise.race([
             handler(req, res, logger, browser),
             new Promise((resolve, reject) => {
@@ -98,13 +97,13 @@ export class Router extends EventEmitter {
                 if (!res.writableEnded) {
                   reject(new Error(`Request closed prior to writing results`));
                 }
-                this.verbose(`Response has been written, resolving`);
+                this.log.trace(`Response has been written, resolving`);
                 resolve(null);
               });
             }),
           ]);
         } finally {
-          this.verbose(`HTTP Request handler has finished.`);
+          this.log.trace(`HTTP Request handler has finished.`);
           this.browserManager.complete(browser);
         }
       }
@@ -119,7 +118,7 @@ export class Router extends EventEmitter {
     ) =>
     async (req: Request, socket: stream.Duplex, head: Buffer) => {
       if (!isConnected(socket)) {
-        this.log(`WebSocket Request has closed prior to running`);
+        this.log.warn(`WebSocket Request has closed prior to running`);
         return Promise.resolve();
       }
       const logger = new this.logger(route.name, req);
@@ -127,10 +126,11 @@ export class Router extends EventEmitter {
         const browser = await this.browserManager.getBrowserForRequest(
           req,
           route,
+          logger,
         );
 
         if (!isConnected(socket)) {
-          this.log(`WebSocket Request has closed prior to running`);
+          this.log.warn(`WebSocket Request has closed prior to running`);
           this.browserManager.complete(browser);
           return Promise.resolve();
         }
@@ -140,10 +140,10 @@ export class Router extends EventEmitter {
         }
 
         try {
-          this.verbose(`Running found WebSocket handler.`);
+          this.log.trace(`Running found WebSocket handler.`);
           await handler(req, socket, head, logger, browser);
         } finally {
-          this.verbose(`WebSocket Request handler has finished.`);
+          this.log.trace(`WebSocket Request handler has finished.`);
           this.browserManager.complete(browser);
         }
         return;
@@ -154,7 +154,7 @@ export class Router extends EventEmitter {
   public registerHTTPRoute(
     route: HTTPRoute | BrowserHTTPRoute,
   ): HTTPRoute | BrowserHTTPRoute {
-    this.verbose(
+    this.log.trace(
       `Registering HTTP ${route.method.toUpperCase()} ${route.path}`,
     );
 
@@ -176,7 +176,7 @@ export class Router extends EventEmitter {
     );
 
     if (duplicatePaths.length) {
-      this.log(`Found duplicate routes: ${duplicatePaths.join(', ')}`);
+      this.log.warn(`Found duplicate routes: ${duplicatePaths.join(', ')}`);
     }
     this.httpRoutes.push(route);
 
@@ -186,7 +186,7 @@ export class Router extends EventEmitter {
   public registerWebSocketRoute(
     route: WebSocketRoute | BrowserWebsocketRoute,
   ): WebSocketRoute | BrowserWebsocketRoute {
-    this.verbose(`Registering WebSocket "${route.path}"`);
+    this.log.trace(`Registering WebSocket "${route.path}"`);
 
     const bound = route.handler.bind(route);
     const wrapped = this.wrapWebSocketHandler(route, bound);
@@ -206,7 +206,7 @@ export class Router extends EventEmitter {
     );
 
     if (duplicatePaths.length) {
-      this.log(`Found duplicate routes: ${duplicatePaths.join(', ')}`);
+      this.log.warn(`Found duplicate routes: ${duplicatePaths.join(', ')}`);
     }
     this.webSocketRoutes.push(route);
     return route;

package/src/routes/management/http/pressure.get.ts

@@ -0,0 +1,135 @@
+import {
+  APITags,
+  BrowserlessRoutes,
+  HTTPManagementRoutes,
+  HTTPRoute,
+  Methods,
+  Request,
+  contentTypes,
+  jsonResponse,
+} from '@browserless.io/browserless';
+import { ServerResponse } from 'http';
+
+export type ResponseSchema = {
+  /**
+   * An integer representing the percentage of CPU being used. For instance 92 means 92%
+   */
+  cpu: number | null;
+
+  /**
+   * A number of milliseconds since epoch, or "Date.now()" equivalent.
+   */
+  date: number;
+
+  /**
+   * Whether or not a session can be connected and immediately ran on a health instance.
+   */
+  isAvailable: boolean;
+
+  /**
+   * The maximum amount of browsers that can be ran at a single time.
+   */
+  maxConcurrent: number;
+
+  /**
+   * The maximum amount of queued connections allowed at a single time.
+   */
+  maxQueued: number;
+
+  /**
+   * An integer representing the percentage of Memory being used. For instance 95 means 95%
+   */
+  memory: number | null;
+
+  /**
+   * A human-readable message as the overall status of the instance.
+   */
+  message: string;
+
+  /**
+   * The current number of connect or API calls pending to run.
+   */
+  queued: number;
+
+  /**
+   * A simple single-word reason as to why an instance may or may not be available.
+   */
+  reason: 'full' | 'cpu' | 'memory' | '';
+
+  /**
+   * The number of recent connections that were rejected due to the queue and concurrency
+   * limits having been filled.
+   */
+  recentlyRejected: number;
+
+  /**
+   * The current number of running connections or API calls.
+   */
+  running: number;
+};
+
+export default class PressureGetRoute extends HTTPRoute {
+  name = BrowserlessRoutes.PressureGetRoute;
+  accepts = [contentTypes.any];
+  auth = true;
+  browser = null;
+  concurrency = false;
+  contentTypes = [contentTypes.json];
+  description = `Returns a JSON body of stats related to the pressure being created on the instance.`;
+  method = Methods.get;
+  path = HTTPManagementRoutes.pressure;
+  tags = [APITags.management];
+  handler = async (_req: Request, res: ServerResponse): Promise<void> => {
+    const monitoring = this.monitoring();
+    const config = this.config();
+    const limiter = this.limiter();
+    const metrics = this.metrics();
+
+    const {
+      cpuInt: cpu,
+      memoryInt: memory,
+      cpuOverloaded,
+      memoryOverloaded,
+    } = await monitoring.overloaded();
+    const date = Date.now();
+    const hasCapacity = limiter.hasCapacity;
+    const queued = limiter.waiting;
+    const isAvailable = hasCapacity && !cpuOverloaded && !memoryOverloaded;
+    const running = limiter.executing;
+    const recentlyRejected = metrics.get().rejected;
+    const maxConcurrent = config.getConcurrent();
+    const maxQueued = config.getQueued();
+
+    const reason = !hasCapacity
+      ? 'full'
+      : cpuOverloaded
+        ? 'cpu'
+        : memoryOverloaded
+          ? 'memory'
+          : '';
+
+    const message = !hasCapacity
+      ? 'Concurrency and queue are full'
+      : cpuOverloaded
+        ? 'CPU is over the configured maximum for cpu percent'
+        : memoryOverloaded
+          ? 'Memory is over the configured maximum for memory percent'
+          : '';
+
+    const response: ResponseSchema = {
+      cpu,
+      date,
+      isAvailable,
+      maxConcurrent,
+      maxQueued,
+      memory,
+      message,
+      queued,
+      reason,
+      recentlyRejected,
+      running,
+    };
+
+    return jsonResponse(res, 200, response);
+  };
+}

package/src/routes/management/http/static.get.ts

@@ -31,7 +31,7 @@ const streamFile = (
 ) =>
   new Promise((resolve, reject) => {
     if (contentType) {
-      logger.log(`Setting content-type ${contentType}`);
+      logger.debug(`Setting content-type ${contentType}`);
       res.setHeader('Content-Type', contentType);
     }
 
@@ -96,15 +96,13 @@ export default class StaticGetRoute extends HTTPRoute {
     }
 
     if (foundFilePaths.length > 1) {
-      logger.log(
+      logger.warn(
         `Multiple files found for request to "${pathname}". Only the first file is served, so please name your files uniquely.`,
       );
     }
 
     const [foundFilePath] = foundFilePaths;
-    logger.verbose(
-      `Found new file "${foundFilePath}", caching path and serving`,
-    );
+    logger.info(`Found new file "${foundFilePath}", caching path and serving`);
 
     const contentType = mimeTypes.get(path.extname(foundFilePath));
 

package/src/routes/management/tests/management.spec.ts

@@ -56,6 +56,19 @@ describe('Management APIs', function () {
     );
   });
 
+  it('allows requests to /pressure', async () => {
+    await start();
+
+    await fetch('http://localhost:3000/pressure?token=6R0W53R135510').then(
+      async (res) => {
+        expect(res.headers.get('content-type')).to.equal(
+          'application/json; charset=UTF-8',
+        );
+        expect(res.status).to.equal(200);
+      },
+    );
+  });
+
   it('allows requests to /sessions', async () => {
     await start();
 
@@ -81,4 +94,17 @@ describe('Management APIs', function () {
       },
     );
   });
+
+  it('allows HEAD requests to /active', async () => {
+    await start();
+
+    await fetch('http://localhost:3000/active?token=6R0W53R135510', {
+      method: 'HEAD',
+    }).then(async (res) => {
+      expect(res.headers.get('content-type')).to.equal(
+        'text/plain; charset=UTF-8',
+      );
+      expect(res.status).to.equal(204);
+    });
+  });
 });

package/src/server.ts

@@ -18,7 +18,6 @@ import {
   WebSocketRoute,
   contentTypes,
   convertPathToURL,
-  createLogger,
   queryParamsToObject,
   readBody,
   shimLegacyRequests,
@@ -41,8 +40,7 @@ export class HTTPServer extends EventEmitter {
   protected server: http.Server = http.createServer();
   protected port: number;
   protected host?: string;
-  protected log = createLogger('server');
-  protected verbose = createLogger('server:verbose');
+  protected logger = new BlessLogger('server');
 
   constructor(
     protected config: Config,
@@ -56,7 +54,7 @@ export class HTTPServer extends EventEmitter {
     this.host = config.getHost();
     this.port = config.getPort();
 
-    this.log(
+    this.logger.info(
       `Server instantiated with host "${this.host}" on port "${
         this.port
       }" using token "${this.config.getToken()}"`,
@@ -64,7 +62,9 @@ export class HTTPServer extends EventEmitter {
   }
 
   protected onHTTPUnauthorized = (_req: Request, res: Response) => {
-    this.log(`HTTP request is not properly authorized, responding with 401`);
+    this.logger.error(
+      `HTTP request is not properly authorized, responding with 401`,
+    );
     this.metrics.addUnauthorized();
     return writeResponse(res, 401, 'Bad or missing authentication.');
   };
@@ -73,7 +73,7 @@ export class HTTPServer extends EventEmitter {
     _req: Request,
     socket: stream.Duplex,
   ) => {
-    this.log(
+    this.logger.error(
       `Websocket request is not properly authorized, responding with 401`,
     );
     this.metrics.addUnauthorized();
@@ -81,7 +81,7 @@ export class HTTPServer extends EventEmitter {
   };
 
   public async start(): Promise<void> {
-    this.log(`HTTP Server is starting`);
+    this.logger.info(`HTTP Server is starting`);
 
     this.server.on('request', this.handleRequest);
     this.server.on('upgrade', this.handleWebSocket);
@@ -98,7 +98,7 @@ export class HTTPServer extends EventEmitter {
         },
         undefined,
         () => {
-          this.log(listenMessage);
+          this.logger.info(listenMessage);
           r(undefined);
         },
       );
@@ -109,7 +109,7 @@ export class HTTPServer extends EventEmitter {
     request: http.IncomingMessage,
     res: http.ServerResponse,
   ) => {
-    this.verbose(
+    this.logger.trace(
       `Handling inbound HTTP request on "${request.method}: ${request.url}"`,
     );
 
@@ -131,6 +131,11 @@ export class HTTPServer extends EventEmitter {
       }
     }
 
+    if (req.method?.toLowerCase() === 'head') {
+      this.logger.debug(`Inbound HEAD request, setting to GET`);
+      req.method = 'GET';
+    }
+
     if (
       this.config.getAllowGetCalls() &&
       req.method === 'GET' &&
@@ -145,17 +150,17 @@ export class HTTPServer extends EventEmitter {
     const route = await this.router.getRouteForHTTPRequest(req);
 
     if (!route) {
-      this.log(
+      this.logger.error(
         `No matching HTTP route handler for "${req.method}: ${req.parsed.href}"`,
       );
       writeResponse(res, 404, 'Not Found');
       return Promise.resolve();
     }
 
-    this.verbose(`Found matching HTTP route handler "${route.path}"`);
+    this.logger.trace(`Found matching HTTP route handler "${route.path}"`);
 
     if (route?.auth) {
-      this.verbose(`Authorizing HTTP request to "${request.url}"`);
+      this.logger.trace(`Authorizing HTTP request to "${request.url}"`);
       const isPermitted = await this.token.isAuthorized(req, route);
 
       if (!isPermitted) {
@@ -179,7 +184,7 @@ export class HTTPServer extends EventEmitter {
     }
 
     if (route.querySchema) {
-      this.verbose(`Validating route query-params with QUERY schema`);
+      this.logger.trace(`Validating route query-params with QUERY schema`);
       try {
         const schema = Enjoi.schema(route.querySchema);
         const valid = schema.validate(req.queryParams, {
@@ -199,7 +204,7 @@ export class HTTPServer extends EventEmitter {
             )
             .join('\n');
 
-          this.log(
+          this.logger.error(
             `HTTP query-params contain errors sending 400:${errorDetails}`,
           );
 
@@ -212,7 +217,7 @@ export class HTTPServer extends EventEmitter {
           return Promise.resolve();
         }
       } catch (e) {
-        this.log(`Error parsing body schema`, e);
+        this.logger.error(`Error parsing body schema`, e);
         writeResponse(
           res,
           500,
@@ -224,7 +229,7 @@ export class HTTPServer extends EventEmitter {
     }
 
     if (route.bodySchema) {
-      this.verbose(`Validating route payload with BODY schema`);
+      this.logger.trace(`Validating route payload with BODY schema`);
       try {
         const schema = Enjoi.schema(route.bodySchema);
         const valid = schema.validate(body, { abortEarly: false });
@@ -242,7 +247,9 @@ export class HTTPServer extends EventEmitter {
             )
             .join('\n');
 
-          this.log(`HTTP body contain errors sending 400:${errorDetails}`);
+          this.logger.error(
+            `HTTP body contain errors sending 400:${errorDetails}`,
+          );
 
           writeResponse(
             res,
@@ -253,7 +260,7 @@ export class HTTPServer extends EventEmitter {
           return Promise.resolve();
         }
       } catch (e) {
-        this.log(`Error parsing body schema`, e);
+        this.logger.error(`Error parsing body schema`, e);
         writeResponse(
           res,
           500,
@@ -267,7 +274,7 @@ export class HTTPServer extends EventEmitter {
     return (route as HTTPRoute)
       .handler(req, res, new this.Logger(route.name, req))
       .then(() => {
-        this.verbose('HTTP connection complete');
+        this.logger.trace('HTTP connection complete');
       })
       .catch((e) => {
         if (e instanceof BadRequest) {
@@ -299,7 +306,7 @@ export class HTTPServer extends EventEmitter {
     socket: stream.Duplex,
     head: Buffer,
   ) => {
-    this.verbose(`Handling inbound WebSocket request on "${request.url}"`);
+    this.logger.trace(`Handling inbound WebSocket request on "${request.url}"`);
 
     const req = request as Request;
     const proceed = await this.hooks.before({ head, req, socket });
@@ -313,10 +320,14 @@ export class HTTPServer extends EventEmitter {
     const route = await this.router.getRouteForWebSocketRequest(req);
 
     if (route) {
-      this.verbose(`Found matching WebSocket route handler "${route.path}"`);
+      this.logger.trace(
+        `Found matching WebSocket route handler "${route.path}"`,
+      );
 
       if (route?.auth) {
-        this.verbose(`Authorizing WebSocket request to "${req.parsed.href}"`);
+        this.logger.trace(
+          `Authorizing WebSocket request to "${req.parsed.href}"`,
+        );
         const isPermitted = await this.token.isAuthorized(req, route);
 
         if (!isPermitted) {
@@ -325,7 +336,7 @@ export class HTTPServer extends EventEmitter {
       }
 
       if (route.querySchema) {
-        this.verbose(`Validating route query-params with QUERY schema`);
+        this.logger.trace(`Validating route query-params with QUERY schema`);
         try {
           const schema = Enjoi.schema(route.querySchema);
           const valid = schema.validate(req.queryParams, {
@@ -345,7 +356,7 @@ export class HTTPServer extends EventEmitter {
               )
               .join('\n');
 
-            this.log(
+            this.logger.error(
               `WebSocket query-params contain errors sending 400:${errorDetails}`,
             );
 
@@ -358,7 +369,7 @@ export class HTTPServer extends EventEmitter {
             return Promise.resolve();
           }
         } catch (e) {
-          this.log(`Error parsing query-params schema`, e);
+          this.logger.error(`Error parsing query-params schema`, e);
           writeResponse(
             socket,
             500,
@@ -372,7 +383,7 @@ export class HTTPServer extends EventEmitter {
       return (route as WebSocketRoute)
         .handler(req, socket, head, new this.Logger(route.name, req))
         .then(() => {
-          this.verbose('Websocket connection complete');
+          this.logger.trace('Websocket connection complete');
         })
         .catch((e) => {
           if (e instanceof BadRequest) {
@@ -391,7 +402,7 @@ export class HTTPServer extends EventEmitter {
             return writeResponse(socket, 429, e.message);
           }
 
-          this.log(
+          this.logger.error(
             `Error handling request at "${route.path}": ${e}\n${e.stack}`,
           );
 
@@ -399,18 +410,20 @@ export class HTTPServer extends EventEmitter {
         });
     }
 
-    this.log(`No matching WebSocket route handler for "${req.parsed.href}"`);
+    this.logger.error(
+      `No matching WebSocket route handler for "${req.parsed.href}"`,
+    );
     return writeResponse(socket, 404, 'Not Found');
   };
 
   public async shutdown(): Promise<void> {
-    this.log(`HTTP Server is shutting down`);
+    this.logger.info(`HTTP Server is shutting down`);
     await new Promise((r) => this.server.close(r));
     this.server && this.server.removeAllListeners();
 
     // @ts-ignore garbage collect this reference
     this.server = null;
-    this.log(`HTTP Server shutdown complete`);
+    this.logger.info(`HTTP Server shutdown complete`);
   }
 
   /**

package/src/shared/content.http.ts

@@ -78,9 +78,10 @@ export default class ChromiumContentPostRoute extends BrowserHTTPRoute {
   handler = async (
     req: Request,
     res: ServerResponse,
-    _logger: Logger,
+    logger: Logger,
     browser: BrowserInstance,
   ): Promise<void> => {
+    logger.info('Content API invoked with body:', req.body);
     const contentType =
       !req.headers.accept || req.headers.accept?.includes('*')
         ? contentTypes.html
@@ -166,6 +167,7 @@ export default class ChromiumContentPostRoute extends BrowserHTTPRoute {
           !!rejectRequestPattern.find((pattern) => req.url().match(pattern)) ||
           rejectResourceTypes.includes(req.resourceType())
         ) {
+          logger.debug(`Aborting request ${req.method()}: ${req.url()}`);
           return req.abort();
         }
         const interceptor = requestInterceptors.find((r) =>
@@ -233,6 +235,8 @@ export default class ChromiumContentPostRoute extends BrowserHTTPRoute {
 
     page.close().catch(noop);
 
+    logger.info('Content API request completed');
+
     return writeResponse(res, 200, markup, contentTypes.html);
   };
 }