@browserless.io/browserless 2.8.0 → 2.10.0

package/build/routes/chromium/http/scrape.post.body.json

@@ -441,14 +441,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@229203": {
+                "__@toStringTag@208093": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@229203",
+                "__@toStringTag@208093",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -483,13 +483,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@229203": {
+        "__@toStringTag@208093": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@229203",
+        "__@toStringTag@208093",
         "byteLength"
       ]
     },
@@ -499,18 +499,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@229304": {
+        "__@species@208194": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@229203": {
+        "__@toStringTag@208093": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@229304",
-        "__@toStringTag@229203",
+        "__@species@208194",
+        "__@toStringTag@208093",
         "byteLength"
       ]
     },

package/build/routes/chromium/http/screenshot.post.body.json

@@ -484,14 +484,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@240378": {
+                "__@toStringTag@241336": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@240378",
+                "__@toStringTag@241336",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -526,13 +526,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@240378": {
+        "__@toStringTag@241336": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@240378",
+        "__@toStringTag@241336",
         "byteLength"
       ]
     },
@@ -542,18 +542,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@240479": {
+        "__@species@241437": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@240378": {
+        "__@toStringTag@241336": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@240479",
-        "__@toStringTag@240378",
+        "__@species@241437",
+        "__@toStringTag@241336",
         "byteLength"
       ]
     },

package/build/routes/management/http/pressure.get.d.ts

@@ -0,0 +1,63 @@
+/// <reference types="node" />
+import { APITags, HTTPManagementRoutes, HTTPRoute, Methods, Request, contentTypes } from '@browserless.io/browserless';
+import { ServerResponse } from 'http';
+export type ResponseSchema = {
+    /**
+     * An integer representing the percentage of CPU being used. For instance 92 means 92%
+     */
+    cpu: number | null;
+    /**
+     * A number of milliseconds since epoch, or "Date.now()" equivalent.
+     */
+    date: number;
+    /**
+     * Whether or not a session can be connected and immediately ran on a health instance.
+     */
+    isAvailable: boolean;
+    /**
+     * The maximum amount of browsers that can be ran at a single time.
+     */
+    maxConcurrent: number;
+    /**
+     * The maximum amount of queued connections allowed at a single time.
+     */
+    maxQueued: number;
+    /**
+     * An integer representing the percentage of Memory being used. For instance 95 means 95%
+     */
+    memory: number | null;
+    /**
+     * A human-readable message as the overall status of the instance.
+     */
+    message: string;
+    /**
+     * The current number of connect or API calls pending to run.
+     */
+    queued: number;
+    /**
+     * A simple single-word reason as to why an instance may or may not be available.
+     */
+    reason: 'full' | 'cpu' | 'memory' | '';
+    /**
+     * The number of recent connections that were rejected due to the queue and concurrency
+     * limits having been filled.
+     */
+    recentlyRejected: number;
+    /**
+     * The current number of running connections or API calls.
+     */
+    running: number;
+};
+export default class PressureGetRoute extends HTTPRoute {
+    name: string;
+    accepts: contentTypes[];
+    auth: boolean;
+    browser: null;
+    concurrency: boolean;
+    contentTypes: contentTypes[];
+    description: string;
+    method: Methods;
+    path: HTTPManagementRoutes;
+    tags: APITags[];
+    handler: (_req: Request, res: ServerResponse) => Promise<void>;
+}

package/build/routes/management/http/pressure.get.js

@@ -0,0 +1,56 @@
+import { APITags, BrowserlessRoutes, HTTPManagementRoutes, HTTPRoute, Methods, contentTypes, jsonResponse, } from '@browserless.io/browserless';
+export default class PressureGetRoute extends HTTPRoute {
+    name = BrowserlessRoutes.PressureGetRoute;
+    accepts = [contentTypes.any];
+    auth = true;
+    browser = null;
+    concurrency = false;
+    contentTypes = [contentTypes.json];
+    description = `Returns a JSON body of stats related to the pressure being created on the instance.`;
+    method = Methods.get;
+    path = HTTPManagementRoutes.pressure;
+    tags = [APITags.management];
+    handler = async (_req, res) => {
+        const monitoring = this.monitoring();
+        const config = this.config();
+        const limiter = this.limiter();
+        const metrics = this.metrics();
+        const { cpuInt: cpu, memoryInt: memory, cpuOverloaded, memoryOverloaded, } = await monitoring.overloaded();
+        const date = Date.now();
+        const hasCapacity = limiter.hasCapacity;
+        const queued = limiter.waiting;
+        const isAvailable = hasCapacity && !cpuOverloaded && !memoryOverloaded;
+        const running = limiter.executing;
+        const recentlyRejected = metrics.get().rejected;
+        const maxConcurrent = config.getConcurrent();
+        const maxQueued = config.getQueued();
+        const reason = !hasCapacity
+            ? 'full'
+            : cpuOverloaded
+                ? 'cpu'
+                : memoryOverloaded
+                    ? 'memory'
+                    : '';
+        const message = !hasCapacity
+            ? 'Concurrency and queue are full'
+            : cpuOverloaded
+                ? 'CPU is over the configured maximum for cpu percent'
+                : memoryOverloaded
+                    ? 'Memory is over the configured maximum for memory percent'
+                    : '';
+        const response = {
+            cpu,
+            date,
+            isAvailable,
+            maxConcurrent,
+            maxQueued,
+            memory,
+            message,
+            queued,
+            reason,
+            recentlyRejected,
+            running,
+        };
+        return jsonResponse(res, 200, response);
+    };
+}

package/build/routes/management/http/pressure.get.response.json

@@ -0,0 +1,76 @@
+{
+  "type": "object",
+  "properties": {
+    "cpu": {
+      "description": "An integer representing the percentage of CPU being used. For instance 92 means 92%",
+      "type": [
+        "null",
+        "number"
+      ]
+    },
+    "date": {
+      "description": "A number of milliseconds since epoch, or \"Date.now()\" equivalent.",
+      "type": "number"
+    },
+    "isAvailable": {
+      "description": "Whether or not a session can be connected and immediately ran on a health instance.",
+      "type": "boolean"
+    },
+    "maxConcurrent": {
+      "description": "The maximum amount of browsers that can be ran at a single time.",
+      "type": "number"
+    },
+    "maxQueued": {
+      "description": "The maximum amount of queued connections allowed at a single time.",
+      "type": "number"
+    },
+    "memory": {
+      "description": "An integer representing the percentage of Memory being used. For instance 95 means 95%",
+      "type": [
+        "null",
+        "number"
+      ]
+    },
+    "message": {
+      "description": "A human-readable message as the overall status of the instance.",
+      "type": "string"
+    },
+    "queued": {
+      "description": "The current number of connect or API calls pending to run.",
+      "type": "number"
+    },
+    "reason": {
+      "description": "A simple single-word reason as to why an instance may or may not be available.",
+      "enum": [
+        "",
+        "cpu",
+        "full",
+        "memory"
+      ],
+      "type": "string"
+    },
+    "recentlyRejected": {
+      "description": "The number of recent connections that were rejected due to the queue and concurrency\nlimits having been filled.",
+      "type": "number"
+    },
+    "running": {
+      "description": "The current number of running connections or API calls.",
+      "type": "number"
+    }
+  },
+  "additionalProperties": false,
+  "required": [
+    "cpu",
+    "date",
+    "isAvailable",
+    "maxConcurrent",
+    "maxQueued",
+    "memory",
+    "message",
+    "queued",
+    "reason",
+    "recentlyRejected",
+    "running"
+  ],
+  "$schema": "http://json-schema.org/draft-07/schema#"
+}

package/build/routes/management/http/static.get.js

@@ -4,7 +4,7 @@ import path from 'path';
 const pathMap = new Map();
 const streamFile = (logger, res, file, contentType) => new Promise((resolve, reject) => {
     if (contentType) {
-        logger.log(`Setting content-type ${contentType}`);
+        logger.debug(`Setting content-type ${contentType}`);
         res.setHeader('Content-Type', contentType);
     }
     return createReadStream(file)
@@ -49,10 +49,10 @@ export default class StaticGetRoute extends HTTPRoute {
             throw new NotFound(`No route or file found for resource ${req.method}: ${pathname}`);
         }
         if (foundFilePaths.length > 1) {
-            logger.log(`Multiple files found for request to "${pathname}". Only the first file is served, so please name your files uniquely.`);
+            logger.warn(`Multiple files found for request to "${pathname}". Only the first file is served, so please name your files uniquely.`);
         }
         const [foundFilePath] = foundFilePaths;
-        logger.verbose(`Found new file "${foundFilePath}", caching path and serving`);
+        logger.info(`Found new file "${foundFilePath}", caching path and serving`);
         const contentType = mimeTypes.get(path.extname(foundFilePath));
         if (contentType) {
             res.setHeader('Content-Type', contentType);

package/build/routes/management/tests/management.spec.js

@@ -31,6 +31,13 @@ describe('Management APIs', function () {
             expect(res.status).to.equal(200);
         });
     });
+    it('allows requests to /pressure', async () => {
+        await start();
+        await fetch('http://localhost:3000/pressure?token=6R0W53R135510').then(async (res) => {
+            expect(res.headers.get('content-type')).to.equal('application/json; charset=UTF-8');
+            expect(res.status).to.equal(200);
+        });
+    });
     it('allows requests to /sessions', async () => {
         await start();
         await fetch('http://localhost:3000/sessions?token=6R0W53R135510').then(async (res) => {
@@ -45,4 +52,13 @@ describe('Management APIs', function () {
             expect(res.status).to.equal(204);
         });
     });
+    it('allows HEAD requests to /active', async () => {
+        await start();
+        await fetch('http://localhost:3000/active?token=6R0W53R135510', {
+            method: 'HEAD',
+        }).then(async (res) => {
+            expect(res.headers.get('content-type')).to.equal('text/plain; charset=UTF-8');
+            expect(res.status).to.equal(204);
+        });
+    });
 });

package/build/server.d.ts

@@ -1,5 +1,4 @@
 /// <reference types="node" />
-/// <reference types="debug" />
 /// <reference types="node" />
 /// <reference types="node" />
 /// <reference types="node" />
@@ -24,8 +23,7 @@ export declare class HTTPServer extends EventEmitter {
     protected server: http.Server;
     protected port: number;
     protected host?: string;
-    protected log: import("debug").Debugger;
-    protected verbose: import("debug").Debugger;
+    protected logger: BlessLogger;
     constructor(config: Config, metrics: Metrics, token: Token, router: Router, hooks: Hooks, Logger: typeof BlessLogger);
     protected onHTTPUnauthorized: (_req: Request, res: Response) => void;
     protected onWebsocketUnauthorized: (_req: Request, socket: stream.Duplex) => void;

package/build/server.js

@@ -1,5 +1,5 @@
 import * as http from 'http';
-import { BadRequest, NotFound, Timeout, TooManyRequests, Unauthorized, contentTypes, convertPathToURL, createLogger, queryParamsToObject, readBody, shimLegacyRequests, writeResponse, } from '@browserless.io/browserless';
+import { BadRequest, Logger as BlessLogger, NotFound, Timeout, TooManyRequests, Unauthorized, contentTypes, convertPathToURL, queryParamsToObject, readBody, shimLegacyRequests, writeResponse, } from '@browserless.io/browserless';
 import { EventEmitter } from 'events';
 // @ts-ignore
 import Enjoi from 'enjoi';
@@ -13,8 +13,7 @@ export class HTTPServer extends EventEmitter {
     server = http.createServer();
     port;
     host;
-    log = createLogger('server');
-    verbose = createLogger('server:verbose');
+    logger = new BlessLogger('server');
     constructor(config, metrics, token, router, hooks, Logger) {
         super();
         this.config = config;
@@ -25,20 +24,20 @@ export class HTTPServer extends EventEmitter {
         this.Logger = Logger;
         this.host = config.getHost();
         this.port = config.getPort();
-        this.log(`Server instantiated with host "${this.host}" on port "${this.port}" using token "${this.config.getToken()}"`);
+        this.logger.info(`Server instantiated with host "${this.host}" on port "${this.port}" using token "${this.config.getToken()}"`);
     }
     onHTTPUnauthorized = (_req, res) => {
-        this.log(`HTTP request is not properly authorized, responding with 401`);
+        this.logger.error(`HTTP request is not properly authorized, responding with 401`);
         this.metrics.addUnauthorized();
         return writeResponse(res, 401, 'Bad or missing authentication.');
     };
     onWebsocketUnauthorized = (_req, socket) => {
-        this.log(`Websocket request is not properly authorized, responding with 401`);
+        this.logger.error(`Websocket request is not properly authorized, responding with 401`);
         this.metrics.addUnauthorized();
         return writeResponse(socket, 401, 'Bad or missing authentication.');
     };
     async start() {
-        this.log(`HTTP Server is starting`);
+        this.logger.info(`HTTP Server is starting`);
         this.server.on('request', this.handleRequest);
         this.server.on('upgrade', this.handleWebSocket);
         const listenMessage = [
@@ -50,13 +49,13 @@ export class HTTPServer extends EventEmitter {
                 host: this.host,
                 port: this.port,
             }, undefined, () => {
-                this.log(listenMessage);
+                this.logger.info(listenMessage);
                 r(undefined);
             });
         });
     }
     handleRequest = async (request, res) => {
-        this.verbose(`Handling inbound HTTP request on "${request.method}: ${request.url}"`);
+        this.logger.trace(`Handling inbound HTTP request on "${request.method}: ${request.url}"`);
         const req = request;
         const proceed = await this.hooks.before({ req, res });
         req.parsed = convertPathToURL(request.url || '', this.config);
@@ -70,6 +69,10 @@ export class HTTPServer extends EventEmitter {
                 return res.end();
             }
         }
+        if (req.method?.toLowerCase() === 'head') {
+            this.logger.debug(`Inbound HEAD request, setting to GET`);
+            req.method = 'GET';
+        }
         if (this.config.getAllowGetCalls() &&
             req.method === 'GET' &&
             req.parsed.searchParams.has('body')) {
@@ -80,13 +83,13 @@ export class HTTPServer extends EventEmitter {
         }
         const route = await this.router.getRouteForHTTPRequest(req);
         if (!route) {
-            this.log(`No matching HTTP route handler for "${req.method}: ${req.parsed.href}"`);
+            this.logger.error(`No matching HTTP route handler for "${req.method}: ${req.parsed.href}"`);
             writeResponse(res, 404, 'Not Found');
             return Promise.resolve();
         }
-        this.verbose(`Found matching HTTP route handler "${route.path}"`);
+        this.logger.trace(`Found matching HTTP route handler "${route.path}"`);
         if (route?.auth) {
-            this.verbose(`Authorizing HTTP request to "${request.url}"`);
+            this.logger.trace(`Authorizing HTTP request to "${request.url}"`);
             const isPermitted = await this.token.isAuthorized(req, route);
             if (!isPermitted) {
                 return this.onHTTPUnauthorized(req, res);
@@ -104,7 +107,7 @@ export class HTTPServer extends EventEmitter {
             return Promise.resolve();
         }
         if (route.querySchema) {
-            this.verbose(`Validating route query-params with QUERY schema`);
+            this.logger.trace(`Validating route query-params with QUERY schema`);
             try {
                 const schema = Enjoi.schema(route.querySchema);
                 const valid = schema.validate(req.queryParams, {
@@ -114,19 +117,19 @@ export class HTTPServer extends EventEmitter {
                     const errorDetails = valid.error.details
                         .map(({ message, context, }) => context?.message || message)
                         .join('\n');
-                    this.log(`HTTP query-params contain errors sending 400:${errorDetails}`);
+                    this.logger.error(`HTTP query-params contain errors sending 400:${errorDetails}`);
                     writeResponse(res, 400, `Query-parameter validation failed: ${errorDetails}`, contentTypes.text);
                     return Promise.resolve();
                 }
             }
             catch (e) {
-                this.log(`Error parsing body schema`, e);
+                this.logger.error(`Error parsing body schema`, e);
                 writeResponse(res, 500, 'There was an error handling your request', contentTypes.text);
                 return Promise.resolve();
             }
         }
         if (route.bodySchema) {
-            this.verbose(`Validating route payload with BODY schema`);
+            this.logger.trace(`Validating route payload with BODY schema`);
             try {
                 const schema = Enjoi.schema(route.bodySchema);
                 const valid = schema.validate(body, { abortEarly: false });
@@ -134,13 +137,13 @@ export class HTTPServer extends EventEmitter {
                     const errorDetails = valid.error.details
                         .map(({ message, context, }) => context?.message || message)
                         .join('\n');
-                    this.log(`HTTP body contain errors sending 400:${errorDetails}`);
+                    this.logger.error(`HTTP body contain errors sending 400:${errorDetails}`);
                     writeResponse(res, 400, `POST Body validation failed: ${errorDetails}`, contentTypes.text);
                     return Promise.resolve();
                 }
             }
             catch (e) {
-                this.log(`Error parsing body schema`, e);
+                this.logger.error(`Error parsing body schema`, e);
                 writeResponse(res, 500, 'There was an error handling your request', contentTypes.text);
                 return Promise.resolve();
             }
@@ -148,7 +151,7 @@ export class HTTPServer extends EventEmitter {
         return route
             .handler(req, res, new this.Logger(route.name, req))
             .then(() => {
-            this.verbose('HTTP connection complete');
+            this.logger.trace('HTTP connection complete');
         })
             .catch((e) => {
             if (e instanceof BadRequest) {
@@ -170,7 +173,7 @@ export class HTTPServer extends EventEmitter {
         });
     };
     handleWebSocket = async (request, socket, head) => {
-        this.verbose(`Handling inbound WebSocket request on "${request.url}"`);
+        this.logger.trace(`Handling inbound WebSocket request on "${request.url}"`);
         const req = request;
         const proceed = await this.hooks.before({ head, req, socket });
         req.parsed = convertPathToURL(request.url || '', this.config);
@@ -180,16 +183,16 @@ export class HTTPServer extends EventEmitter {
         req.queryParams = queryParamsToObject(req.parsed.searchParams);
         const route = await this.router.getRouteForWebSocketRequest(req);
         if (route) {
-            this.verbose(`Found matching WebSocket route handler "${route.path}"`);
+            this.logger.trace(`Found matching WebSocket route handler "${route.path}"`);
             if (route?.auth) {
-                this.verbose(`Authorizing WebSocket request to "${req.parsed.href}"`);
+                this.logger.trace(`Authorizing WebSocket request to "${req.parsed.href}"`);
                 const isPermitted = await this.token.isAuthorized(req, route);
                 if (!isPermitted) {
                     return this.onWebsocketUnauthorized(req, socket);
                 }
             }
             if (route.querySchema) {
-                this.verbose(`Validating route query-params with QUERY schema`);
+                this.logger.trace(`Validating route query-params with QUERY schema`);
                 try {
                     const schema = Enjoi.schema(route.querySchema);
                     const valid = schema.validate(req.queryParams, {
@@ -199,13 +202,13 @@ export class HTTPServer extends EventEmitter {
                         const errorDetails = valid.error.details
                             .map(({ message, context, }) => context?.message || message)
                             .join('\n');
-                        this.log(`WebSocket query-params contain errors sending 400:${errorDetails}`);
+                        this.logger.error(`WebSocket query-params contain errors sending 400:${errorDetails}`);
                         writeResponse(socket, 400, `Query-parameter validation failed: ${errorDetails}`, contentTypes.text);
                         return Promise.resolve();
                     }
                 }
                 catch (e) {
-                    this.log(`Error parsing query-params schema`, e);
+                    this.logger.error(`Error parsing query-params schema`, e);
                     writeResponse(socket, 500, 'There was an error handling your request', contentTypes.text);
                     return Promise.resolve();
                 }
@@ -213,7 +216,7 @@ export class HTTPServer extends EventEmitter {
             return route
                 .handler(req, socket, head, new this.Logger(route.name, req))
                 .then(() => {
-                this.verbose('Websocket connection complete');
+                this.logger.trace('Websocket connection complete');
             })
                 .catch((e) => {
                 if (e instanceof BadRequest) {
@@ -228,20 +231,20 @@ export class HTTPServer extends EventEmitter {
                 if (e instanceof TooManyRequests) {
                     return writeResponse(socket, 429, e.message);
                 }
-                this.log(`Error handling request at "${route.path}": ${e}\n${e.stack}`);
+                this.logger.error(`Error handling request at "${route.path}": ${e}\n${e.stack}`);
                 return writeResponse(socket, 500, e.message);
             });
         }
-        this.log(`No matching WebSocket route handler for "${req.parsed.href}"`);
+        this.logger.error(`No matching WebSocket route handler for "${req.parsed.href}"`);
         return writeResponse(socket, 404, 'Not Found');
     };
     async shutdown() {
-        this.log(`HTTP Server is shutting down`);
+        this.logger.info(`HTTP Server is shutting down`);
         await new Promise((r) => this.server.close(r));
         this.server && this.server.removeAllListeners();
         // @ts-ignore garbage collect this reference
         this.server = null;
-        this.log(`HTTP Server shutdown complete`);
+        this.logger.info(`HTTP Server shutdown complete`);
     }
     /**
      * Left blank for downstream SDK modules to optionally implement.

package/build/shared/content.http.d.ts

@@ -43,5 +43,5 @@ export default class ChromiumContentPostRoute extends BrowserHTTPRoute {
     method: Methods;
     path: HTTPRoutes[];
     tags: APITags[];
-    handler: (req: Request, res: ServerResponse, _logger: Logger, browser: BrowserInstance) => Promise<void>;
+    handler: (req: Request, res: ServerResponse, logger: Logger, browser: BrowserInstance) => Promise<void>;
 }

package/build/shared/content.http.js

@@ -10,7 +10,8 @@ export default class ChromiumContentPostRoute extends BrowserHTTPRoute {
     method = Methods.post;
     path = [HTTPRoutes.content, HTTPRoutes.chromiumContent];
     tags = [APITags.browserAPI];
-    handler = async (req, res, _logger, browser) => {
+    handler = async (req, res, logger, browser) => {
+        logger.info('Content API invoked with body:', req.body);
         const contentType = !req.headers.accept || req.headers.accept?.includes('*')
             ? contentTypes.html
             : req.headers.accept;
@@ -53,6 +54,7 @@ export default class ChromiumContentPostRoute extends BrowserHTTPRoute {
             page.on('request', (req) => {
                 if (!!rejectRequestPattern.find((pattern) => req.url().match(pattern)) ||
                     rejectResourceTypes.includes(req.resourceType())) {
+                    logger.debug(`Aborting request ${req.method()}: ${req.url()}`);
                     return req.abort();
                 }
                 const interceptor = requestInterceptors.find((r) => req.url().match(r.pattern));
@@ -102,6 +104,7 @@ export default class ChromiumContentPostRoute extends BrowserHTTPRoute {
         }
         const markup = await page.content();
         page.close().catch(noop);
+        logger.info('Content API request completed');
         return writeResponse(res, 200, markup, contentTypes.html);
     };
 }