@browserless.io/browserless 2.4.0-beta-3 → 2.5.0-beta-1

package/build/routes/chrome/http/scrape.post.body.json

@@ -441,14 +441,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@108664": {
+                "__@toStringTag@130859": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@108664",
+                "__@toStringTag@130859",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -483,13 +483,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@108664": {
+        "__@toStringTag@130859": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@108664",
+        "__@toStringTag@130859",
         "byteLength"
       ]
     },
@@ -499,18 +499,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@108765": {
+        "__@species@130960": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@108664": {
+        "__@toStringTag@130859": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@108765",
-        "__@toStringTag@108664",
+        "__@species@130960",
+        "__@toStringTag@130859",
         "byteLength"
       ]
     },

package/build/routes/chrome/http/screenshot.post.body.json

@@ -484,14 +484,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@119813": {
+                "__@toStringTag@108705": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@119813",
+                "__@toStringTag@108705",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -526,13 +526,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@119813": {
+        "__@toStringTag@108705": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@119813",
+        "__@toStringTag@108705",
         "byteLength"
       ]
     },
@@ -542,18 +542,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@119914": {
+        "__@species@108806": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@119813": {
+        "__@toStringTag@108705": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@119914",
-        "__@toStringTag@119813",
+        "__@species@108806",
+        "__@toStringTag@108705",
         "byteLength"
       ]
     },

package/build/routes/chromium/http/content.post.body.json

@@ -394,14 +394,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@141676": {
+                "__@toStringTag@119767": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@141676",
+                "__@toStringTag@119767",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -436,13 +436,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@141676": {
+        "__@toStringTag@119767": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@141676",
+        "__@toStringTag@119767",
         "byteLength"
       ]
     },
@@ -452,18 +452,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@141777": {
+        "__@species@119868": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@141676": {
+        "__@toStringTag@119767": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@141777",
-        "__@toStringTag@141676",
+        "__@species@119868",
+        "__@toStringTag@119767",
         "byteLength"
       ]
     },

package/build/routes/chromium/http/pdf.post.body.json

@@ -20,9 +20,6 @@
       "description": "When bestAttempt is set to true, browserless attempt to proceed\nwhen \"awaited\" events fail or timeout. This includes things like\ngoto, waitForSelector, and more.",
       "type": "boolean"
     },
-    "blockModals": {
-      "type": "boolean"
-    },
     "cookies": {
       "type": "array",
       "items": {
@@ -538,14 +535,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@206722": {
+                "__@toStringTag@206779": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@206722",
+                "__@toStringTag@206779",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -580,13 +577,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@206722": {
+        "__@toStringTag@206779": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@206722",
+        "__@toStringTag@206779",
         "byteLength"
       ]
     },
@@ -596,18 +593,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@206823": {
+        "__@species@206880": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@206722": {
+        "__@toStringTag@206779": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@206823",
-        "__@toStringTag@206722",
+        "__@species@206880",
+        "__@toStringTag@206779",
         "byteLength"
       ]
     },

package/build/routes/chromium/http/scrape.post.body.json

@@ -441,14 +441,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@217791": {
+                "__@toStringTag@228657": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@217791",
+                "__@toStringTag@228657",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -483,13 +483,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@217791": {
+        "__@toStringTag@228657": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@217791",
+        "__@toStringTag@228657",
         "byteLength"
       ]
     },
@@ -499,18 +499,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@217892": {
+        "__@species@228758": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@217791": {
+        "__@toStringTag@228657": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@217892",
-        "__@toStringTag@217791",
+        "__@species@228758",
+        "__@toStringTag@228657",
         "byteLength"
       ]
     },

package/build/routes/chromium/http/screenshot.post.body.json

@@ -484,14 +484,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@239740": {
+                "__@toStringTag@239806": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@239740",
+                "__@toStringTag@239806",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -526,13 +526,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@239740": {
+        "__@toStringTag@239806": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@239740",
+        "__@toStringTag@239806",
         "byteLength"
       ]
     },
@@ -542,18 +542,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@239841": {
+        "__@species@239907": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@239740": {
+        "__@toStringTag@239806": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@239841",
-        "__@toStringTag@239740",
+        "__@species@239907",
+        "__@toStringTag@239806",
         "byteLength"
       ]
     },

package/build/routes/management/http/metrics-total.get.js

@@ -14,7 +14,7 @@ export default class MetricsTotalGetRoute extends HTTPRoute {
     handler = async (_req, res) => {
         const fileSystem = this.fileSystem();
         const config = this.config();
-        const metrics = (await fileSystem.read(config.getMetricsJSONPath())).map((m) => JSON.parse(m));
+        const metrics = (await fileSystem.read(config.getMetricsJSONPath(), false)).map((m) => JSON.parse(m));
         const availableMetrics = metrics.length;
         const totals = metrics.reduce((accum, metric) => ({
             error: accum.error + metric.error,

package/build/routes/management/http/metrics.get.js

@@ -6,14 +6,14 @@ export default class MetricsGetRoute extends HTTPRoute {
     browser = null;
     concurrency = false;
     contentTypes = [contentTypes.json];
-    description = `Gets total metric details from the time the server started.`;
+    description = `Returns a list of metric details as far back as possible.`;
     method = Methods.get;
     path = HTTPManagementRoutes.metrics;
     tags = [APITags.management];
     handler = async (_req, res) => {
         const fileSystem = this.fileSystem();
         const config = this.config();
-        const stats = await fileSystem.read(config.getMetricsJSONPath());
+        const stats = await fileSystem.read(config.getMetricsJSONPath(), false);
         const response = `[${stats.join(',')}]`;
         return writeResponse(res, 200, response, contentTypes.json);
     };

package/build/sdk-utils.d.ts

@@ -0,0 +1,13 @@
+export declare const getArgSwitches: () => {
+    [key: string]: string | true;
+};
+export declare const getSourceFiles: (cwd: string) => Promise<{
+    files: string[];
+    httpRoutes: string[];
+    webSocketRoutes: string[];
+}>;
+export declare const camelCase: (str: string) => string;
+export declare const prompt: (question: string) => Promise<unknown>;
+export declare const installDependencies: (workingDirectory: string) => Promise<void>;
+export declare const buildDockerImage: (cmd: string, projectDir: string) => Promise<void>;
+export declare const buildTypeScript: (buildDir: string, projectDir: string) => Promise<void>;

package/build/sdk-utils.js

@@ -0,0 +1,94 @@
+import { createInterface } from 'readline';
+import debug from 'debug';
+import fs from 'fs/promises';
+import path from 'path';
+import { spawn } from 'child_process';
+export const getArgSwitches = () => {
+    return process.argv.reduce((accum, arg, idx) => {
+        if (!arg.startsWith('--')) {
+            return accum;
+        }
+        if (arg.includes('=')) {
+            const [parameter, value] = arg.split('=');
+            accum[parameter.replace(/-/gi, '')] = value || true;
+            return accum;
+        }
+        const nextSwitchOrParameter = process.argv[idx + 1];
+        const param = arg.replace(/-/gi, '');
+        if (typeof nextSwitchOrParameter === 'undefined' ||
+            nextSwitchOrParameter?.startsWith('--')) {
+            accum[param] = true;
+            return accum;
+        }
+        accum[param] = nextSwitchOrParameter;
+        return accum;
+    }, {});
+};
+export const getSourceFiles = async (cwd) => {
+    const buildDir = path.join(cwd, 'build');
+    const files = await fs.readdir(buildDir, { recursive: true });
+    const [httpRoutes, webSocketRoutes] = files.reduce(([httpRoutes, webSocketRoutes], file) => {
+        const parsed = path.parse(file);
+        if (parsed.name.endsWith('http')) {
+            httpRoutes.push(path.join(buildDir, file));
+        }
+        if (parsed.name.endsWith('ws')) {
+            webSocketRoutes.push(path.join(buildDir, file));
+        }
+        return [httpRoutes, webSocketRoutes];
+    }, [[], []]);
+    return {
+        files,
+        httpRoutes,
+        webSocketRoutes,
+    };
+};
+export const camelCase = (str) => str.replace(/-([a-z])/g, (_, w) => w.toUpperCase());
+export const prompt = async (question) => {
+    const promptLog = debug('browserless.io:prompt');
+    const rl = createInterface({
+        input: process.stdin,
+        output: process.stdout,
+    });
+    return new Promise((resolve) => {
+        promptLog(question);
+        rl.question('  > ', (a) => {
+            rl.close();
+            resolve(a.trim());
+        });
+    });
+};
+export const installDependencies = async (workingDirectory) => new Promise((resolve, reject) => {
+    spawn('npm', ['i'], {
+        cwd: workingDirectory,
+        stdio: 'inherit',
+    }).once('close', (code) => {
+        if (code === 0) {
+            return resolve();
+        }
+        return reject(`Error when installing dependencies, see output for more details`);
+    });
+});
+export const buildDockerImage = async (cmd, projectDir) => new Promise((resolve, reject) => {
+    const [docker, ...args] = cmd.split(' ');
+    spawn(docker, args, {
+        cwd: projectDir,
+        stdio: 'inherit',
+    }).once('close', (code) => {
+        if (code === 0) {
+            return resolve();
+        }
+        return reject(`Error when building Docker image, see output for more details`);
+    });
+});
+export const buildTypeScript = async (buildDir, projectDir) => new Promise((resolve, reject) => {
+    spawn('npx', ['tsc', '--outDir', buildDir], {
+        cwd: projectDir,
+        stdio: 'inherit',
+    }).once('close', (code) => {
+        if (code === 0) {
+            return resolve();
+        }
+        return reject(`Error in building TypeScript, see output for more details`);
+    });
+});

package/build/server.d.ts

@@ -5,7 +5,7 @@
 /// <reference types="node" />
 import * as http from 'http';
 import * as stream from 'stream';
-import { Config, Metrics, Request, Response, Router, Token } from '@browserless.io/browserless';
+import { Config, Hooks, Metrics, Request, Response, Router, Token } from '@browserless.io/browserless';
 import { EventEmitter } from 'events';
 export interface HTTPServerOptions {
     concurrent: number;
@@ -19,12 +19,13 @@ export declare class HTTPServer extends EventEmitter {
     protected metrics: Metrics;
     protected token: Token;
     protected router: Router;
+    protected hooks: Hooks;
     protected server: http.Server;
     protected port: number;
     protected host?: string;
     protected log: import("debug").Debugger;
     protected verbose: import("debug").Debugger;
-    constructor(config: Config, metrics: Metrics, token: Token, router: Router);
+    constructor(config: Config, metrics: Metrics, token: Token, router: Router, hooks: Hooks);
     protected onHTTPUnauthorized: (_req: Request, res: Response) => void;
     protected onWebsocketUnauthorized: (_req: Request, socket: stream.Duplex) => void;
     start(): Promise<void>;

package/build/server.js

@@ -1,5 +1,5 @@
 import * as http from 'http';
-import { BadRequest, NotFound, Timeout, TooManyRequests, Unauthorized, beforeRequest, contentTypes, convertPathToURL, createLogger, queryParamsToObject, readBody, shimLegacyRequests, writeResponse, } from '@browserless.io/browserless';
+import { BadRequest, NotFound, Timeout, TooManyRequests, Unauthorized, contentTypes, convertPathToURL, createLogger, queryParamsToObject, readBody, shimLegacyRequests, writeResponse, } from '@browserless.io/browserless';
 import { EventEmitter } from 'events';
 // @ts-ignore
 import Enjoi from 'enjoi';
@@ -8,17 +8,19 @@ export class HTTPServer extends EventEmitter {
     metrics;
     token;
     router;
+    hooks;
     server = http.createServer();
     port;
     host;
     log = createLogger('server');
     verbose = createLogger('server:verbose');
-    constructor(config, metrics, token, router) {
+    constructor(config, metrics, token, router, hooks) {
         super();
         this.config = config;
         this.metrics = metrics;
         this.token = token;
         this.router = router;
+        this.hooks = hooks;
         this.host = config.getHost();
         this.port = config.getPort();
         this.log(`Server instantiated with host "${this.host}" on port "${this.port}" using token "${this.config.getToken()}"`);
@@ -54,7 +56,7 @@ export class HTTPServer extends EventEmitter {
     handleRequest = async (request, res) => {
         this.verbose(`Handling inbound HTTP request on "${request.method}: ${request.url}"`);
         const req = request;
-        const proceed = await beforeRequest({ req, res });
+        const proceed = await this.hooks.before({ req, res });
         req.parsed = convertPathToURL(request.url || '', this.config);
         shimLegacyRequests(req.parsed);
         if (!proceed)
@@ -168,7 +170,7 @@ export class HTTPServer extends EventEmitter {
     handleWebSocket = async (request, socket, head) => {
         this.verbose(`Handling inbound WebSocket request on "${request.url}"`);
         const req = request;
-        const proceed = await beforeRequest({ head, req, socket });
+        const proceed = await this.hooks.before({ head, req, socket });
         req.parsed = convertPathToURL(request.url || '', this.config);
         shimLegacyRequests(req.parsed);
         if (!proceed)

package/build/shared/content.http.js

@@ -62,6 +62,7 @@ export default class ChromiumContentPostRoute extends BrowserHTTPRoute {
                 return req.continue();
             });
         }
+        const gotoResponse = await gotoCall(content, gotoOptions).catch(bestAttemptCatch(bestAttempt));
         if (addStyleTag.length) {
             for (const tag in addStyleTag) {
                 await page.addStyleTag(addStyleTag[tag]);
@@ -72,7 +73,6 @@ export default class ChromiumContentPostRoute extends BrowserHTTPRoute {
                 await page.addScriptTag(addScriptTag[tag]);
             }
         }
-        const gotoResponse = await gotoCall(content, gotoOptions).catch(bestAttemptCatch(bestAttempt));
         if (waitForTimeout) {
             await sleep(waitForTimeout).catch(bestAttemptCatch(bestAttempt));
         }

package/build/shared/pdf.http.d.ts

@@ -7,7 +7,6 @@ export interface BodySchema {
     addStyleTag?: Array<Parameters<Page['addStyleTag']>[0]>;
     authenticate?: Parameters<Page['authenticate']>[0];
     bestAttempt?: bestAttempt;
-    blockModals?: boolean;
     cookies?: Array<Parameters<Page['setCookie']>[0]>;
     emulateMediaType?: Parameters<Page['emulateMediaType']>[0];
     gotoOptions?: Parameters<Page['goto']>[1];

package/build/shared/pdf.http.js

@@ -69,6 +69,7 @@ export default class ChromiumPDFPostRoute extends BrowserHTTPRoute {
                 return req.continue();
             });
         }
+        const gotoResponse = await gotoCall(content, gotoOptions).catch(bestAttemptCatch(bestAttempt));
         if (addStyleTag.length) {
             for (const tag in addStyleTag) {
                 await page.addStyleTag(addStyleTag[tag]);
@@ -79,7 +80,6 @@ export default class ChromiumPDFPostRoute extends BrowserHTTPRoute {
                 await page.addScriptTag(addScriptTag[tag]);
             }
         }
-        const gotoResponse = await gotoCall(content, gotoOptions).catch(bestAttemptCatch(bestAttempt));
         if (waitForTimeout) {
             await sleep(waitForTimeout).catch(bestAttemptCatch(bestAttempt));
         }

package/build/shared/screenshot.http.js

@@ -70,6 +70,7 @@ export default class ScreenshotPost extends BrowserHTTPRoute {
                 return req.continue();
             });
         }
+        const gotoResponse = await gotoCall(content, gotoOptions).catch(bestAttemptCatch(bestAttempt));
         if (addStyleTag.length) {
             for (const tag in addStyleTag) {
                 await page.addStyleTag(addStyleTag[tag]);
@@ -80,7 +81,6 @@ export default class ScreenshotPost extends BrowserHTTPRoute {
                 await page.addScriptTag(addScriptTag[tag]);
             }
         }
-        const gotoResponse = await gotoCall(content, gotoOptions).catch(bestAttemptCatch(bestAttempt));
         if (waitForTimeout) {
             await sleep(waitForTimeout).catch(bestAttemptCatch(bestAttempt));
         }

package/build/types.d.ts

@@ -100,7 +100,7 @@ declare abstract class Route {
      * and injected by browserless after initialization.
      * @returns BrowserManager
      */
-    browserManager: () => import("@browserless.io/browserless").BrowserManager;
+    browserManager: () => import("src/browsers").BrowserManager;
     /**
      * Helper function that loads the config module. Defined and injected by
      * browserless after initialization.
@@ -120,7 +120,7 @@ declare abstract class Route {
      * browserless after initialization.
      * @returns FileSystem
      */
-    fileSystem: () => import("@browserless.io/browserless").FileSystem;
+    fileSystem: () => import("src/file-system").FileSystem;
     /**
      * Helper function that loads the metrics module for
      * collecting and aggregating statistics. Defined and injected by
@@ -134,7 +134,7 @@ declare abstract class Route {
      * browserless after initialization.
      * @returns Monitor
      */
-    monitoring: () => import("@browserless.io/browserless").Monitoring;
+    monitoring: () => import("src/monitoring").Monitoring;
     /**
      * When running in an SDK environment, this returns the fully-qualified
      * directory of that static directory. When "null" then no SDK directory

package/build/utils.js

@@ -536,7 +536,8 @@ export const encrypt = (text, secret) => {
     ].join(encryptionSep);
 };
 export const decrypt = (encryptedText, secret) => {
-    const [encrypted, iv] = encryptedText.split(encryptionSep);
+    const [encrypted, iv] = encryptedText.toString().split(encryptionSep);
+    console.log('>>', encryptedText.toString());
     if (!iv)
         throw new ServerError('Bad or invalid encrypted format');
     const decipher = crypto.createDecipheriv(encryptionAlgo, secret, Buffer.from(iv, 'hex'));