@browserless.io/browserless 2.0.0-beta-1

package/build/exports.d.ts

@@ -0,0 +1,20 @@
+export * from './browserless.js';
+export * from './config.js';
+export * from './constants.js';
+export * from './file-system.js';
+export * from './hooks.js';
+export * from './http.js';
+export * from './limiter.js';
+export * from './metrics.js';
+export * from './mime-types.js';
+export * from './monitoring.js';
+export * from './server.js';
+export * from './shim.js';
+export * from './types.js';
+export * from './utils.js';
+export * from './webhooks.js';
+export * from './browsers/index.js';
+export * from './browsers/cdp-chromium.js';
+export * from './browsers/playwright-chromium.js';
+export * from './browsers/playwright-firefox.js';
+export * from './browsers/playwright-webkit.js';

package/build/exports.js

@@ -0,0 +1,22 @@
+// Main Files
+export * from './browserless.js';
+export * from './config.js';
+export * from './constants.js';
+export * from './file-system.js';
+export * from './hooks.js';
+export * from './http.js';
+export * from './limiter.js';
+export * from './metrics.js';
+export * from './mime-types.js';
+export * from './monitoring.js';
+export * from './server.js';
+export * from './shim.js';
+export * from './types.js';
+export * from './utils.js';
+export * from './webhooks.js';
+// Browsers and Helpers
+export * from './browsers/index.js';
+export * from './browsers/cdp-chromium.js';
+export * from './browsers/playwright-chromium.js';
+export * from './browsers/playwright-firefox.js';
+export * from './browsers/playwright-webkit.js';

package/build/file-system.d.ts

@@ -0,0 +1,24 @@
+import { Config } from '@browserless.io/browserless';
+export declare class FileSystem {
+    private config;
+    private fsMap;
+    constructor(config: Config);
+    /**
+     * Appends contents to a file-path for persistance. File contents are
+     * encrypted before being saved to disk. Reads happen via the in-memory
+     * lookup of the internal map.
+     *
+     * @param path The filepath to persist contents to
+     * @param newContent A string of new content to add to the file
+     * @returns void
+     */
+    append: (path: string, newContent: string) => Promise<void>;
+    /**
+     * Reads contents from the local map, if any exist, or loads
+     * from the file system and hydrates the cache for the particular filepath
+     *
+     * @param path The filepath of the contents to read
+     * @returns Promise of the contents separated by newlines
+     */
+    read: (path: string) => Promise<string[]>;
+}

package/build/file-system.js

@@ -0,0 +1,46 @@
+import { decrypt, encrypt } from '@browserless.io/browserless';
+import { readFile, writeFile } from 'fs/promises';
+export class FileSystem {
+    config;
+    fsMap = new Map();
+    constructor(config) {
+        this.config = config;
+    }
+    /**
+     * Appends contents to a file-path for persistance. File contents are
+     * encrypted before being saved to disk. Reads happen via the in-memory
+     * lookup of the internal map.
+     *
+     * @param path The filepath to persist contents to
+     * @param newContent A string of new content to add to the file
+     * @returns void
+     */
+    append = async (path, newContent) => {
+        const contents = await this.read(path);
+        contents.push(newContent);
+        this.fsMap.set(path, contents);
+        const key = this.config.getAESKey();
+        const encoded = await encrypt(contents.join('\n'), Buffer.from(key));
+        return writeFile(path, encoded.toString());
+    };
+    /**
+     * Reads contents from the local map, if any exist, or loads
+     * from the file system and hydrates the cache for the particular filepath
+     *
+     * @param path The filepath of the contents to read
+     * @returns Promise of the contents separated by newlines
+     */
+    read = async (path) => {
+        const hasKey = this.fsMap.has(path);
+        if (hasKey) {
+            return this.fsMap.get(path);
+        }
+        const key = this.config.getAESKey();
+        const contents = (await readFile(path).catch(() => '')).toString();
+        const splitContents = contents.length
+            ? (await decrypt(contents, Buffer.from(key))).split('\n')
+            : [];
+        this.fsMap.set(path, splitContents);
+        return splitContents;
+    };
+}

package/build/hooks.d.ts

@@ -0,0 +1,5 @@
+import { AfterResponse, BeforeRequest, BrowserHook, PageHook } from '@browserless.io/browserless';
+export declare const beforeRequest: (args: BeforeRequest) => boolean;
+export declare const afterRequest: (args: AfterResponse | unknown) => boolean;
+export declare const browserHook: (opts: BrowserHook) => Promise<boolean>;
+export declare const pageHook: (opts: PageHook) => Promise<boolean>;

package/build/hooks.js

@@ -0,0 +1,20 @@
+import path, { dirname } from 'path';
+import { fileURLToPath } from 'url';
+import fs from 'fs';
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const beforeHookPath = path.join(__dirname, '..', 'external', 'before.js');
+const afterHookPath = path.join(__dirname, '..', 'external', 'after.js');
+const browserSetupPath = path.join(__dirname, '..', 'external', 'browser.js');
+const pageSetupPath = path.join(__dirname, '..', 'external', 'page.js');
+export const beforeRequest = fs.existsSync(beforeHookPath)
+    ? (await import(beforeHookPath)).default
+    : () => true;
+export const afterRequest = fs.existsSync(afterHookPath)
+    ? (await import(afterHookPath)).default
+    : () => true;
+export const browserHook = fs.existsSync(browserSetupPath)
+    ? (await import(browserSetupPath)).default
+    : () => Promise.resolve(true);
+export const pageHook = fs.existsSync(pageSetupPath)
+    ? (await import(pageSetupPath)).default
+    : () => Promise.resolve(true);

package/build/http.d.ts

@@ -0,0 +1,175 @@
+/// <reference types="node" />
+import { BrowserServerOptions, CDPLaunchOptions } from '@browserless.io/browserless';
+import http from 'http';
+export declare const errorCodes: {
+    readonly 400: {
+        readonly code: 400;
+        readonly description: "The request contains errors or didn't properly encode content.";
+        readonly message: "HTTP/1.1 400 Bad Request";
+    };
+    readonly 401: {
+        readonly code: 401;
+        readonly description: "The request is missing, or contains bad, authorization credentials.";
+        readonly message: "HTTP/1.1 401 Unauthorized";
+    };
+    readonly 404: {
+        readonly code: 404;
+        readonly description: "Resource couldn't be found.";
+        readonly message: "HTTP/1.1 404 Not Found";
+    };
+    readonly 408: {
+        readonly code: 408;
+        readonly description: "The request took has taken too long to process.";
+        readonly message: "HTTP/1.1 408 Request Timeout";
+    };
+    readonly 429: {
+        readonly code: 429;
+        readonly description: "Too many requests are currently being processed.";
+        readonly message: "HTTP/1.1 429 Too Many Requests";
+    };
+    readonly 500: {
+        readonly code: 500;
+        readonly description: "An internal error occurred when handling the request.";
+        readonly message: "HTTP/1.1 500 Internal Server Error";
+    };
+};
+export declare const okCodes: {
+    readonly 200: {
+        readonly code: 200;
+        readonly description: "The request ran successfully and returned an OK response.";
+        readonly message: "HTTP/1.1 200 OK";
+    };
+    readonly 204: {
+        readonly code: 204;
+        readonly description: "The request ran successfully, but no response was necessary.";
+        readonly message: "HTTP/1.1 204 No Content";
+    };
+};
+export declare const codes: {
+    readonly 200: {
+        readonly code: 200;
+        readonly description: "The request ran successfully and returned an OK response.";
+        readonly message: "HTTP/1.1 200 OK";
+    };
+    readonly 204: {
+        readonly code: 204;
+        readonly description: "The request ran successfully, but no response was necessary.";
+        readonly message: "HTTP/1.1 204 No Content";
+    };
+    readonly 400: {
+        readonly code: 400;
+        readonly description: "The request contains errors or didn't properly encode content.";
+        readonly message: "HTTP/1.1 400 Bad Request";
+    };
+    readonly 401: {
+        readonly code: 401;
+        readonly description: "The request is missing, or contains bad, authorization credentials.";
+        readonly message: "HTTP/1.1 401 Unauthorized";
+    };
+    readonly 404: {
+        readonly code: 404;
+        readonly description: "Resource couldn't be found.";
+        readonly message: "HTTP/1.1 404 Not Found";
+    };
+    readonly 408: {
+        readonly code: 408;
+        readonly description: "The request took has taken too long to process.";
+        readonly message: "HTTP/1.1 408 Request Timeout";
+    };
+    readonly 429: {
+        readonly code: 429;
+        readonly description: "Too many requests are currently being processed.";
+        readonly message: "HTTP/1.1 429 Too Many Requests";
+    };
+    readonly 500: {
+        readonly code: 500;
+        readonly description: "An internal error occurred when handling the request.";
+        readonly message: "HTTP/1.1 500 Internal Server Error";
+    };
+};
+export declare enum contentTypes {
+    any = "*/*",
+    html = "text/html",
+    javascript = "application/javascript",
+    jpeg = "image/jpeg",
+    json = "application/json",
+    pdf = "application/pdf",
+    png = "image/png",
+    text = "text/plain",
+    zip = "application/zip"
+}
+export declare enum encodings {
+    utf8 = "UTF-8"
+}
+export declare enum Methods {
+    delete = "delete",
+    get = "get",
+    post = "post",
+    put = "put"
+}
+export declare enum WebsocketRoutes {
+    '/' = "?(/)",
+    browser = "/devtools/browser/*",
+    page = "/devtools/page/*",
+    playwrightChromium = "/playwright/chromium",
+    playwrightFirefox = "/playwright/firefox",
+    playwrightWebkit = "/playwright/webkit"
+}
+export declare enum HTTPRoutes {
+    content = "/content",
+    download = "/download",
+    function = "/function",
+    pdf = "/pdf",
+    performance = "/performance",
+    scrape = "/scrape",
+    screenshot = "/screenshot"
+}
+export declare enum HTTPManagementRoutes {
+    config = "/config",
+    metrics = "/metrics",
+    metricsTotal = "/metrics/total",
+    sessions = "/sessions",
+    static = "/"
+}
+export declare enum APITags {
+    'browserAPI' = "Browser APIs",
+    'browserWS' = "Browser WebSockets",
+    'management' = "Management APIs"
+}
+export interface Request extends http.IncomingMessage {
+    body: unknown;
+    parsed: URL;
+    queryParams: Record<string, unknown>;
+}
+export type Response = http.ServerResponse;
+export interface SystemQueryParameters {
+    /**
+     * Whether or nor to load ad-blocking extensions for the session.
+     * This currently uses uBlock Origin and may cause certain sites
+     * to not load properly.
+     */
+    blockAds?: boolean;
+    /**
+     * Launch options, which can be either an object
+     * of puppeteer.launch options or playwright.launchServer
+     * options, depending on the API. Must be either JSON
+     * object, or a base64-encoded JSON object.
+     */
+    launch?: CDPLaunchOptions | BrowserServerOptions | string;
+    /**
+     * Whether or nor to record the session. The browser will run
+     * in "head-full" mode, and recording is started and closed
+     * via the embedded browserless API. Please refer to the "Recording"
+     * section in the live documentation site for more details.
+     */
+    record?: boolean;
+    /**
+     * Override the system-level timeout for this request.
+     * Accepts a value in milliseconds.
+     */
+    timeout?: number;
+    /**
+     * The authorization token
+     */
+    token?: string;
+}

package/build/http.js

@@ -0,0 +1,104 @@
+export const errorCodes = {
+    400: {
+        code: 400,
+        description: `The request contains errors or didn't properly encode content.`,
+        message: 'HTTP/1.1 400 Bad Request',
+    },
+    401: {
+        code: 401,
+        description: `The request is missing, or contains bad, authorization credentials.`,
+        message: 'HTTP/1.1 401 Unauthorized',
+    },
+    404: {
+        code: 404,
+        description: `Resource couldn't be found.`,
+        message: 'HTTP/1.1 404 Not Found',
+    },
+    408: {
+        code: 408,
+        description: `The request took has taken too long to process.`,
+        message: 'HTTP/1.1 408 Request Timeout',
+    },
+    429: {
+        code: 429,
+        description: `Too many requests are currently being processed.`,
+        message: 'HTTP/1.1 429 Too Many Requests',
+    },
+    500: {
+        code: 500,
+        description: `An internal error occurred when handling the request.`,
+        message: 'HTTP/1.1 500 Internal Server Error',
+    },
+};
+export const okCodes = {
+    200: {
+        code: 200,
+        description: `The request ran successfully and returned an OK response.`,
+        message: 'HTTP/1.1 200 OK',
+    },
+    204: {
+        code: 204,
+        description: `The request ran successfully, but no response was necessary.`,
+        message: 'HTTP/1.1 204 No Content',
+    },
+};
+export const codes = {
+    ...errorCodes,
+    ...okCodes,
+};
+export var contentTypes;
+(function (contentTypes) {
+    contentTypes["any"] = "*/*";
+    contentTypes["html"] = "text/html";
+    contentTypes["javascript"] = "application/javascript";
+    contentTypes["jpeg"] = "image/jpeg";
+    contentTypes["json"] = "application/json";
+    contentTypes["pdf"] = "application/pdf";
+    contentTypes["png"] = "image/png";
+    contentTypes["text"] = "text/plain";
+    contentTypes["zip"] = "application/zip";
+})(contentTypes || (contentTypes = {}));
+export var encodings;
+(function (encodings) {
+    encodings["utf8"] = "UTF-8";
+})(encodings || (encodings = {}));
+export var Methods;
+(function (Methods) {
+    Methods["delete"] = "delete";
+    Methods["get"] = "get";
+    Methods["post"] = "post";
+    Methods["put"] = "put";
+})(Methods || (Methods = {}));
+export var WebsocketRoutes;
+(function (WebsocketRoutes) {
+    WebsocketRoutes["/"] = "?(/)";
+    WebsocketRoutes["browser"] = "/devtools/browser/*";
+    WebsocketRoutes["page"] = "/devtools/page/*";
+    WebsocketRoutes["playwrightChromium"] = "/playwright/chromium";
+    WebsocketRoutes["playwrightFirefox"] = "/playwright/firefox";
+    WebsocketRoutes["playwrightWebkit"] = "/playwright/webkit";
+})(WebsocketRoutes || (WebsocketRoutes = {}));
+export var HTTPRoutes;
+(function (HTTPRoutes) {
+    HTTPRoutes["content"] = "/content";
+    HTTPRoutes["download"] = "/download";
+    HTTPRoutes["function"] = "/function";
+    HTTPRoutes["pdf"] = "/pdf";
+    HTTPRoutes["performance"] = "/performance";
+    HTTPRoutes["scrape"] = "/scrape";
+    HTTPRoutes["screenshot"] = "/screenshot";
+})(HTTPRoutes || (HTTPRoutes = {}));
+export var HTTPManagementRoutes;
+(function (HTTPManagementRoutes) {
+    HTTPManagementRoutes["config"] = "/config";
+    HTTPManagementRoutes["metrics"] = "/metrics";
+    HTTPManagementRoutes["metricsTotal"] = "/metrics/total";
+    HTTPManagementRoutes["sessions"] = "/sessions";
+    HTTPManagementRoutes["static"] = "/";
+})(HTTPManagementRoutes || (HTTPManagementRoutes = {}));
+export var APITags;
+(function (APITags) {
+    APITags["browserAPI"] = "Browser APIs";
+    APITags["browserWS"] = "Browser WebSockets";
+    APITags["management"] = "Management APIs";
+})(APITags || (APITags = {}));

package/build/index.d.ts

@@ -0,0 +1 @@
+export {};

package/build/index.js

@@ -0,0 +1,39 @@
+import { Browserless, createLogger } from '@browserless.io/browserless';
+(async () => {
+    const browserless = new Browserless();
+    const debug = createLogger('index.js');
+    browserless.start();
+    process
+        .on('unhandledRejection', async (reason, promise) => {
+        console.error('Unhandled Rejection at:', promise, 'reason:', reason);
+    })
+        .once('uncaughtException', async (err, origin) => {
+        console.error('Unhandled exception at:', origin, 'error:', err);
+        await browserless.stop();
+        process.exit(1);
+    })
+        .once('SIGTERM', async () => {
+        debug(`SIGTERM received, saving and closing down`);
+        await browserless.stop();
+        process.exit(0);
+    })
+        .once('SIGINT', async () => {
+        debug(`SIGINT received, saving and closing down`);
+        await browserless.stop();
+        process.exit(0);
+    })
+        .once('SIGHUP', async () => {
+        debug(`SIGHUP received, saving and closing down`);
+        await browserless.stop();
+        process.exit(0);
+    })
+        .once('SIGUSR2', async () => {
+        debug(`SIGUSR2 received, saving and closing down`);
+        await browserless.stop();
+        process.exit(0);
+    })
+        .once('exit', () => {
+        debug(`Process is finished, exiting`);
+        process.exit(0);
+    });
+})();

package/build/limiter.d.ts

@@ -0,0 +1,24 @@
+import { Config, Metrics, Monitoring, WebHooks } from '@browserless.io/browserless';
+import q from 'queue';
+export type LimitFn<TArgs extends unknown[], TResult> = (...args: TArgs) => Promise<TResult>;
+export type ErrorFn<TArgs extends unknown[]> = (...args: TArgs) => void;
+export declare class Limiter extends q {
+    private config;
+    private metrics;
+    private monitor;
+    private webhooks;
+    private queued;
+    private debug;
+    constructor(config: Config, metrics: Metrics, monitor: Monitoring, webhooks: WebHooks);
+    private handleEnd;
+    private handleSuccess;
+    private handleJobTimeout;
+    private handleFail;
+    private logQueue;
+    get executing(): number;
+    get waiting(): number;
+    get willQueue(): boolean;
+    get concurrencySize(): number;
+    get hasCapacity(): boolean;
+    limit: <TArgs extends unknown[], TResult>(limitFn: LimitFn<TArgs, TResult>, overCapacityFn: ErrorFn<TArgs>, onTimeoutFn: ErrorFn<TArgs>, timeoutOverrideFn: (...args: TArgs) => number | undefined) => LimitFn<TArgs, unknown>;
+}

package/build/limiter.js

@@ -0,0 +1,147 @@
+import { TooManyRequests, afterRequest, createLogger, } from '@browserless.io/browserless';
+import q from 'queue';
+export class Limiter extends q {
+    config;
+    metrics;
+    monitor;
+    webhooks;
+    queued;
+    debug = createLogger('limiter');
+    constructor(config, metrics, monitor, webhooks) {
+        super({
+            autostart: true,
+            concurrency: config.getConcurrent(),
+            timeout: config.getTimeout(),
+        });
+        this.config = config;
+        this.metrics = metrics;
+        this.monitor = monitor;
+        this.webhooks = webhooks;
+        this.queued = config.getQueued();
+        this.debug(`Concurrency: ${this.concurrency} queue: ${this.queued} timeout: ${this.timeout}ms`);
+        config.on('concurrent', (concurrency) => {
+            this.debug(`Concurrency updated to ${concurrency}`);
+            this.concurrency = concurrency;
+        });
+        config.on('queued', (queued) => {
+            this.debug(`Queue updated to ${queued}`);
+            this.queued = queued;
+        });
+        config.on('timeout', (timeout) => {
+            this.debug(`Timeout updated to ${timeout}ms`);
+            this.timeout = timeout <= 0 ? 0 : timeout;
+        });
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        this.addEventListener('timeout', this.handleJobTimeout);
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        this.addEventListener('success', this.handleSuccess);
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        this.addEventListener('error', this.handleFail);
+        this.addEventListener('end', this.handleEnd);
+    }
+    handleEnd() {
+        this.logQueue('All jobs complete.');
+    }
+    handleSuccess({ detail: { job } }) {
+        const timeUsed = Date.now() - job.start;
+        this.debug(`Job has succeeded after ${timeUsed.toLocaleString()}ms of activity.`);
+        this.metrics.addSuccessful(Date.now() - job.start);
+        // @TODO Figure out a better argument handling for jobs
+        afterRequest({
+            req: job.args[0],
+            start: job.start,
+            status: 'successful',
+        });
+    }
+    handleJobTimeout({ detail: { next, job }, }) {
+        const timeUsed = Date.now() - job.start;
+        this.debug(`Job has hit timeout after ${timeUsed.toLocaleString()}ms of activity.`);
+        this.metrics.addTimedout(Date.now() - job.start);
+        this.webhooks.callTimeoutAlertURL();
+        this.debug(`Calling timeout handler`);
+        job?.onTimeoutFn(job);
+        afterRequest({
+            req: job.args[0],
+            start: job.start,
+            status: 'timedout',
+        });
+        next();
+    }
+    handleFail({ detail: { error, job }, }) {
+        this.debug(`Recording failed stat, cleaning up: "${error?.toString()}"`);
+        this.metrics.addError(Date.now() - job.start);
+        this.webhooks.callErrorAlertURL(error?.toString() ?? 'Unknown Error');
+        afterRequest({
+            req: job.args[0],
+            start: job.start,
+            status: 'error',
+        });
+    }
+    logQueue(message) {
+        this.debug(`(Running: ${this.executing}, Pending: ${this.waiting}) ${message} `);
+    }
+    get executing() {
+        return this.length > this.concurrency ? this.concurrency : this.length;
+    }
+    get waiting() {
+        return this.length > this.concurrency ? this.length - this.concurrency : 0;
+    }
+    get willQueue() {
+        return this.length >= this.concurrency;
+    }
+    get concurrencySize() {
+        return this.concurrency;
+    }
+    get hasCapacity() {
+        return this.length < this.concurrency + this.queued;
+    }
+    limit = (limitFn, overCapacityFn, onTimeoutFn, timeoutOverrideFn) => {
+        return (...args) => new Promise(async (res, rej) => {
+            const timeout = timeoutOverrideFn(...args) ?? this.timeout;
+            this.logQueue(`Adding to queue, max time allowed is ${timeout.toLocaleString()}ms`);
+            if (this.config.getHealthChecksEnabled()) {
+                const { cpuOverloaded, memoryOverloaded } = await this.monitor.overloaded();
+                if (cpuOverloaded || memoryOverloaded) {
+                    this.logQueue(`Health checks have failed, rejecting`);
+                    this.webhooks.callFailedHealthURL();
+                    this.metrics.addRejected();
+                    overCapacityFn(...args);
+                    return rej(new Error(`Health checks have failed, rejecting`));
+                }
+            }
+            if (!this.hasCapacity) {
+                this.logQueue(`Concurrency and queue is at capacity`);
+                this.webhooks.callRejectAlertURL();
+                this.metrics.addRejected();
+                overCapacityFn(...args);
+                return rej(new TooManyRequests(`Concurrency and queue is at capacity`));
+            }
+            if (this.willQueue) {
+                this.logQueue(`Concurrency is at capacity, queueing`);
+                this.webhooks.callQueueAlertURL();
+                this.metrics.addQueued();
+            }
+            const bound = async () => {
+                this.logQueue(`Starting new job`);
+                this.metrics.addRunning();
+                try {
+                    const result = await limitFn(...args);
+                    res(result);
+                    return;
+                }
+                catch (err) {
+                    rej(err);
+                    throw err;
+                }
+            };
+            const job = Object.assign(bound, {
+                args,
+                onTimeoutFn: () => onTimeoutFn(...args),
+                start: Date.now(),
+                timeout,
+            });
+            this.push(job);
+            return bound;
+        });
+    };
+}

package/build/limiter.spec.d.ts

@@ -0,0 +1 @@
+export {};