@browserless.io/browserless 2.0.0-beta-1 → 2.0.0-beta-2

package/docker/sdk/Dockerfile

@@ -1,7 +1,6 @@
-ARG FROM=ghcr.io/browserless/multi
-ARG VERSION=latest
+ARG FROM=ghcr.io/browserless/multi:latest
 
-FROM ${FROM}:${VERSION}
+FROM ${FROM}
 
 # Change to root for install
 USER root
@@ -17,9 +16,6 @@ COPY src src
 COPY package.json .
 COPY package-lock.json .
 COPY tsconfig.json .
-
-# Optional files that might not always be there
-COPY *browserless.io-browserless-*.tgz .
 COPY *README.md .
 
 # Install dependencies

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@browserless.io/browserless",
-  "version": "2.0.0-beta-1",
+  "version": "2.0.0-beta-2",
   "license": "SSPL",
   "description": "The browserless platform",
   "author": "browserless.io",
@@ -71,13 +71,13 @@
     "@types/mocha": "^10.0.6",
     "@types/node": "^20.10.5",
     "@types/sinon": "^17.0.2",
-    "@typescript-eslint/eslint-plugin": "^6.14.0",
-    "@typescript-eslint/parser": "^6.14.0",
+    "@typescript-eslint/eslint-plugin": "^6.15.0",
+    "@typescript-eslint/parser": "^6.15.0",
     "assert": "^2.0.0",
     "chai": "^4.3.6",
     "cross-env": "^7.0.3",
     "env-cmd": "^10.1.0",
-    "esbuild": "^0.19.9",
+    "esbuild": "^0.19.10",
     "esbuild-plugin-polyfill-node": "^0.3.0",
     "eslint": "^8.56.0",
     "eslint-plugin-import": "^2.29.1",

package/src/browserless.ts

@@ -11,6 +11,8 @@ import {
   Limiter,
   Metrics,
   Monitoring,
+  Router,
+  Token,
   WebHooks,
   WebSocketRoute,
   availableBrowsers,
@@ -26,14 +28,16 @@ import { userInfo } from 'os';
 const routeSchemas = ['body', 'query'];
 
 export class Browserless {
-  private config: Config;
-  private monitoring: Monitoring;
-  private metrics: Metrics;
-  private fileSystem: FileSystem;
-  private browserManager: BrowserManager;
-  private limiter: Limiter;
-  private webhooks: WebHooks;
-  private debug: debug.Debugger = createLogger('index');
+  protected debug: debug.Debugger = createLogger('index');
+  protected browserManager: BrowserManager;
+  protected config: Config;
+  protected fileSystem: FileSystem;
+  protected limiter: Limiter;
+  protected metrics: Metrics;
+  protected monitoring: Monitoring;
+  protected router: Router;
+  protected token: Token;
+  protected webhooks: WebHooks;
 
   webSocketRouteFiles: string[] = [];
   httpRouteFiles: string[] = [];
@@ -44,22 +48,27 @@ export class Browserless {
   constructor({
     browserManager,
     config,
-    monitoring,
+    fileSystem,
     limiter,
     metrics,
-    fileSystem,
+    monitoring,
+    router,
+    token,
     webhooks,
   }: {
-    browserManager?: BrowserManager;
-    config?: Config;
-    fileSystem?: FileSystem;
-    limiter?: Limiter;
-    metrics?: Metrics;
-    monitoring?: Monitoring;
-    webhooks?: WebHooks;
+    browserManager?: Browserless['browserManager'];
+    config?: Browserless['config'];
+    fileSystem?: Browserless['fileSystem'];
+    limiter?: Browserless['limiter'];
+    metrics?: Browserless['metrics'];
+    monitoring?: Browserless['monitoring'];
+    router?: Browserless['router'];
+    token?: Browserless['token'];
+    webhooks?: Browserless['webhooks'];
   } = {}) {
     this.config = config || new Config();
     this.metrics = metrics || new Metrics();
+    this.token = token || new Token(this.config);
     this.webhooks = webhooks || new WebHooks(this.config);
     this.browserManager = browserManager || new BrowserManager(this.config);
     this.monitoring = monitoring || new Monitoring(this.config);
@@ -67,9 +76,11 @@ export class Browserless {
     this.limiter =
       limiter ||
       new Limiter(this.config, this.metrics, this.monitoring, this.webhooks);
+    this.router =
+      router || new Router(this.config, this.browserManager, this.limiter);
   }
 
-  private saveMetrics = async (): Promise<void> => {
+  protected saveMetrics = async (): Promise<void> => {
     const metricsPath = this.config.getMetricsJSONPath();
     const { cpu, memory } = await this.monitoring.getMachineStats();
     const metrics = await this.metrics.get();
@@ -175,11 +186,11 @@ export class Browserless {
 
         route.bodySchema = safeParse(bodySchema);
         route.querySchema = safeParse(querySchema);
-        route._config = () => this.config;
-        route._metrics = () => this.metrics;
-        route._monitor = () => this.monitoring;
-        route._fileSystem = () => this.fileSystem;
-        route._debug = () => logger;
+        route.getConfig = () => this.config;
+        route.getMetrics = () => this.metrics;
+        route.getMonitoring = () => this.monitoring;
+        route.getFileSystem = () => this.fileSystem;
+        route.getDebug = () => logger;
 
         httpRoutes.push(route);
       }
@@ -210,11 +221,11 @@ export class Browserless {
         );
 
         route.querySchema = safeParse(querySchema);
-        route._config = () => this.config;
-        route._metrics = () => this.metrics;
-        route._monitor = () => this.monitoring;
-        route._fileSystem = () => this.fileSystem;
-        route._debug = () => logger;
+        route.getConfig = () => this.config;
+        route.getMetrics = () => this.metrics;
+        route.getMonitoring = () => this.monitoring;
+        route.getFileSystem = () => this.fileSystem;
+        route.getDebug = () => logger;
 
         wsRoutes.push(route);
       }
@@ -232,15 +243,16 @@ export class Browserless {
       }
     });
 
+    httpRoutes.forEach((r) => this.router.registerHTTPRoute(r));
+    wsRoutes.forEach((r) => this.router.registerWebSocketRoute(r));
+
     this.debug(`Imported and validated all route files, starting up server.`);
 
     this.server = new HTTPServer(
       this.config,
       this.metrics,
-      this.browserManager,
-      this.limiter,
-      httpRoutes,
-      wsRoutes,
+      this.token,
+      this.router,
     );
 
     await this.server.start();

package/src/browsers/cdp-chromium.ts

@@ -24,16 +24,16 @@ import puppeteerStealth from 'puppeteer-extra';
 puppeteerStealth.use(StealthPlugin());
 
 export class CDPChromium extends EventEmitter {
-  private config: Config;
-  private userDataDir: string | null;
-  private record: boolean;
-  private blockAds: boolean;
-  private running = false;
-  private browser: Browser | null = null;
-  private browserWSEndpoint: string | null = null;
-  private port?: number;
-  private debug = createLogger('browsers:cdp:chromium');
-  private proxy = httpProxy.createProxyServer();
+  protected config: Config;
+  protected userDataDir: string | null;
+  protected record: boolean;
+  protected blockAds: boolean;
+  protected running = false;
+  protected browser: Browser | null = null;
+  protected browserWSEndpoint: string | null = null;
+  protected port?: number;
+  protected debug = createLogger('browsers:cdp:chromium');
+  protected proxy = httpProxy.createProxyServer();
 
   constructor({
     userDataDir,
@@ -55,12 +55,12 @@ export class CDPChromium extends EventEmitter {
     this.debug(`Starting new browser instance`);
   }
 
-  private cleanListeners() {
+  protected cleanListeners() {
     this.browser?.removeAllListeners();
     this.removeAllListeners();
   }
 
-  private setUpEmbeddedAPI = async (
+  protected setUpEmbeddedAPI = async (
     page: Page,
     id: string,
     record: boolean,
@@ -189,7 +189,7 @@ export class CDPChromium extends EventEmitter {
     return makeExternalURL(serverAddress, 'live', query);
   };
 
-  private onTargetCreated = async (target: Target) => {
+  protected onTargetCreated = async (target: Target) => {
     if (target.type() === 'page') {
       const page = await target.page().catch((e) => {
         this.debug(`Error in new page ${e}`);

package/src/browsers/index.ts

@@ -17,7 +17,6 @@ import {
   convertIfBase64,
   createLogger,
   exists,
-  getTokenFromRequest,
   id,
   makeExternalURL,
   noop,
@@ -29,14 +28,14 @@ import { deleteAsync } from 'del';
 import { mkdir } from 'fs/promises';
 
 export class BrowserManager {
-  private browsers: Map<BrowserInstance, BrowserlessSession> = new Map();
-  private launching: Map<string, Promise<unknown>> = new Map();
-  private timers: Map<string, number> = new Map();
-  private debug = createLogger('browser-manager');
+  protected browsers: Map<BrowserInstance, BrowserlessSession> = new Map();
+  protected launching: Map<string, Promise<unknown>> = new Map();
+  protected timers: Map<string, number> = new Map();
+  protected debug = createLogger('browser-manager');
 
-  constructor(private config: Config) {}
+  constructor(protected config: Config) {}
 
-  private removeUserDataDir = async (userDataDir: string | null) => {
+  protected removeUserDataDir = async (userDataDir: string | null) => {
     if (userDataDir && (await exists(userDataDir))) {
       this.debug(`Deleting data directory "${userDataDir}"`);
       await deleteAsync(userDataDir, { force: true }).catch((err) => {
@@ -56,7 +55,7 @@ export class BrowserManager {
    * @param sessionId The ID of the session
    * @returns Promise<string> of the fully-qualified path of the directory
    */
-  private generateDataDir = async (
+  protected generateDataDir = async (
     sessionId: string = id(),
   ): Promise<string> => {
     const baseDirectory = await this.config.getDataDir();
@@ -83,7 +82,7 @@ export class BrowserManager {
     return dataDirPath;
   };
 
-  private generateSessionJson = (
+  protected generateSessionJson = (
     browser: BrowserInstance,
     session: BrowserlessSession,
   ) => {
@@ -127,17 +126,9 @@ export class BrowserManager {
     await Promise.all(cleanupACtions.map((a) => a()));
   };
 
-  public getAllSessions = async (
-    req: Request,
-  ): Promise<BrowserlessSessionJSON[]> => {
+  public getAllSessions = async (): Promise<BrowserlessSessionJSON[]> => {
     const sessions = Array.from(this.browsers);
 
-    const requestToken = getTokenFromRequest(req);
-    const token = this.config.getToken();
-    if (token && !requestToken) {
-      throw new BadRequest(`Couldn't locate your API token`);
-    }
-
     return sessions.map(([browser, session]) =>
       this.generateSessionJson(browser, session),
     );
@@ -203,12 +194,6 @@ export class BrowserManager {
         `Error parsing launch-options: ${err}. Launch options must be a JSON or base64-encoded JSON object`,
       );
     }
-    const requestToken = getTokenFromRequest(req);
-    const token = this.config.getToken();
-
-    if (token && !requestToken) {
-      throw new ServerError(`Error locating authorization token`);
-    }
 
     const routerOptions =
       typeof router.defaultLaunchOptions === 'function'

package/src/browsers/playwright-chromium.ts

@@ -11,14 +11,14 @@ import { EventEmitter } from 'events';
 import httpProxy from 'http-proxy';
 
 export class PlaywrightChromium extends EventEmitter {
-  private config: Config;
-  private userDataDir: string | null;
-  private record: boolean;
-  private running = false;
-  private proxy = httpProxy.createProxyServer();
-  private browser: playwright.BrowserServer | null = null;
-  private browserWSEndpoint: string | null = null;
-  private debug = createLogger('browsers:playwright:chromium');
+  protected config: Config;
+  protected userDataDir: string | null;
+  protected record: boolean;
+  protected running = false;
+  protected proxy = httpProxy.createProxyServer();
+  protected browser: playwright.BrowserServer | null = null;
+  protected browserWSEndpoint: string | null = null;
+  protected debug = createLogger('browsers:playwright:chromium');
 
   constructor({
     config,
@@ -38,7 +38,7 @@ export class PlaywrightChromium extends EventEmitter {
     this.debug(`Starting new browser instance`);
   }
 
-  private cleanListeners() {
+  protected cleanListeners() {
     this.removeAllListeners();
   }
 

package/src/browsers/playwright-firefox.ts

@@ -11,14 +11,14 @@ import { EventEmitter } from 'events';
 import httpProxy from 'http-proxy';
 
 export class PlaywrightFirefox extends EventEmitter {
-  private config: Config;
-  private userDataDir: string | null;
-  private record: boolean;
-  private running = false;
-  private proxy = httpProxy.createProxyServer();
-  private browser: playwright.BrowserServer | null = null;
-  private browserWSEndpoint: string | null = null;
-  private debug = createLogger('browsers:playwright:firefox');
+  protected config: Config;
+  protected userDataDir: string | null;
+  protected record: boolean;
+  protected running = false;
+  protected proxy = httpProxy.createProxyServer();
+  protected browser: playwright.BrowserServer | null = null;
+  protected browserWSEndpoint: string | null = null;
+  protected debug = createLogger('browsers:playwright:firefox');
 
   constructor({
     config,
@@ -38,7 +38,7 @@ export class PlaywrightFirefox extends EventEmitter {
     this.debug(`Starting new browser instance`);
   }
 
-  private cleanListeners() {
+  protected cleanListeners() {
     this.removeAllListeners();
   }
 

package/src/browsers/playwright-webkit.ts

@@ -11,14 +11,14 @@ import { EventEmitter } from 'events';
 import httpProxy from 'http-proxy';
 
 export class PlaywrightWebkit extends EventEmitter {
-  private config: Config;
-  private userDataDir: string | null;
-  private record: boolean;
-  private running = false;
-  private proxy = httpProxy.createProxyServer();
-  private browser: playwright.BrowserServer | null = null;
-  private browserWSEndpoint: string | null = null;
-  private debug = createLogger('browsers:playwright:webkit');
+  protected config: Config;
+  protected userDataDir: string | null;
+  protected record: boolean;
+  protected running = false;
+  protected proxy = httpProxy.createProxyServer();
+  protected browser: playwright.BrowserServer | null = null;
+  protected browserWSEndpoint: string | null = null;
+  protected debug = createLogger('browsers:playwright:webkit');
 
   constructor({
     config,
@@ -38,7 +38,7 @@ export class PlaywrightWebkit extends EventEmitter {
     this.debug(`Starting new browser instance`);
   }
 
-  private cleanListeners() {
+  protected cleanListeners() {
     this.removeAllListeners();
   }
 

package/src/config.ts

@@ -109,60 +109,61 @@ const getDebug = () => {
 };
 
 export class Config extends EventEmitter {
-  private readonly debug = getDebug();
-  private readonly host = process.env.HOST ?? 'localhost';
-  private readonly external = process.env.PROXY_URL ?? process.env.EXTERNAL;
-  private readonly isWin = process.platform === 'win32';
+  protected readonly debug = getDebug();
+  protected readonly host = process.env.HOST ?? 'localhost';
+  protected readonly external = process.env.PROXY_URL ?? process.env.EXTERNAL;
+  protected readonly isWin = process.platform === 'win32';
 
-  private port = +(process.env.PORT ?? '3000');
+  protected port = +(process.env.PORT ?? '3000');
 
-  private downloadsDir = process.env.DOWNLOAD_DIR
+  protected downloadsDir = process.env.DOWNLOAD_DIR
     ? untildify(process.env.DOWNLOAD_DIR)
     : path.join(tmpdir(), 'browserless-download-dirs');
 
-  private dataDir = process.env.DATA_DIR
+  protected dataDir = process.env.DATA_DIR
     ? untildify(process.env.DATA_DIR)
     : path.join(tmpdir(), 'browserless-data-dirs');
 
-  private metricsJSONPath = process.env.METRICS_JSON_PATH
+  protected metricsJSONPath = process.env.METRICS_JSON_PATH
     ? untildify(process.env.METRICS_JSON_PATH)
     : path.join(tmpdir(), 'browserless-metrics.json');
 
-  private createDataDir = !process.env.DATA_DIR;
-  private createDownloadsDir = !process.env.DOWNLOAD_DIR;
+  protected createDataDir = !process.env.DATA_DIR;
+  protected createDownloadsDir = !process.env.DOWNLOAD_DIR;
 
-  private routes = process.env.ROUTES
+  protected routes = process.env.ROUTES
     ? untildify(process.env.ROUTES)
     : path.join(__dirname, '..', 'build', 'routes');
 
-  private token = process.env.TOKEN || null;
-  private concurrent = +(
+  protected token = process.env.TOKEN || null;
+  protected concurrent = +(
     process.env.CONCURRENT ??
     process.env.MAX_CONCURRENT_SESSIONS ??
     '10'
   );
-  private queued = +(process.env.QUEUE_LENGTH ?? process.env.QUEUED ?? '10');
-  private timeout = +(
+  protected queued = +(process.env.QUEUE_LENGTH ?? process.env.QUEUED ?? '10');
+  protected timeout = +(
     process.env.TIMEOUT ??
     process.env.CONNECTION_TIMEOUT ??
     '30000'
   );
-  private static = process.env.STATIC ?? path.join(__dirname, '..', 'static');
-  private retries = +(process.env.RETRIES ?? '5');
-  private allowFileProtocol = !!parseEnvVars(false, 'ALLOW_FILE_PROTOCOL');
-  private allowGet = !!parseEnvVars(false, 'ALLOW_GET', 'ENABLE_API_GET');
-  private allowCors = !!parseEnvVars(false, 'CORS', 'ENABLE_CORS');
-  private corsMethods = process.env.CORS_ALLOW_METHODS ?? 'OPTIONS, POST, GET';
-  private corsOrigin = process.env.CORS_ALLOW_ORIGIN ?? '*';
-  private corsMaxAge = +(process.env.CORS_MAX_AGE ?? '2592000');
-  private maxCpu = +(process.env.MAX_CPU_PERCENT ?? '99');
-  private maxMemory = +(process.env.MAX_MEMORY_PERCENT ?? '99');
-  private healthCheck = !!parseEnvVars(false, 'HEALTH');
-  private failedHealthURL = process.env.FAILED_HEALTH_URL ?? null;
-  private queueAlertURL = process.env.QUEUE_ALERT_URL ?? null;
-  private rejectAlertURL = process.env.REJECT_ALERT_URL ?? null;
-  private timeoutAlertURL = process.env.TIMEOUT_ALERT_URL ?? null;
-  private errorAlertURL = process.env.ERROR_ALERT_URL ?? null;
+  protected static = process.env.STATIC ?? path.join(__dirname, '..', 'static');
+  protected retries = +(process.env.RETRIES ?? '5');
+  protected allowFileProtocol = !!parseEnvVars(false, 'ALLOW_FILE_PROTOCOL');
+  protected allowGet = !!parseEnvVars(false, 'ALLOW_GET', 'ENABLE_API_GET');
+  protected allowCors = !!parseEnvVars(false, 'CORS', 'ENABLE_CORS');
+  protected corsMethods =
+    process.env.CORS_ALLOW_METHODS ?? 'OPTIONS, POST, GET';
+  protected corsOrigin = process.env.CORS_ALLOW_ORIGIN ?? '*';
+  protected corsMaxAge = +(process.env.CORS_MAX_AGE ?? '2592000');
+  protected maxCpu = +(process.env.MAX_CPU_PERCENT ?? '99');
+  protected maxMemory = +(process.env.MAX_MEMORY_PERCENT ?? '99');
+  protected healthCheck = !!parseEnvVars(false, 'HEALTH');
+  protected failedHealthURL = process.env.FAILED_HEALTH_URL ?? null;
+  protected queueAlertURL = process.env.QUEUE_ALERT_URL ?? null;
+  protected rejectAlertURL = process.env.REJECT_ALERT_URL ?? null;
+  protected timeoutAlertURL = process.env.TIMEOUT_ALERT_URL ?? null;
+  protected errorAlertURL = process.env.ERROR_ALERT_URL ?? null;
 
   public getRoutes = (): string => this.routes;
   public getHost = (): string => this.host;

package/src/exports.ts

@@ -9,8 +9,10 @@ export * from './limiter.js';
 export * from './metrics.js';
 export * from './mime-types.js';
 export * from './monitoring.js';
+export * from './router.js';
 export * from './server.js';
 export * from './shim.js';
+export * from './token.js';
 export * from './types.js';
 export * from './utils.js';
 export * from './webhooks.js';

package/src/file-system.ts

@@ -2,9 +2,9 @@ import { Config, decrypt, encrypt } from '@browserless.io/browserless';
 import { readFile, writeFile } from 'fs/promises';
 
 export class FileSystem {
-  private fsMap: Map<string, string[]> = new Map();
+  protected fsMap: Map<string, string[]> = new Map();
 
-  constructor(private config: Config) {}
+  constructor(protected config: Config) {}
 
   /**
    * Appends contents to a file-path for persistance. File contents are

package/src/limiter.ts

@@ -25,14 +25,14 @@ interface Job {
 }
 
 export class Limiter extends q {
-  private queued: number;
-  private debug = createLogger('limiter');
+  protected queued: number;
+  protected debug = createLogger('limiter');
 
   constructor(
-    private config: Config,
-    private metrics: Metrics,
-    private monitor: Monitoring,
-    private webhooks: WebHooks,
+    protected config: Config,
+    protected metrics: Metrics,
+    protected monitor: Monitoring,
+    protected webhooks: WebHooks,
   ) {
     super({
       autostart: true,
@@ -73,11 +73,11 @@ export class Limiter extends q {
     this.addEventListener('end', this.handleEnd);
   }
 
-  private handleEnd() {
+  protected handleEnd() {
     this.logQueue('All jobs complete.');
   }
 
-  private handleSuccess({ detail: { job } }: { detail: { job: Job } }) {
+  protected handleSuccess({ detail: { job } }: { detail: { job: Job } }) {
     const timeUsed = Date.now() - job.start;
     this.debug(
       `Job has succeeded after ${timeUsed.toLocaleString()}ms of activity.`,
@@ -91,7 +91,7 @@ export class Limiter extends q {
     } as AfterResponse);
   }
 
-  private handleJobTimeout({
+  protected handleJobTimeout({
     detail: { next, job },
   }: {
     detail: { job: Job; next: Job };
@@ -113,7 +113,7 @@ export class Limiter extends q {
     next();
   }
 
-  private handleFail({
+  protected handleFail({
     detail: { error, job },
   }: {
     detail: { error: unknown; job: Job };
@@ -128,7 +128,7 @@ export class Limiter extends q {
     } as AfterResponse);
   }
 
-  private logQueue(message: string) {
+  protected logQueue(message: string) {
     this.debug(
       `(Running: ${this.executing}, Pending: ${this.waiting}) ${message} `,
     );

package/src/metrics.ts

@@ -1,16 +1,16 @@
 import { IBrowserlessStats } from '@browserless.io/browserless';
 
 export class Metrics {
-  private sessionTimes: number[] = [];
-  private successful = 0;
-  private queued = 0;
-  private rejected = 0;
-  private unauthorized = 0;
-  private concurrent = 0;
-  private timedout = 0;
-  private running = 0;
-  private unhealthy = 0;
-  private error = 0;
+  protected sessionTimes: number[] = [];
+  protected successful = 0;
+  protected queued = 0;
+  protected rejected = 0;
+  protected unauthorized = 0;
+  protected concurrent = 0;
+  protected timedout = 0;
+  protected running = 0;
+  protected unhealthy = 0;
+  protected error = 0;
 
   addSuccessful = (sessionTime: number): number => {
     --this.running;
@@ -90,7 +90,7 @@ export class Metrics {
     this.sessionTimes = [];
   };
 
-  private calculateStats(sessionTimes: number[]) {
+  protected calculateStats(sessionTimes: number[]) {
     return {
       maxTime: Math.max(...sessionTimes) || 0,
       meanTime: sessionTimes.reduce(

package/src/monitoring.ts

@@ -6,8 +6,8 @@ import {
 import si from 'systeminformation';
 
 export class Monitoring {
-  private log = createLogger('hardware');
-  constructor(private config: Config) {}
+  protected log = createLogger('hardware');
+  constructor(protected config: Config) {}
 
   public getMachineStats = async (): Promise<IResourceLoad> => {
     const [cpuLoad, memLoad] = await Promise.all([