@browserless.io/browserless 2.0.0-beta-1 → 2.0.0-beta-2

package/bin/browserless.js

@@ -5,6 +5,7 @@ import { Browserless } from '@browserless.io/browserless';
 import buildOpenAPI from '../scripts/build-open-api.js';
 import buildSchemas from '../scripts/build-schemas.js';
 
+import { createInterface } from 'readline';
 import debug from 'debug';
 import { fileURLToPath } from 'url';
 import fs from 'fs/promises';
@@ -12,6 +13,7 @@ import path from 'path';
 import { spawn } from 'child_process';
 
 const log = debug('browserless:sdk:log');
+const promptLog = debug('browserless:prompt');
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const cmd = process.argv[2];
@@ -25,6 +27,21 @@ if (!allowedCMDs.includes(cmd)) {
   );
 }
 
+const prompt = async (question) => {
+  const rl = createInterface({
+    input: process.stdin,
+    output: process.stdout,
+  });
+
+  return new Promise((resolve) => {
+    promptLog(question);
+    rl.question('  > ', (a) => {
+      rl.close();
+      resolve(a.trim());
+    });
+  });
+};
+
 const importClassOverride = async (files, className) => {
   const classModuleFile = files.find((f) =>
     path.parse(f).name.endsWith(className),
@@ -43,6 +60,24 @@ const importClassOverride = async (files, className) => {
   return (await import(classModuleFullFilePath)).default;
 };
 
+const buildDockerImage = async (cmd) => {
+  new Promise((resolve, reject) => {
+    const [docker, ...args] = cmd.split(' ');
+    spawn(docker, args, {
+      cwd,
+      stdio: 'inherit',
+    }).once('close', (code) => {
+      if (code === 0) {
+        log(`Successfully built the docker image.`);
+        return resolve();
+      }
+      return reject(
+        `Error when building Docker image, see output for more details`,
+      );
+    });
+  });
+};
+
 const buildTypeScript = async () =>
   new Promise((resolve, reject) => {
     spawn('npx', ['tsc', '--outDir', 'build'], {
@@ -83,6 +118,35 @@ const getSourceFiles = async () => {
   };
 };
 
+const getArgSwitches = () => {
+  return process.argv.reduce((accum, arg, idx) => {
+    if (!arg.startsWith('--')) {
+      return accum;
+    }
+
+    if (arg.includes('=')) {
+      const [parameter, value] = arg.split('=');
+      accum[parameter.replace(/-/gi, '')] = value || true;
+      return accum;
+    }
+
+    const nextSwitchOrParameter = process.argv[idx + 1];
+    const param = arg.replace(/-/gi, '');
+
+    if (
+      typeof nextSwitchOrParameter === 'undefined' ||
+      nextSwitchOrParameter?.startsWith('--')
+    ) {
+      accum[param] = true;
+      return accum;
+    }
+
+    accum[param] = nextSwitchOrParameter;
+
+    return accum;
+  }, {});
+};
+
 /**
  * Build
  * Responsible for compiling TypeScript, generating routes meta-data
@@ -119,6 +183,7 @@ const start = async (dev = false) => {
     : await getSourceFiles();
 
   log(`Importing all class overrides if present`);
+
   const [
     browserManager,
     config,
@@ -126,6 +191,8 @@ const start = async (dev = false) => {
     limiter,
     metrics,
     monitoring,
+    router,
+    token,
     webhooks,
   ] = await Promise.all([
     importClassOverride(files, 'browser-manager'),
@@ -134,10 +201,13 @@ const start = async (dev = false) => {
     importClassOverride(files, 'limiter'),
     importClassOverride(files, 'metrics'),
     importClassOverride(files, 'monitoring'),
+    importClassOverride(files, 'router'),
+    importClassOverride(files, 'token'),
     importClassOverride(files, 'webhooks'),
   ]);
 
   log(`Starting Browserless`);
+
   const browserless = new Browserless({
     browserManager,
     config,
@@ -145,6 +215,8 @@ const start = async (dev = false) => {
     limiter,
     metrics,
     monitoring,
+    router,
+    token,
     webhooks,
   });
 
@@ -191,21 +263,69 @@ const start = async (dev = false) => {
 };
 
 const buildDocker = async () => {
-  const from = process.argv[3] ?? 'ghcr.io/browserless/multi';
-  const version = process.argv[4] ?? 'latest';
   const finalDockerPath = path.join(cwd, 'build', 'Dockerfile');
+  const argSwitches = getArgSwitches();
+
+  await build();
 
   const dockerContents = (
     await fs.readFile(path.join(__dirname, '..', 'docker', 'sdk', 'Dockerfile'))
   ).toString();
 
-  log(`Creating Dockerfile in "${finalDockerPath}"`);
-  await fs.writeFile(
-    path.join(cwd, 'build', 'Dockerfile'),
-    dockerContents,
-  );
+  log(`Generating Dockerfile at "${finalDockerPath}"`);
+
+  await fs.writeFile(path.join(cwd, 'build', 'Dockerfile'), dockerContents);
+
+  const from =
+    argSwitches.from ||
+    (await prompt(
+      'Which docker image do you want to use (defaults to: ghcr.io/browserless/multi)?',
+    )) ||
+    'ghcr.io/browserless/multi';
+
+  const action =
+    argSwitches.action ||
+    (await prompt(
+      'Do you want to push the image or load it locally (defaults to load)?',
+    )) ||
+    'load';
+
+  const tag =
+    argSwitches.tag ||
+    (await prompt(
+      'What do you want to name the resulting image (eg, my-browserless:latest)?',
+    ));
 
-  log(`Building docker image from repo: "${from}:${version}"`);
+  if (!tag || !tag.includes(':')) {
+    throw new Error(`A name for the image is required with a ":" and version.`);
+  }
+
+  const platformsPrompt =
+    action === 'load'
+      ? `Which platform do you want to build for (defaults to linux/amd64)?`
+      : `Which platforms do you want to build? (defaults to "linux/amd64", must be comma-separated)?`;
+
+  const platforms =
+    argSwitches.platform || (await prompt(platformsPrompt)) || 'linux/amd64';
+
+  if (action === 'load' && platforms.includes(',')) {
+    throw new Error(
+      `When "load" is specified, only one platform can be built due to limitations in buildx.`,
+    );
+  }
+
+  const cmd = `docker buildx build --build-arg FROM=${from} --platform ${platforms} --${action} -f ./build/Dockerfile -t ${tag} .`;
+
+  const proceed =
+    argSwitches.proceed ||
+    (await prompt(`Will execute "${cmd}" Proceed (y/n)?`)) ||
+    'n';
+
+  if (proceed || !proceed.includes('n')) {
+    log(`Starting docker build`);
+    await buildDockerImage(cmd);
+    process.exit(0);
+  }
 };
 
 switch (cmd) {

package/build/browserless.d.ts

@@ -1,29 +1,34 @@
+/// <reference types="debug" />
 /// <reference types="node" />
-import { BrowserManager, Config, FileSystem, HTTPServer, Limiter, Metrics, Monitoring, WebHooks } from '@browserless.io/browserless';
+import { BrowserManager, Config, FileSystem, HTTPServer, Limiter, Metrics, Monitoring, Router, Token, WebHooks } from '@browserless.io/browserless';
 export declare class Browserless {
-    private config;
-    private monitoring;
-    private metrics;
-    private fileSystem;
-    private browserManager;
-    private limiter;
-    private webhooks;
-    private debug;
+    protected debug: debug.Debugger;
+    protected browserManager: BrowserManager;
+    protected config: Config;
+    protected fileSystem: FileSystem;
+    protected limiter: Limiter;
+    protected metrics: Metrics;
+    protected monitoring: Monitoring;
+    protected router: Router;
+    protected token: Token;
+    protected webhooks: WebHooks;
     webSocketRouteFiles: string[];
     httpRouteFiles: string[];
     server?: HTTPServer;
     metricsSaveInterval: number;
     metricsSaveIntervalID?: NodeJS.Timer;
-    constructor({ browserManager, config, monitoring, limiter, metrics, fileSystem, webhooks, }?: {
-        browserManager?: BrowserManager;
-        config?: Config;
-        fileSystem?: FileSystem;
-        limiter?: Limiter;
-        metrics?: Metrics;
-        monitoring?: Monitoring;
-        webhooks?: WebHooks;
+    constructor({ browserManager, config, fileSystem, limiter, metrics, monitoring, router, token, webhooks, }?: {
+        browserManager?: Browserless['browserManager'];
+        config?: Browserless['config'];
+        fileSystem?: Browserless['fileSystem'];
+        limiter?: Browserless['limiter'];
+        metrics?: Browserless['metrics'];
+        monitoring?: Browserless['monitoring'];
+        router?: Browserless['router'];
+        token?: Browserless['token'];
+        webhooks?: Browserless['webhooks'];
     });
-    private saveMetrics;
+    protected saveMetrics: () => Promise<void>;
     setMetricsSaveInterval: (interval: number) => void;
     addHTTPRoute(httpRouteFilePath: string): void;
     addWebSocketRoute(webSocketRouteFilePath: string): void;

package/build/browserless.js

@@ -1,25 +1,28 @@
 import * as path from 'path';
-import { BrowserManager, Config, FileSystem, HTTPServer, Limiter, Metrics, Monitoring, WebHooks, availableBrowsers, createLogger, getRouteFiles, makeExternalURL, printLogo, safeParse, } from '@browserless.io/browserless';
+import { BrowserManager, Config, FileSystem, HTTPServer, Limiter, Metrics, Monitoring, Router, Token, WebHooks, availableBrowsers, createLogger, getRouteFiles, makeExternalURL, printLogo, safeParse, } from '@browserless.io/browserless';
 import { readFile } from 'fs/promises';
 import { userInfo } from 'os';
 const routeSchemas = ['body', 'query'];
 export class Browserless {
+    debug = createLogger('index');
+    browserManager;
     config;
-    monitoring;
-    metrics;
     fileSystem;
-    browserManager;
     limiter;
+    metrics;
+    monitoring;
+    router;
+    token;
     webhooks;
-    debug = createLogger('index');
     webSocketRouteFiles = [];
     httpRouteFiles = [];
     server;
     metricsSaveInterval = 5 * 60 * 1000;
     metricsSaveIntervalID;
-    constructor({ browserManager, config, monitoring, limiter, metrics, fileSystem, webhooks, } = {}) {
+    constructor({ browserManager, config, fileSystem, limiter, metrics, monitoring, router, token, webhooks, } = {}) {
         this.config = config || new Config();
         this.metrics = metrics || new Metrics();
+        this.token = token || new Token(this.config);
         this.webhooks = webhooks || new WebHooks(this.config);
         this.browserManager = browserManager || new BrowserManager(this.config);
         this.monitoring = monitoring || new Monitoring(this.config);
@@ -27,6 +30,8 @@ export class Browserless {
         this.limiter =
             limiter ||
                 new Limiter(this.config, this.metrics, this.monitoring, this.webhooks);
+        this.router =
+            router || new Router(this.config, this.browserManager, this.limiter);
     }
     saveMetrics = async () => {
         const metricsPath = this.config.getMetricsJSONPath();
@@ -101,11 +106,11 @@ export class Browserless {
                 const { default: route } = await import(routeImport + `?cb=${Date.now()}`);
                 route.bodySchema = safeParse(bodySchema);
                 route.querySchema = safeParse(querySchema);
-                route._config = () => this.config;
-                route._metrics = () => this.metrics;
-                route._monitor = () => this.monitoring;
-                route._fileSystem = () => this.fileSystem;
-                route._debug = () => logger;
+                route.getConfig = () => this.config;
+                route.getMetrics = () => this.metrics;
+                route.getMonitoring = () => this.monitoring;
+                route.getFileSystem = () => this.fileSystem;
+                route.getDebug = () => logger;
                 httpRoutes.push(route);
             }
         }
@@ -122,11 +127,11 @@ export class Browserless {
                 const logger = createLogger(`ws:${name}`);
                 const { default: route, } = await import(wsImport + `?cb=${Date.now()}`);
                 route.querySchema = safeParse(querySchema);
-                route._config = () => this.config;
-                route._metrics = () => this.metrics;
-                route._monitor = () => this.monitoring;
-                route._fileSystem = () => this.fileSystem;
-                route._debug = () => logger;
+                route.getConfig = () => this.config;
+                route.getMetrics = () => this.metrics;
+                route.getMonitoring = () => this.monitoring;
+                route.getFileSystem = () => this.fileSystem;
+                route.getDebug = () => logger;
                 wsRoutes.push(route);
             }
         }
@@ -137,8 +142,10 @@ export class Browserless {
                 throw new Error(`Couldn't load route "${route.path}" due to missing browser of "${route.browser.name}"`);
             }
         });
+        httpRoutes.forEach((r) => this.router.registerHTTPRoute(r));
+        wsRoutes.forEach((r) => this.router.registerWebSocketRoute(r));
         this.debug(`Imported and validated all route files, starting up server.`);
-        this.server = new HTTPServer(this.config, this.metrics, this.browserManager, this.limiter, httpRoutes, wsRoutes);
+        this.server = new HTTPServer(this.config, this.metrics, this.token, this.router);
         await this.server.start();
         this.debug(`Starting metrics collection.`);
         this.metricsSaveIntervalID = setInterval(() => this.saveMetrics(), this.metricsSaveInterval);

package/build/browsers/cdp-chromium.d.ts

@@ -1,33 +1,36 @@
+/// <reference types="debug" />
+/// <reference types="node" />
 /// <reference types="node" />
 /// <reference types="node" />
 /// <reference types="node" />
 /// <reference types="node" />
 import { CDPLaunchOptions, Config, Request } from '@browserless.io/browserless';
-import { Browser, Page } from 'puppeteer-core';
+import { Browser, Page, Target } from 'puppeteer-core';
 import { Duplex } from 'stream';
 import { EventEmitter } from 'events';
+import httpProxy from 'http-proxy';
 export declare class CDPChromium extends EventEmitter {
-    private config;
-    private userDataDir;
-    private record;
-    private blockAds;
-    private running;
-    private browser;
-    private browserWSEndpoint;
-    private port?;
-    private debug;
-    private proxy;
+    protected config: Config;
+    protected userDataDir: string | null;
+    protected record: boolean;
+    protected blockAds: boolean;
+    protected running: boolean;
+    protected browser: Browser | null;
+    protected browserWSEndpoint: string | null;
+    protected port?: number;
+    protected debug: import("debug").Debugger;
+    protected proxy: httpProxy<import("http").IncomingMessage, import("http").ServerResponse<import("http").IncomingMessage>>;
     constructor({ userDataDir, config, record, blockAds, }: {
         blockAds: boolean;
         config: Config;
         record: boolean;
         userDataDir: CDPChromium['userDataDir'];
     });
-    private cleanListeners;
-    private setUpEmbeddedAPI;
+    protected cleanListeners(): void;
+    protected setUpEmbeddedAPI: (page: Page, id: string, record: boolean) => Promise<void>;
     getPageId: (page: Page) => string;
     makeLiveURL: (browserId: string, pageId: string) => string;
-    private onTargetCreated;
+    protected onTargetCreated: (target: Target) => Promise<void>;
     isRunning: () => boolean;
     newPage: () => Promise<Page>;
     close: () => Promise<void>;

package/build/browsers/index.d.ts

@@ -1,12 +1,13 @@
-import { BrowserHTTPRoute, BrowserInstance, BrowserWebsocketRoute, BrowserlessSession, BrowserlessSessionJSON, Config, Request } from '@browserless.io/browserless';
+/// <reference types="debug" />
+import { BrowserHTTPRoute, BrowserInstance, BrowserServerOptions, BrowserWebsocketRoute, BrowserlessSession, BrowserlessSessionJSON, CDPLaunchOptions, Config, Request } from '@browserless.io/browserless';
 export declare class BrowserManager {
-    private config;
-    private browsers;
-    private launching;
-    private timers;
-    private debug;
+    protected config: Config;
+    protected browsers: Map<BrowserInstance, BrowserlessSession>;
+    protected launching: Map<string, Promise<unknown>>;
+    protected timers: Map<string, number>;
+    protected debug: import("debug").Debugger;
     constructor(config: Config);
-    private removeUserDataDir;
+    protected removeUserDataDir: (userDataDir: string | null) => Promise<void>;
     /**
      * Generates a directory for the user-data-dir contents to be saved in. Uses
      * the provided sessionId, or creates one when omitted,
@@ -16,10 +17,26 @@ export declare class BrowserManager {
      * @param sessionId The ID of the session
      * @returns Promise<string> of the fully-qualified path of the directory
      */
-    private generateDataDir;
-    private generateSessionJson;
+    protected generateDataDir: (sessionId?: string) => Promise<string>;
+    protected generateSessionJson: (browser: BrowserInstance, session: BrowserlessSession) => {
+        browser: string;
+        browserId: string | undefined;
+        initialConnectURL: string;
+        killURL: string | null;
+        running: boolean;
+        timeAliveMs: number;
+        id: string | null;
+        isTempDataDir: boolean;
+        launchOptions: CDPLaunchOptions | BrowserServerOptions;
+        numbConnected: number;
+        resolver: (val: unknown) => void;
+        routePath: string;
+        startedOn: number;
+        ttl: number;
+        userDataDir: string | null;
+    };
     close: (browser: BrowserInstance, session: BrowserlessSession) => Promise<void>;
-    getAllSessions: (req: Request) => Promise<BrowserlessSessionJSON[]>;
+    getAllSessions: () => Promise<BrowserlessSessionJSON[]>;
     complete: (browser: BrowserInstance) => Promise<void>;
     getBrowserForRequest: (req: Request, router: BrowserHTTPRoute | BrowserWebsocketRoute) => Promise<BrowserInstance>;
     stop: () => Promise<void>;

package/build/browsers/index.js

@@ -1,4 +1,4 @@
-import { BadRequest, CDPChromium, HTTPManagementRoutes, NotFound, ServerError, browserHook, convertIfBase64, createLogger, exists, getTokenFromRequest, id, makeExternalURL, noop, pageHook, parseBooleanParam, } from '@browserless.io/browserless';
+import { BadRequest, CDPChromium, HTTPManagementRoutes, NotFound, ServerError, browserHook, convertIfBase64, createLogger, exists, id, makeExternalURL, noop, pageHook, parseBooleanParam, } from '@browserless.io/browserless';
 import path, { join } from 'path';
 import { deleteAsync } from 'del';
 import { mkdir } from 'fs/promises';
@@ -67,13 +67,8 @@ export class BrowserManager {
         }
         await Promise.all(cleanupACtions.map((a) => a()));
     };
-    getAllSessions = async (req) => {
+    getAllSessions = async () => {
         const sessions = Array.from(this.browsers);
-        const requestToken = getTokenFromRequest(req);
-        const token = this.config.getToken();
-        if (token && !requestToken) {
-            throw new BadRequest(`Couldn't locate your API token`);
-        }
         return sessions.map(([browser, session]) => this.generateSessionJson(browser, session));
     };
     complete = async (browser) => {
@@ -113,11 +108,6 @@ export class BrowserManager {
         catch (err) {
             throw new BadRequest(`Error parsing launch-options: ${err}. Launch options must be a JSON or base64-encoded JSON object`);
         }
-        const requestToken = getTokenFromRequest(req);
-        const token = this.config.getToken();
-        if (token && !requestToken) {
-            throw new ServerError(`Error locating authorization token`);
-        }
         const routerOptions = typeof router.defaultLaunchOptions === 'function'
             ? router.defaultLaunchOptions(req)
             : router.defaultLaunchOptions;

package/build/browsers/playwright-chromium.d.ts

@@ -1,25 +1,28 @@
 /// <reference types="node" />
+/// <reference types="debug" />
+/// <reference types="node" />
 /// <reference types="node" />
 /// <reference types="node" />
 import { BrowserServerOptions, Config, Request } from '@browserless.io/browserless';
 import playwright, { Page } from 'playwright-core';
 import { Duplex } from 'stream';
 import { EventEmitter } from 'events';
+import httpProxy from 'http-proxy';
 export declare class PlaywrightChromium extends EventEmitter {
-    private config;
-    private userDataDir;
-    private record;
-    private running;
-    private proxy;
-    private browser;
-    private browserWSEndpoint;
-    private debug;
+    protected config: Config;
+    protected userDataDir: string | null;
+    protected record: boolean;
+    protected running: boolean;
+    protected proxy: httpProxy<import("http").IncomingMessage, import("http").ServerResponse<import("http").IncomingMessage>>;
+    protected browser: playwright.BrowserServer | null;
+    protected browserWSEndpoint: string | null;
+    protected debug: import("debug").Debugger;
     constructor({ config, userDataDir, record, }: {
         config: Config;
         record: boolean;
         userDataDir: PlaywrightChromium['userDataDir'];
     });
-    private cleanListeners;
+    protected cleanListeners(): void;
     isRunning: () => boolean;
     close: () => Promise<void>;
     pages: () => Promise<[]>;

package/build/browsers/playwright-firefox.d.ts

@@ -1,25 +1,28 @@
 /// <reference types="node" />
+/// <reference types="debug" />
+/// <reference types="node" />
 /// <reference types="node" />
 /// <reference types="node" />
 import { BrowserServerOptions, Config, Request } from '@browserless.io/browserless';
 import playwright, { Page } from 'playwright-core';
 import { Duplex } from 'stream';
 import { EventEmitter } from 'events';
+import httpProxy from 'http-proxy';
 export declare class PlaywrightFirefox extends EventEmitter {
-    private config;
-    private userDataDir;
-    private record;
-    private running;
-    private proxy;
-    private browser;
-    private browserWSEndpoint;
-    private debug;
+    protected config: Config;
+    protected userDataDir: string | null;
+    protected record: boolean;
+    protected running: boolean;
+    protected proxy: httpProxy<import("http").IncomingMessage, import("http").ServerResponse<import("http").IncomingMessage>>;
+    protected browser: playwright.BrowserServer | null;
+    protected browserWSEndpoint: string | null;
+    protected debug: import("debug").Debugger;
     constructor({ config, userDataDir, record, }: {
         config: Config;
         record: boolean;
         userDataDir: PlaywrightFirefox['userDataDir'];
     });
-    private cleanListeners;
+    protected cleanListeners(): void;
     isRunning: () => boolean;
     close: () => Promise<void>;
     pages: () => Promise<[]>;

package/build/browsers/playwright-webkit.d.ts

@@ -1,25 +1,28 @@
 /// <reference types="node" />
+/// <reference types="debug" />
+/// <reference types="node" />
 /// <reference types="node" />
 /// <reference types="node" />
 import { BrowserServerOptions, Config, Request } from '@browserless.io/browserless';
 import playwright, { Page } from 'playwright-core';
 import { Duplex } from 'stream';
 import { EventEmitter } from 'events';
+import httpProxy from 'http-proxy';
 export declare class PlaywrightWebkit extends EventEmitter {
-    private config;
-    private userDataDir;
-    private record;
-    private running;
-    private proxy;
-    private browser;
-    private browserWSEndpoint;
-    private debug;
+    protected config: Config;
+    protected userDataDir: string | null;
+    protected record: boolean;
+    protected running: boolean;
+    protected proxy: httpProxy<import("http").IncomingMessage, import("http").ServerResponse<import("http").IncomingMessage>>;
+    protected browser: playwright.BrowserServer | null;
+    protected browserWSEndpoint: string | null;
+    protected debug: import("debug").Debugger;
     constructor({ config, userDataDir, record, }: {
         config: Config;
         record: boolean;
         userDataDir: PlaywrightWebkit['userDataDir'];
     });
-    private cleanListeners;
+    protected cleanListeners(): void;
     isRunning: () => boolean;
     close: () => Promise<void>;
     pages: () => Promise<[]>;