@browserless.io/browserless 2.0.0-beta-5 → 2.0.0-beta-7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@browserless.io/browserless",
-  "version": "2.0.0-beta-5",
+  "version": "2.0.0-beta-7",
   "license": "SSPL",
   "description": "The browserless platform",
   "author": "browserless.io",
@@ -29,7 +29,7 @@
     "install:dev": "npm run install:browsers && npm run install:cdp-json",
     "lint": "eslint . --ext .ts --fix",
     "prepack": "npm run build:dev",
-    "prettier": "prettier '{src,functions,scripts,bin,external}/**/*.{js,ts,json}' --log-level error --write",
+    "prettier": "prettier '{src,functions,scripts,bin,external,bin}/**/*.{js,ts,json}' --log-level error --write",
     "test": "cross-env DEBUG=quiet mocha"
   },
   "files": [
@@ -71,15 +71,15 @@
     "@types/http-proxy": "^1.17.14",
     "@types/micromatch": "^4.0.6",
     "@types/mocha": "^10.0.6",
-    "@types/node": "^20.10.5",
+    "@types/node": "^20.10.8",
     "@types/sinon": "^17.0.2",
-    "@typescript-eslint/eslint-plugin": "^6.15.0",
-    "@typescript-eslint/parser": "^6.15.0",
+    "@typescript-eslint/eslint-plugin": "^6.18.1",
+    "@typescript-eslint/parser": "^6.18.1",
     "assert": "^2.0.0",
-    "chai": "^4.3.6",
+    "chai": "^5.0.0",
     "cross-env": "^7.0.3",
     "env-cmd": "^10.1.0",
-    "esbuild": "^0.19.10",
+    "esbuild": "^0.19.11",
     "esbuild-plugin-polyfill-node": "^0.3.0",
     "eslint": "^8.56.0",
     "eslint-plugin-import": "^2.29.1",
@@ -158,18 +158,6 @@
     "timeout": 30000,
     "slow": 5000
   },
-  "nodemonConfig": {
-    "ignoreRoot": [
-      ".git",
-      ".no-git"
-    ],
-    "watch": [
-      "src"
-    ],
-    "exec": "npx tsc && node ./build/index.js",
-    "ext": "ts json",
-    "signal": "SIGTERM"
-  },
   "prettier": {
     "semi": true,
     "trailingComma": "all",

package/scripts/build-open-api.js

@@ -56,17 +56,19 @@ const buildOpenAPI = async (
   const changelog = marked.parse(
     (await fs.readFile('CHANGELOG.md').catch(() => '')).toString(),
   );
+
   const [httpRoutes, wsRoutes] = await getRouteFiles(new Config());
   const swaggerJSON = {
     customSiteTitle: 'Browserless Documentation',
     definitions: {},
     info: {
-      description: readme + changelog,
+      // Concatenation necessary for Changelog to show up in sidebar
+      description: readme + `\n# Changelog\n` + changelog,
       title: 'Browserless',
       version: JSON.parse(packageJSON.toString()).version,
       'x-logo': {
         altText: 'browserless logo',
-        url: './docs/browserless-logo.png',
+        url: './docs/browserless-logo-inline.svg',
       },
     },
     openapi: '3.0.0',
@@ -86,10 +88,11 @@ const buildOpenAPI = async (
       .sort()
       .map(async (routeModule) => {
         const routeImport = `${isWin ? 'file:///' : ''}${routeModule}`;
-        const { default: route } = await import(routeImport);
-        if (!route) {
+        const { default: Route } = await import(routeImport);
+        if (!Route) {
           throw new Error(`Invalid route file to import docs ${routeModule}`);
         }
+        const route = new Route();
         const { name } = parse(routeModule);
         const body = routeModule.replace('.js', '.body.json');
         const query = routeModule.replace('.js', '.query.json');

package/src/browserless.ts

@@ -27,6 +27,10 @@ import { userInfo } from 'os';
 
 const routeSchemas = ['body', 'query'];
 
+type Implements<T> = {
+  new (...args: unknown[]): T;
+};
+
 export class Browserless {
   protected debug: debug.Debugger = createLogger('index');
   protected browserManager: BrowserManager;
@@ -181,16 +185,25 @@ export class Browserless {
           this.config.getIsWin() ? 'file:///' : ''
         }${httpRoute}`;
         const logger = createLogger(`http:${name}`);
-        const { default: route }: { default: HTTPRoute | BrowserHTTPRoute } =
+        const {
+          default: Route,
+        }: { default: Implements<HTTPRoute> | Implements<BrowserHTTPRoute> } =
           await import(routeImport + `?cb=${Date.now()}`);
-
+        const route = new Route(
+          this.browserManager,
+          this.config,
+          this.fileSystem,
+          logger,
+          this.metrics,
+          this.monitoring,
+        );
         route.bodySchema = safeParse(bodySchema);
         route.querySchema = safeParse(querySchema);
-        route.getConfig = () => this.config;
-        route.getMetrics = () => this.metrics;
-        route.getMonitoring = () => this.monitoring;
-        route.getFileSystem = () => this.fileSystem;
-        route.getDebug = () => logger;
+        route.config = () => this.config;
+        route.metrics = () => this.metrics;
+        route.monitoring = () => this.monitoring;
+        route.fileSystem = () => this.fileSystem;
+        route.debug = () => logger;
 
         httpRoutes.push(route);
       }
@@ -215,17 +228,27 @@ export class Browserless {
         }${wsRoute}`;
         const logger = createLogger(`ws:${name}`);
         const {
-          default: route,
-        }: { default: WebSocketRoute | BrowserWebsocketRoute } = await import(
-          wsImport + `?cb=${Date.now()}`
-        );
+          default: Route,
+        }: {
+          default:
+            | Implements<WebSocketRoute>
+            | Implements<BrowserWebsocketRoute>;
+        } = await import(wsImport + `?cb=${Date.now()}`);
 
+        const route = new Route(
+          this.browserManager,
+          this.config,
+          this.fileSystem,
+          logger,
+          this.metrics,
+          this.monitoring,
+        );
         route.querySchema = safeParse(querySchema);
-        route.getConfig = () => this.config;
-        route.getMetrics = () => this.metrics;
-        route.getMonitoring = () => this.monitoring;
-        route.getFileSystem = () => this.fileSystem;
-        route.getDebug = () => logger;
+        route.config = () => this.config;
+        route.metrics = () => this.metrics;
+        route.monitoring = () => this.monitoring;
+        route.fileSystem = () => this.fileSystem;
+        route.debug = () => logger;
 
         wsRoutes.push(route);
       }
@@ -234,11 +257,12 @@ export class Browserless {
     // Validate that browsers are installed and route paths are unique
     [...httpRoutes, ...wsRoutes].forEach((route) => {
       if (
+        'browser' in route &&
         route.browser &&
-        !installedBrowsers.some((b) => b.name === route.browser.name)
+        !installedBrowsers.some((b) => b.name === route.browser?.name)
       ) {
         throw new Error(
-          `Couldn't load route "${route.path}" due to missing browser of "${route.browser.name}"`,
+          `Couldn't load route "${route.path}" due to missing browser of "${route.browser?.name}"`,
         );
       }
     });

package/src/browsers/index.ts

@@ -82,27 +82,52 @@ export class BrowserManager {
     return dataDirPath;
   };
 
-  protected generateSessionJson = (
+  private generateSessionJson = async (
     browser: BrowserInstance,
     session: BrowserlessSession,
   ) => {
     const serverAddress = this.config.getExternalAddress();
 
-    return {
-      ...session,
-      browser: browser.constructor.name,
-      browserId: browser.wsEndpoint()?.split('/').pop(),
-      initialConnectURL: new URL(session.initialConnectURL, serverAddress).href,
-      killURL: session.id
-        ? makeExternalURL(
-            serverAddress,
-            HTTPManagementRoutes.sessions,
-            session.id,
-          )
-        : null,
-      running: browser.isRunning(),
-      timeAliveMs: Date.now() - session.startedOn,
-    };
+    const sessions = [
+      {
+        ...session,
+        browser: browser.constructor.name,
+        browserId: browser.wsEndpoint()?.split('/').pop(),
+        initialConnectURL: new URL(session.initialConnectURL, serverAddress)
+          .href,
+        killURL: session.id
+          ? makeExternalURL(
+              serverAddress,
+              HTTPManagementRoutes.sessions,
+              session.id,
+            )
+          : null,
+        running: browser.isRunning(),
+        timeAliveMs: Date.now() - session.startedOn,
+        type: 'browser',
+      },
+    ];
+
+    const wsEndpoint = browser.wsEndpoint();
+    if (browser.constructor.name === 'CDPChromium' && wsEndpoint) {
+      const port = new URL(wsEndpoint).port;
+      const response = await fetch(`http://127.0.0.1:${port}/json/list`, {
+        headers: {
+          Host: '127.0.0.1',
+        },
+      });
+      if (response.ok) {
+        const body = await response.json();
+        for (const page of body) {
+          sessions.push({
+            ...sessions[0],
+            ...page,
+            browserWSEndpoint: wsEndpoint,
+          });
+        }
+      }
+    }
+    return sessions;
   };
 
   public close = async (
@@ -129,9 +154,12 @@ export class BrowserManager {
   public getAllSessions = async (): Promise<BrowserlessSessionJSON[]> => {
     const sessions = Array.from(this.browsers);
 
-    return sessions.map(([browser, session]) =>
-      this.generateSessionJson(browser, session),
-    );
+    const formattedSessions: BrowserlessSessionJSON[] = [];
+    for (const [browser, session] of sessions) {
+      const formattedSession = await this.generateSessionJson(browser, session);
+      formattedSessions.push(...formattedSession);
+    }
+    return formattedSessions;
   };
 
   public complete = async (browser: BrowserInstance): Promise<void> => {
@@ -208,7 +236,7 @@ export class BrowserManager {
     const manualUserDataDir =
       launchOptions.args
         ?.find((arg) => arg.includes('--user-data-dir='))
-        ?.split('=')[2] || (launchOptions as CDPLaunchOptions).userDataDir;
+        ?.split('=')[1] || (launchOptions as CDPLaunchOptions).userDataDir;
 
     // Always specify a user-data-dir since plugins can "inject" their own
     // unless it's playwright which takes care of its own data-dirs

package/src/file-system.spec.ts

@@ -0,0 +1,58 @@
+import { Config, FileSystem, sleep } from '@browserless.io/browserless';
+import { readFile, unlink } from 'fs/promises';
+import { expect } from 'chai';
+
+const filePath = '/tmp/_browserless_test_fs_';
+
+describe.only('File-System', () => {
+  afterEach(async () => unlink(filePath));
+
+  it('saves and encodes files', async () => {
+    const mySecretContents = 'pony-foo';
+    const config = new Config();
+    config.setToken('browserless.io');
+    const f = new FileSystem(config);
+
+    await f.append(filePath, mySecretContents);
+
+    expect(await f.read(filePath)).to.eql([mySecretContents]);
+    const rawText = (await readFile(filePath)).toString();
+
+    expect(rawText.toString()).to.not.include(mySecretContents);
+  });
+
+  it('appends newlines to files', async () => {
+    const mySecretContents = 'pony-foo';
+    const moreSecretContents = 'pony-pony-foo-foo';
+    const config = new Config();
+    config.setToken('browserless.io');
+    const f = new FileSystem(config);
+
+    await f.append(filePath, mySecretContents);
+
+    expect(await f.read(filePath)).to.eql([mySecretContents]);
+
+    await f.append(filePath, moreSecretContents);
+
+    expect(await f.read(filePath)).to.eql([
+      mySecretContents,
+      moreSecretContents,
+    ]);
+  });
+
+  it('re-encodes files on token changes', async () => {
+    const config = new Config();
+    config.setToken('browserless.io');
+    const f = new FileSystem(config);
+    const mySecretContents = 'pony-foo';
+
+    await f.append(filePath, mySecretContents);
+    const oldText = (await readFile(filePath)).toString();
+    config.setToken('super-browserless-64');
+    await sleep(200);
+    const newText = (await readFile(filePath)).toString();
+
+    expect(oldText).to.not.equal(newText);
+    expect(await f.read(filePath)).to.eql([mySecretContents]);
+  });
+});

package/src/file-system.ts

@@ -1,10 +1,39 @@
-import { Config, decrypt, encrypt } from '@browserless.io/browserless';
+import {
+  Config,
+  createLogger,
+  decrypt,
+  encrypt,
+} from '@browserless.io/browserless';
 import { readFile, writeFile } from 'fs/promises';
 
 export class FileSystem {
   protected fsMap: Map<string, string[]> = new Map();
+  protected currentAESKey: Buffer;
+  protected log = createLogger('file-system');
 
-  constructor(protected config: Config) {}
+  constructor(protected config: Config) {
+    this.currentAESKey = config.getAESKey();
+    this.config.on('token', this.handleTokenChange);
+  }
+
+  private handleTokenChange = async () => {
+    this.log(`Token has changed, updating file-system contents`);
+    const start = Date.now();
+    const newAESKey = this.config.getAESKey();
+    await Promise.all(
+      Array.from(this.fsMap).map(async ([filePath, contents]) => {
+        const newlyEncoded = encrypt(
+          contents.join('\n'),
+          Buffer.from(newAESKey),
+        );
+        return writeFile(filePath, newlyEncoded);
+      }),
+    ).catch((e) => {
+      this.log(`Error in setting new token: "${e}"`);
+    });
+    this.log(`Successfully updated file encodings in ${Date.now() - start}ms`);
+    this.currentAESKey = this.config.getAESKey();
+  };
 
   /**
    * Appends contents to a file-path for persistance. File contents are
@@ -20,9 +49,10 @@ export class FileSystem {
 
     contents.push(newContent);
     this.fsMap.set(path, contents);
-
-    const key = this.config.getAESKey();
-    const encoded = await encrypt(contents.join('\n'), Buffer.from(key));
+    const encoded = await encrypt(
+      contents.join('\n'),
+      Buffer.from(this.currentAESKey),
+    );
 
     return writeFile(path, encoded.toString());
   };
@@ -40,11 +70,9 @@ export class FileSystem {
     if (hasKey) {
       return this.fsMap.get(path) as string[];
     }
-
-    const key = this.config.getAESKey();
     const contents = (await readFile(path).catch(() => '')).toString();
     const splitContents = contents.length
-      ? (await decrypt(contents, Buffer.from(key))).split('\n')
+      ? (await decrypt(contents, Buffer.from(this.currentAESKey))).split('\n')
       : [];
 
     this.fsMap.set(path, splitContents);

package/src/http.ts

@@ -107,9 +107,9 @@ export enum HTTPManagementRoutes {
 }
 
 export enum APITags {
-  'browserAPI' = 'Browser APIs',
-  'browserWS' = 'Browser WebSockets',
-  'management' = 'Management APIs',
+  'browserAPI' = 'Browser REST APIs',
+  'browserWS' = 'Browser WebSocket APIs',
+  'management' = 'Management REST APIs',
 }
 
 export interface Request extends http.IncomingMessage {

package/src/router.ts

@@ -67,7 +67,7 @@ export class Router {
         return Promise.resolve();
       }
 
-      if (route.browser) {
+      if ('browser' in route && route.browser) {
         const browser = await this.browserManager.getBrowserForRequest(
           req,
           route,
@@ -111,7 +111,7 @@ export class Router {
         return Promise.resolve();
       }
 
-      if (route.browser) {
+      if ('browser' in route && route.browser) {
         const browser = await this.browserManager.getBrowserForRequest(
           req,
           route,
@@ -146,8 +146,6 @@ export class Router {
       `Registering HTTP ${route.method.toUpperCase()} ${route.path}`,
     );
 
-    route.getBrowserManager = () => this.browserManager;
-
     const bound = route.handler.bind(route);
     const wrapped = this.wrapHTTPHandler(route, bound);
 
@@ -170,8 +168,6 @@ export class Router {
   ): WebSocketRoute | BrowserWebsocketRoute {
     this.verbose(`Registering WebSocket "${route.path}"`);
 
-    route.getBrowserManager = () => this.browserManager;
-
     const bound = route.handler.bind(route);
     const wrapped = this.wrapWebSocketHandler(route, bound);
 

package/src/routes/chromium/http/content-post.ts

@@ -61,15 +61,17 @@ export type QuerySchema = SystemQueryParameters & {
   launch?: CDPLaunchOptions | string;
 };
 
-const route: BrowserHTTPRoute = {
-  accepts: [contentTypes.json],
-  auth: true,
-  browser: CDPChromium,
-  concurrency: true,
-  contentTypes: [contentTypes.html],
-  description: `A JSON-based API. Given a "url" or "html" field, runs and returns HTML content after the page has loaded and JavaScript has parsed.`,
-
-  handler: async (
+export default class ContentPostRoute extends BrowserHTTPRoute {
+  accepts = [contentTypes.json];
+  auth = true;
+  browser = CDPChromium;
+  concurrency = true;
+  contentTypes = [contentTypes.html];
+  description = `A JSON-based API. Given a "url" or "html" field, runs and returns HTML content after the page has loaded and JavaScript has parsed.`;
+  method = Methods.post;
+  path = HTTPRoutes.content;
+  tags = [APITags.browserAPI];
+  handler = async (
     req: Request,
     res: ServerResponse,
     browser: BrowserInstance,
@@ -229,10 +231,5 @@ const route: BrowserHTTPRoute = {
     page.close().catch(noop);
 
     return writeResponse(res, 200, markup, contentTypes.html);
-  },
-  method: Methods.post,
-  path: HTTPRoutes.content,
-  tags: [APITags.browserAPI],
-};
-
-export default route;
+  };
+}

package/src/routes/chromium/http/download-post.ts

@@ -8,7 +8,6 @@ import {
   Methods,
   NotFound,
   Request,
-  ServerError,
   SystemQueryParameters,
   contentTypes,
   dedent,
@@ -41,35 +40,30 @@ export interface QuerySchema extends SystemQueryParameters {
  */
 export type ResponseSchema = unknown;
 
-const route: BrowserHTTPRoute = {
-  accepts: [contentTypes.json, contentTypes.javascript],
-  auth: true,
-  browser: CDPChromium,
-  concurrency: true,
-  contentTypes: [contentTypes.any],
-  description: dedent(`
+export default class DownloadPost extends BrowserHTTPRoute {
+  accepts = [contentTypes.json, contentTypes.javascript];
+  auth = true;
+  browser = CDPChromium;
+  concurrency = true;
+  contentTypes = [contentTypes.any];
+  description = dedent(`
   A JSON or JavaScript content-type API for returning files Chrome has downloaded during
   the execution of puppeteer code, which is ran inside context of the browser.
   Browserless sets up a blank page, a fresh download directory, injects your puppeteer code, and then executes it.
   You can load external libraries via the "import" syntax, and import ESM-style modules
   that are written for execution inside of the browser. Once your script is finished, any
-  downloaded files from Chromium are returned back with the appropriate content-type header.`),
-  handler: async (
+  downloaded files from Chromium are returned back with the appropriate content-type header.`);
+  method = Methods.post;
+  path = HTTPRoutes.download;
+  tags = [APITags.browserAPI];
+  handler = async (
     req: Request,
     res: ServerResponse,
     browser: BrowserInstance,
   ): Promise<void> =>
     new Promise(async (resolve, reject) => {
-      const { getConfig: getConfig, getDebug: getDebug } = route;
-
-      if (!getConfig || !getDebug) {
-        return reject(
-          new ServerError(`Couldn't load configuration for request`),
-        );
-      }
-
-      const debug = getDebug();
-      const config = getConfig();
+      const debug = this.debug();
+      const config = this.config();
       const downloadPath = path.join(
         await config.getDownloadsDir(),
         `.browserless.download.${id()}`,
@@ -153,10 +147,5 @@ const route: BrowserHTTPRoute = {
           return resolve();
         })
         .pipe(res);
-    }),
-  method: Methods.post,
-  path: HTTPRoutes.download,
-  tags: [APITags.browserAPI],
-};
-
-export default route;
+    });
+}

package/src/routes/chromium/http/function-post.ts

@@ -8,7 +8,6 @@ import {
   HTTPRoutes,
   Methods,
   Request,
-  ServerError,
   SystemQueryParameters,
   contentTypes,
   dedent,
@@ -38,31 +37,28 @@ export interface QuerySchema extends SystemQueryParameters {
  */
 export type ResponseSchema = unknown;
 
-const route: BrowserHTTPRoute = {
-  accepts: [contentTypes.json, contentTypes.javascript],
-  auth: true,
-  browser: CDPChromium,
-  concurrency: true,
-  contentTypes: [contentTypes.any],
-  description: dedent(`
+export default class FunctionPost extends BrowserHTTPRoute {
+  accepts = [contentTypes.json, contentTypes.javascript];
+  auth = true;
+  browser = CDPChromium;
+  concurrency = true;
+  contentTypes = [contentTypes.any];
+  description = dedent(`
   A JSON or JavaScript content-type API for running puppeteer code in the browser's context.
   Browserless sets up a blank page, injects your puppeteer code, and runs it.
   You can optionally load external libraries via the "import" module that are meant for browser usage.
   Values returned from the function are checked and an appropriate content-type and response is sent back
-  to your HTTP call.`),
-  handler: async (
+  to your HTTP call.`);
+  method = Methods.post;
+  path = HTTPRoutes.function;
+  tags = [APITags.browserAPI];
+  handler = async (
     req: Request,
     res: ServerResponse,
     browser: BrowserInstance,
   ): Promise<void> => {
-    const { getConfig: getConfig, getDebug: getDebug } = route;
-
-    if (!getConfig || !getDebug) {
-      throw new ServerError(`Couldn't load configuration for request`);
-    }
-
-    const debug = getDebug();
-    const config = getConfig();
+    const debug = this.debug();
+    const config = this.config();
     const handler = functionHandler(config, debug);
     const { contentType, payload, page } = await handler(req, browser);
 
@@ -89,10 +85,5 @@ const route: BrowserHTTPRoute = {
     }
 
     return;
-  },
-  method: Methods.post,
-  path: HTTPRoutes.function,
-  tags: [APITags.browserAPI],
-};
-
-export default route;
+  };
+}