@browserless.io/browserless 2.4.0-beta-3 → 2.5.0-beta-1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@browserless.io/browserless",
-  "version": "2.4.0-beta-3",
+  "version": "2.5.0-beta-1",
   "license": "SSPL",
   "description": "The browserless platform",
   "author": "browserless.io",
@@ -29,7 +29,7 @@
     "install:dev": "npm run install:browsers",
     "lint": "eslint . --ext .ts --fix",
     "prepack": "npm run build:dev",
-    "prettier": "prettier '{src,functions,scripts,bin,external,bin}/**/*.{js,ts,json}' --log-level error --write",
+    "prettier": "prettier '{src,functions,scripts,bin,external,.github}/**/*.{js,ts,json,yml,yaml,md}' --log-level error --write",
     "test": "cross-env DEBUG=quiet mocha",
     "start": "env-cmd -f .env node build"
   },
@@ -57,12 +57,12 @@
     "http-proxy": "^1.18.1",
     "lighthouse": "^11.1.0",
     "micromatch": "^4.0.4",
-    "playwright-core": "^1.42.1",
-    "puppeteer-core": "^22.3.0",
+    "playwright-core": "1.42.1",
+    "puppeteer-core": "^22.4.1",
     "puppeteer-extra": "^3.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2",
     "queue": "^7.0.0",
-    "systeminformation": "^5.22.0"
+    "systeminformation": "^5.22.1"
   },
   "optionalDependencies": {
     "@types/chai": "^4.3.12",
@@ -71,10 +71,10 @@
     "@types/http-proxy": "^1.17.14",
     "@types/micromatch": "^4.0.6",
     "@types/mocha": "^10.0.6",
-    "@types/node": "^20.11.24",
+    "@types/node": "^20.11.26",
     "@types/sinon": "^17.0.3",
-    "@typescript-eslint/eslint-plugin": "^7.1.1",
-    "@typescript-eslint/parser": "^7.1.1",
+    "@typescript-eslint/eslint-plugin": "^7.2.0",
+    "@typescript-eslint/parser": "^7.2.0",
     "assert": "^2.0.0",
     "chai": "^5.1.0",
     "cross-env": "^7.0.3",
@@ -84,13 +84,13 @@
     "eslint": "^8.57.0",
     "eslint-plugin-typescript-sort-keys": "^3.2.0",
     "extract-zip": "^2.0.1",
-    "marked": "^12.0.0",
+    "marked": "^12.0.1",
     "mocha": "^10.3.0",
     "move-file": "^3.1.0",
     "prettier": "^3.2.5",
     "sinon": "^17.0.1",
     "ts-node": "^10.9.2",
-    "typescript": "^5.3.3",
+    "typescript": "^5.4.2",
     "typescript-json-schema": "^0.63.0"
   },
   "eslintConfig": {

package/src/browserless.ts

@@ -11,6 +11,7 @@ import {
   FirefoxPlaywright,
   HTTPRoute,
   HTTPServer,
+  Hooks,
   IBrowserlessStats,
   Limiter,
   Metrics,
@@ -48,6 +49,7 @@ export class Browserless extends EventEmitter {
   protected browserManager: BrowserManager;
   protected config: Config;
   protected fileSystem: FileSystem;
+  protected hooks: Hooks;
   protected limiter: Limiter;
   protected metrics: Metrics;
   protected monitoring: Monitoring;
@@ -67,6 +69,7 @@ export class Browserless extends EventEmitter {
     browserManager,
     config,
     fileSystem,
+    hooks,
     limiter,
     metrics,
     monitoring,
@@ -77,6 +80,7 @@ export class Browserless extends EventEmitter {
     browserManager?: Browserless['browserManager'];
     config?: Browserless['config'];
     fileSystem?: Browserless['fileSystem'];
+    hooks?: Browserless['hooks'];
     limiter?: Browserless['limiter'];
     metrics?: Browserless['metrics'];
     monitoring?: Browserless['monitoring'];
@@ -88,13 +92,21 @@ export class Browserless extends EventEmitter {
     this.config = config || new Config();
     this.metrics = metrics || new Metrics();
     this.token = token || new Token(this.config);
+    this.hooks = hooks || new Hooks();
     this.webhooks = webhooks || new WebHooks(this.config);
-    this.browserManager = browserManager || new BrowserManager(this.config);
+    this.browserManager =
+      browserManager || new BrowserManager(this.config, this.hooks);
     this.monitoring = monitoring || new Monitoring(this.config);
     this.fileSystem = fileSystem || new FileSystem(this.config);
     this.limiter =
       limiter ||
-      new Limiter(this.config, this.metrics, this.monitoring, this.webhooks);
+      new Limiter(
+        this.config,
+        this.metrics,
+        this.monitoring,
+        this.webhooks,
+        this.hooks,
+      );
     this.router =
       router || new Router(this.config, this.browserManager, this.limiter);
   }
@@ -129,7 +141,11 @@ export class Browserless extends EventEmitter {
 
     if (metricsPath) {
       this.debug(`Saving metrics to "${metricsPath}"`);
-      this.fileSystem.append(metricsPath, JSON.stringify(aggregatedStats));
+      this.fileSystem.append(
+        metricsPath,
+        JSON.stringify(aggregatedStats),
+        false,
+      );
     }
   };
 
@@ -214,8 +230,8 @@ export class Browserless extends EventEmitter {
     this.debug('Starting import of HTTP Routes');
 
     for (const httpRoute of [
-      ...internalHttpRouteFiles,
       ...this.httpRouteFiles,
+      ...internalHttpRouteFiles,
     ]) {
       if (httpRoute.endsWith('js')) {
         const { name } = path.parse(httpRoute);
@@ -264,8 +280,8 @@ export class Browserless extends EventEmitter {
 
     this.debug('Starting import of WebSocket Routes');
     for (const wsRoute of [
-      ...internalWsRouteFiles,
       ...this.webSocketRouteFiles,
+      ...internalWsRouteFiles,
     ]) {
       if (wsRoute.endsWith('js')) {
         const { name } = path.parse(wsRoute);
@@ -350,6 +366,7 @@ export class Browserless extends EventEmitter {
       this.metrics,
       this.token,
       this.router,
+      this.hooks,
     );
 
     await this.server.start();

package/src/browsers/index.ts

@@ -16,19 +16,18 @@ import {
   Config,
   FirefoxPlaywright,
   HTTPManagementRoutes,
+  Hooks,
   NotFound,
   Request,
   ServerError,
   WebkitPlaywright,
   availableBrowsers,
-  browserHook,
   convertIfBase64,
   createLogger,
   exists,
   generateDataDir,
   makeExternalURL,
   noop,
-  pageHook,
   parseBooleanParam,
 } from '@browserless.io/browserless';
 import { deleteAsync } from 'del';
@@ -47,7 +46,10 @@ export class BrowserManager {
     WebkitPlaywright.name,
   ];
 
-  constructor(protected config: Config) {}
+  constructor(
+    protected config: Config,
+    protected hooks: Hooks,
+  ) {}
 
   private browserIsChrome = (b: BrowserInstance) =>
     this.chromeBrowsers.some((chromeBrowser) => b instanceof chromeBrowser);
@@ -64,7 +66,7 @@ export class BrowserManager {
   };
 
   protected onNewPage = async (req: Request, page: unknown) => {
-    await pageHook({ meta: req.parsed, page });
+    await this.hooks.page({ meta: req.parsed, page });
   };
 
   /**
@@ -463,7 +465,7 @@ export class BrowserManager {
     this.browsers.set(browser, connectionMeta);
 
     await browser.launch(launchOptions as object);
-    await browserHook({ browser, meta: req.parsed });
+    await this.hooks.browser({ browser, meta: req.parsed });
 
     browser.on('newPage', async (page) => {
       await this.onNewPage(req, page);

package/src/exports.ts

@@ -10,6 +10,7 @@ export * from './metrics.js';
 export * from './mime-types.js';
 export * from './monitoring.js';
 export * from './router.js';
+export * from './sdk-utils.js';
 export * from './server.js';
 export * from './shim.js';
 export * from './token.js';

package/src/file-system.spec.ts

@@ -1,11 +1,11 @@
-import { Config, FileSystem, sleep } from '@browserless.io/browserless';
+import { Config, FileSystem, noop } from '@browserless.io/browserless';
 import { readFile, unlink } from 'fs/promises';
 import { expect } from 'chai';
 
 const filePath = '/tmp/_browserless_test_fs_';
 
 describe('File-System', () => {
-  afterEach(async () => unlink(filePath));
+  afterEach(async () => unlink(filePath).catch(noop));
 
   it('saves and encodes files', async () => {
     const mySecretContents = 'pony-foo';
@@ -13,46 +13,71 @@ describe('File-System', () => {
     config.setToken('browserless.io');
     const f = new FileSystem(config);
 
-    await f.append(filePath, mySecretContents);
+    await f.append(filePath, mySecretContents, true);
 
-    expect(await f.read(filePath)).to.eql([mySecretContents]);
+    expect(await f.read(filePath, true)).to.eql([mySecretContents]);
     const rawText = (await readFile(filePath)).toString();
 
     expect(rawText.toString()).to.not.include(mySecretContents);
   });
 
-  it('appends newlines to files', async () => {
+  it('saves files without encoding', async () => {
+    const mySecretContents = 'pony-foo';
+    const config = new Config();
+    config.setToken('browserless.io');
+    const f = new FileSystem(config);
+
+    await f.append(filePath, mySecretContents, false);
+
+    expect(await f.read(filePath, false)).to.eql([mySecretContents]);
+    const rawText = (await readFile(filePath)).toString();
+
+    expect(rawText.toString()).to.include(mySecretContents);
+  });
+
+  it('appends newlines to files and encodes them', async () => {
     const mySecretContents = 'pony-foo';
     const moreSecretContents = 'pony-pony-foo-foo';
     const config = new Config();
     config.setToken('browserless.io');
     const f = new FileSystem(config);
 
-    await f.append(filePath, mySecretContents);
+    await f.append(filePath, mySecretContents, true);
 
-    expect(await f.read(filePath)).to.eql([mySecretContents]);
+    expect(await f.read(filePath, true)).to.eql([mySecretContents]);
 
-    await f.append(filePath, moreSecretContents);
+    await f.append(filePath, moreSecretContents, true);
 
-    expect(await f.read(filePath)).to.eql([
+    expect(await f.read(filePath, true)).to.eql([
       mySecretContents,
       moreSecretContents,
     ]);
+    const rawText = (await readFile(filePath)).toString();
+
+    expect(rawText).to.not.include(mySecretContents);
+    expect(rawText).to.not.include(moreSecretContents);
   });
 
-  it('re-encodes files on token changes', async () => {
+  it('appends newlines to files and does not encode them', async () => {
+    const mySecretContents = 'pony-foo';
+    const moreSecretContents = 'pony-pony-foo-foo';
     const config = new Config();
     config.setToken('browserless.io');
     const f = new FileSystem(config);
-    const mySecretContents = 'pony-foo';
 
-    await f.append(filePath, mySecretContents);
-    const oldText = (await readFile(filePath)).toString();
-    config.setToken('super-browserless-64');
-    await sleep(200);
-    const newText = (await readFile(filePath)).toString();
+    await f.append(filePath, mySecretContents, false);
+
+    expect(await f.read(filePath, false)).to.eql([mySecretContents]);
+
+    await f.append(filePath, moreSecretContents, false);
+
+    expect(await f.read(filePath, false)).to.eql([
+      mySecretContents,
+      moreSecretContents,
+    ]);
+    const rawText = (await readFile(filePath)).toString();
 
-    expect(oldText).to.not.equal(newText);
-    expect(await f.read(filePath)).to.eql([mySecretContents]);
+    expect(rawText).to.include(mySecretContents);
+    expect(rawText).to.include(moreSecretContents);
   });
 });

package/src/file-system.ts

@@ -15,28 +15,8 @@ export class FileSystem extends EventEmitter {
   constructor(protected config: Config) {
     super();
     this.currentAESKey = config.getAESKey();
-    this.config.on('token', this.handleTokenChange);
   }
 
-  private handleTokenChange = async () => {
-    this.log(`Token has changed, updating file-system contents`);
-    const start = Date.now();
-    const newAESKey = this.config.getAESKey();
-    await Promise.all(
-      Array.from(this.fsMap).map(async ([filePath, contents]) => {
-        const newlyEncoded = encrypt(
-          contents.join('\n'),
-          Buffer.from(newAESKey),
-        );
-        return writeFile(filePath, newlyEncoded);
-      }),
-    ).catch((e) => {
-      this.log(`Error in setting new token: "${e}"`);
-    });
-    this.log(`Successfully updated file encodings in ${Date.now() - start}ms`);
-    this.currentAESKey = this.config.getAESKey();
-  };
-
   /**
    * Appends contents to a file-path for persistance. File contents are
    * encrypted before being saved to disk. Reads happen via the in-memory
@@ -46,15 +26,19 @@ export class FileSystem extends EventEmitter {
    * @param newContent A string of new content to add to the file
    * @returns void
    */
-  append = async (path: string, newContent: string): Promise<void> => {
-    const contents = await this.read(path);
+  append = async (
+    path: string,
+    newContent: string,
+    shouldEncode: boolean,
+  ): Promise<void> => {
+    const contents = await this.read(path, shouldEncode);
 
     contents.push(newContent);
     this.fsMap.set(path, contents);
-    const encoded = await encrypt(
-      contents.join('\n'),
-      Buffer.from(this.currentAESKey),
-    );
+
+    const encoded = shouldEncode
+      ? await encrypt(contents.join('\n'), this.currentAESKey)
+      : contents.join('\n');
 
     return writeFile(path, encoded.toString());
   };
@@ -66,16 +50,18 @@ export class FileSystem extends EventEmitter {
    * @param path The filepath of the contents to read
    * @returns Promise of the contents separated by newlines
    */
-  read = async (path: string): Promise<string[]> => {
+  read = async (path: string, encoded: boolean): Promise<string[]> => {
     const hasKey = this.fsMap.has(path);
 
     if (hasKey) {
       return this.fsMap.get(path) as string[];
     }
     const contents = (await readFile(path).catch(() => '')).toString();
-    const splitContents = contents.length
-      ? (await decrypt(contents, Buffer.from(this.currentAESKey))).split('\n')
-      : [];
+    const decoded =
+      encoded && contents.length
+        ? await decrypt(contents, this.currentAESKey)
+        : contents;
+    const splitContents = decoded.length ? decoded.split('\n') : [];
 
     this.fsMap.set(path, splitContents);
 

package/src/hooks.ts

@@ -1,8 +1,32 @@
-// @ts-ignore
-export { default as beforeRequest } from '../external/before.js';
-// @ts-ignore
-export { default as afterRequest } from '../external/after.js';
-// @ts-ignore
-export { default as pageHook } from '../external/page.js';
-// @ts-ignore
-export { default as browserHook } from '../external/browser.js';
+// @ts-nocheck Unknown external files
+import { EventEmitter } from 'events';
+
+// KEPT for backwards compatibility reasons since some downstream
+// docker images will override these files to inject their own hook
+// behaviors
+import { default as afterRequest } from '../external/after.js';
+import { default as beforeRequest } from '../external/before.js';
+import { default as browserHook } from '../external/browser.js';
+import { default as pageHook } from '../external/page.js';
+
+export class Hooks extends EventEmitter {
+  constructor() {
+    super();
+  }
+
+  before(...args: unknown[]): Promise<boolean> {
+    return beforeRequest(...args);
+  }
+
+  after(...args: unknown[]): Promise<boolean> {
+    return afterRequest(...args);
+  }
+
+  page(...args: unknown[]): Promise<unknown> {
+    return pageHook(...args);
+  }
+
+  browser(...args: unknown[]): Promise<unknown> {
+    return browserHook(...args);
+  }
+}