@browserless.io/browserless 2.4.0 → 2.5.0-beta-2

package/src/browserless.ts

@@ -11,6 +11,7 @@ import {
   FirefoxPlaywright,
   HTTPRoute,
   HTTPServer,
+  Hooks,
   IBrowserlessStats,
   Limiter,
   Metrics,
@@ -48,6 +49,7 @@ export class Browserless extends EventEmitter {
   protected browserManager: BrowserManager;
   protected config: Config;
   protected fileSystem: FileSystem;
+  protected hooks: Hooks;
   protected limiter: Limiter;
   protected metrics: Metrics;
   protected monitoring: Monitoring;
@@ -67,6 +69,7 @@ export class Browserless extends EventEmitter {
     browserManager,
     config,
     fileSystem,
+    hooks,
     limiter,
     metrics,
     monitoring,
@@ -77,6 +80,7 @@ export class Browserless extends EventEmitter {
     browserManager?: Browserless['browserManager'];
     config?: Browserless['config'];
     fileSystem?: Browserless['fileSystem'];
+    hooks?: Browserless['hooks'];
     limiter?: Browserless['limiter'];
     metrics?: Browserless['metrics'];
     monitoring?: Browserless['monitoring'];
@@ -88,13 +92,21 @@ export class Browserless extends EventEmitter {
     this.config = config || new Config();
     this.metrics = metrics || new Metrics();
     this.token = token || new Token(this.config);
+    this.hooks = hooks || new Hooks();
     this.webhooks = webhooks || new WebHooks(this.config);
-    this.browserManager = browserManager || new BrowserManager(this.config);
+    this.browserManager =
+      browserManager || new BrowserManager(this.config, this.hooks);
     this.monitoring = monitoring || new Monitoring(this.config);
     this.fileSystem = fileSystem || new FileSystem(this.config);
     this.limiter =
       limiter ||
-      new Limiter(this.config, this.metrics, this.monitoring, this.webhooks);
+      new Limiter(
+        this.config,
+        this.metrics,
+        this.monitoring,
+        this.webhooks,
+        this.hooks,
+      );
     this.router =
       router || new Router(this.config, this.browserManager, this.limiter);
   }
@@ -129,7 +141,11 @@ export class Browserless extends EventEmitter {
 
     if (metricsPath) {
       this.debug(`Saving metrics to "${metricsPath}"`);
-      this.fileSystem.append(metricsPath, JSON.stringify(aggregatedStats));
+      this.fileSystem.append(
+        metricsPath,
+        JSON.stringify(aggregatedStats),
+        false,
+      );
     }
   };
 
@@ -190,6 +206,7 @@ export class Browserless extends EventEmitter {
       this.router.shutdown(),
       this.token.shutdown(),
       this.webhooks.shutdown(),
+      this.hooks.shutdown(),
     ]);
   }
 
@@ -350,6 +367,7 @@ export class Browserless extends EventEmitter {
       this.metrics,
       this.token,
       this.router,
+      this.hooks,
     );
 
     await this.server.start();

package/src/browsers/index.ts

@@ -16,21 +16,21 @@ import {
   Config,
   FirefoxPlaywright,
   HTTPManagementRoutes,
+  Hooks,
   NotFound,
   Request,
   ServerError,
   WebkitPlaywright,
   availableBrowsers,
-  browserHook,
   convertIfBase64,
   createLogger,
   exists,
   generateDataDir,
   makeExternalURL,
   noop,
-  pageHook,
   parseBooleanParam,
 } from '@browserless.io/browserless';
+import { Page } from 'puppeteer-core';
 import { deleteAsync } from 'del';
 import path from 'path';
 
@@ -47,7 +47,10 @@ export class BrowserManager {
     WebkitPlaywright.name,
   ];
 
-  constructor(protected config: Config) {}
+  constructor(
+    protected config: Config,
+    protected hooks: Hooks,
+  ) {}
 
   private browserIsChrome = (b: BrowserInstance) =>
     this.chromeBrowsers.some((chromeBrowser) => b instanceof chromeBrowser);
@@ -63,8 +66,8 @@ export class BrowserManager {
     }
   };
 
-  protected onNewPage = async (req: Request, page: unknown) => {
-    await pageHook({ meta: req.parsed, page });
+  protected onNewPage = async (req: Request, page: Page) => {
+    await this.hooks.page({ meta: req.parsed, page });
   };
 
   /**
@@ -463,7 +466,7 @@ export class BrowserManager {
     this.browsers.set(browser, connectionMeta);
 
     await browser.launch(launchOptions as object);
-    await browserHook({ browser, meta: req.parsed });
+    await this.hooks.browser({ browser, meta: req.parsed });
 
     browser.on('newPage', async (page) => {
       await this.onNewPage(req, page);

package/src/exports.ts

@@ -10,6 +10,7 @@ export * from './metrics.js';
 export * from './mime-types.js';
 export * from './monitoring.js';
 export * from './router.js';
+export * from './sdk-utils.js';
 export * from './server.js';
 export * from './shim.js';
 export * from './token.js';

package/src/file-system.spec.ts

@@ -1,11 +1,11 @@
-import { Config, FileSystem, sleep } from '@browserless.io/browserless';
+import { Config, FileSystem, noop } from '@browserless.io/browserless';
 import { readFile, unlink } from 'fs/promises';
 import { expect } from 'chai';
 
 const filePath = '/tmp/_browserless_test_fs_';
 
 describe('File-System', () => {
-  afterEach(async () => unlink(filePath));
+  afterEach(async () => unlink(filePath).catch(noop));
 
   it('saves and encodes files', async () => {
     const mySecretContents = 'pony-foo';
@@ -13,46 +13,71 @@ describe('File-System', () => {
     config.setToken('browserless.io');
     const f = new FileSystem(config);
 
-    await f.append(filePath, mySecretContents);
+    await f.append(filePath, mySecretContents, true);
 
-    expect(await f.read(filePath)).to.eql([mySecretContents]);
+    expect(await f.read(filePath, true)).to.eql([mySecretContents]);
     const rawText = (await readFile(filePath)).toString();
 
     expect(rawText.toString()).to.not.include(mySecretContents);
   });
 
-  it('appends newlines to files', async () => {
+  it('saves files without encoding', async () => {
+    const mySecretContents = 'pony-foo';
+    const config = new Config();
+    config.setToken('browserless.io');
+    const f = new FileSystem(config);
+
+    await f.append(filePath, mySecretContents, false);
+
+    expect(await f.read(filePath, false)).to.eql([mySecretContents]);
+    const rawText = (await readFile(filePath)).toString();
+
+    expect(rawText.toString()).to.include(mySecretContents);
+  });
+
+  it('appends newlines to files and encodes them', async () => {
     const mySecretContents = 'pony-foo';
     const moreSecretContents = 'pony-pony-foo-foo';
     const config = new Config();
     config.setToken('browserless.io');
     const f = new FileSystem(config);
 
-    await f.append(filePath, mySecretContents);
+    await f.append(filePath, mySecretContents, true);
 
-    expect(await f.read(filePath)).to.eql([mySecretContents]);
+    expect(await f.read(filePath, true)).to.eql([mySecretContents]);
 
-    await f.append(filePath, moreSecretContents);
+    await f.append(filePath, moreSecretContents, true);
 
-    expect(await f.read(filePath)).to.eql([
+    expect(await f.read(filePath, true)).to.eql([
       mySecretContents,
       moreSecretContents,
     ]);
+    const rawText = (await readFile(filePath)).toString();
+
+    expect(rawText).to.not.include(mySecretContents);
+    expect(rawText).to.not.include(moreSecretContents);
   });
 
-  it('re-encodes files on token changes', async () => {
+  it('appends newlines to files and does not encode them', async () => {
+    const mySecretContents = 'pony-foo';
+    const moreSecretContents = 'pony-pony-foo-foo';
     const config = new Config();
     config.setToken('browserless.io');
     const f = new FileSystem(config);
-    const mySecretContents = 'pony-foo';
 
-    await f.append(filePath, mySecretContents);
-    const oldText = (await readFile(filePath)).toString();
-    config.setToken('super-browserless-64');
-    await sleep(200);
-    const newText = (await readFile(filePath)).toString();
+    await f.append(filePath, mySecretContents, false);
+
+    expect(await f.read(filePath, false)).to.eql([mySecretContents]);
+
+    await f.append(filePath, moreSecretContents, false);
+
+    expect(await f.read(filePath, false)).to.eql([
+      mySecretContents,
+      moreSecretContents,
+    ]);
+    const rawText = (await readFile(filePath)).toString();
 
-    expect(oldText).to.not.equal(newText);
-    expect(await f.read(filePath)).to.eql([mySecretContents]);
+    expect(rawText).to.include(mySecretContents);
+    expect(rawText).to.include(moreSecretContents);
   });
 });

package/src/file-system.ts

@@ -15,28 +15,8 @@ export class FileSystem extends EventEmitter {
   constructor(protected config: Config) {
     super();
     this.currentAESKey = config.getAESKey();
-    this.config.on('token', this.handleTokenChange);
   }
 
-  private handleTokenChange = async () => {
-    this.log(`Token has changed, updating file-system contents`);
-    const start = Date.now();
-    const newAESKey = this.config.getAESKey();
-    await Promise.all(
-      Array.from(this.fsMap).map(async ([filePath, contents]) => {
-        const newlyEncoded = encrypt(
-          contents.join('\n'),
-          Buffer.from(newAESKey),
-        );
-        return writeFile(filePath, newlyEncoded);
-      }),
-    ).catch((e) => {
-      this.log(`Error in setting new token: "${e}"`);
-    });
-    this.log(`Successfully updated file encodings in ${Date.now() - start}ms`);
-    this.currentAESKey = this.config.getAESKey();
-  };
-
   /**
    * Appends contents to a file-path for persistance. File contents are
    * encrypted before being saved to disk. Reads happen via the in-memory
@@ -46,15 +26,19 @@ export class FileSystem extends EventEmitter {
    * @param newContent A string of new content to add to the file
    * @returns void
    */
-  append = async (path: string, newContent: string): Promise<void> => {
-    const contents = await this.read(path);
+  append = async (
+    path: string,
+    newContent: string,
+    shouldEncode: boolean,
+  ): Promise<void> => {
+    const contents = await this.read(path, shouldEncode);
 
     contents.push(newContent);
     this.fsMap.set(path, contents);
-    const encoded = await encrypt(
-      contents.join('\n'),
-      Buffer.from(this.currentAESKey),
-    );
+
+    const encoded = shouldEncode
+      ? await encrypt(contents.join('\n'), this.currentAESKey)
+      : contents.join('\n');
 
     return writeFile(path, encoded.toString());
   };
@@ -66,16 +50,18 @@ export class FileSystem extends EventEmitter {
    * @param path The filepath of the contents to read
    * @returns Promise of the contents separated by newlines
    */
-  read = async (path: string): Promise<string[]> => {
+  read = async (path: string, encoded: boolean): Promise<string[]> => {
     const hasKey = this.fsMap.has(path);
 
     if (hasKey) {
       return this.fsMap.get(path) as string[];
     }
     const contents = (await readFile(path).catch(() => '')).toString();
-    const splitContents = contents.length
-      ? (await decrypt(contents, Buffer.from(this.currentAESKey))).split('\n')
-      : [];
+    const decoded =
+      encoded && contents.length
+        ? await decrypt(contents, this.currentAESKey)
+        : contents;
+    const splitContents = decoded.length ? decoded.split('\n') : [];
 
     this.fsMap.set(path, splitContents);
 

package/src/hooks.ts

@@ -1,8 +1,50 @@
+import {
+  AfterResponse,
+  BeforeRequest,
+  BrowserHook,
+  PageHook,
+} from '@browserless.io/browserless';
+import { EventEmitter } from 'events';
+
+// KEPT for backwards compatibility reasons since some downstream
+// docker images will override these files to inject their own hook
+// behaviors
 // @ts-ignore
-export { default as beforeRequest } from '../external/before.js';
+import { default as afterRequest } from '../external/after.js';
 // @ts-ignore
-export { default as afterRequest } from '../external/after.js';
+import { default as beforeRequest } from '../external/before.js';
 // @ts-ignore
-export { default as pageHook } from '../external/page.js';
+import { default as browserHook } from '../external/browser.js';
 // @ts-ignore
-export { default as browserHook } from '../external/browser.js';
+import { default as pageHook } from '../external/page.js';
+
+export class Hooks extends EventEmitter {
+  before(args: BeforeRequest): Promise<boolean> {
+    return beforeRequest(args);
+  }
+
+  after(args: AfterResponse): Promise<unknown> {
+    return afterRequest(args);
+  }
+
+  page(args: PageHook): Promise<unknown> {
+    return pageHook(args);
+  }
+
+  browser(args: BrowserHook): Promise<unknown> {
+    return browserHook(args);
+  }
+
+  /**
+   * Implement any browserless-core-specific shutdown logic here.
+   * Calls the empty-SDK stop method for downstream implementations.
+   */
+  public shutdown = async () => {
+    await this.stop();
+  };
+
+  /**
+   * Left blank for downstream SDK modules to optionally implement.
+   */
+  public stop = () => {};
+}

package/src/limiter.spec.ts

@@ -1,23 +1,19 @@
 import {
   Config,
+  Hooks,
   Limiter,
   Metrics,
   Monitoring,
   WebHooks,
   sleep,
 } from '@browserless.io/browserless';
+import Sinon, { spy } from 'sinon';
 import { expect } from 'chai';
-import { spy } from 'sinon';
 
 const asyncNoop = () => Promise.resolve(undefined);
 const noop = () => undefined;
-const webHooks = {
-  callErrorAlertURL: spy(),
-  callFailedHealthURL: spy(),
-  callQueueAlertURL: spy(),
-  callRejectAlertURL: spy(),
-  callTimeoutAlertURL: spy(),
-};
+const webHooks = Sinon.createStubInstance(WebHooks);
+const hooks = Sinon.createStubInstance(Hooks);
 
 describe(`Limiter`, () => {
   afterEach(() => {
@@ -26,9 +22,13 @@ describe(`Limiter`, () => {
     webHooks.callRejectAlertURL.resetHistory();
     webHooks.callTimeoutAlertURL.resetHistory();
     webHooks.callErrorAlertURL.resetHistory();
+    hooks.before.resetHistory();
+    hooks.after.resetHistory();
+    hooks.browser.resetHistory();
+    hooks.page.resetHistory();
   });
 
-  it('limits and queues function calls and calls queue alert urls', async () => {
+  it('limits and queues function calls, calls hooks, and calls queue alert urls', async () => {
     return new Promise((resolve, reject) => {
       const config = new Config();
       config.setQueueAlertURL('https://example.com');
@@ -40,12 +40,7 @@ describe(`Limiter`, () => {
       config.setQueued(1);
       config.setTimeout(-1);
 
-      const limiter = new Limiter(
-        config,
-        metrics,
-        monitoring,
-        webHooks as unknown as WebHooks,
-      );
+      const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
       const handler = spy();
       const job = limiter.limit(handler, asyncNoop, asyncNoop, noop);
 
@@ -56,6 +51,7 @@ describe(`Limiter`, () => {
 
       limiter.addEventListener('end', () => {
         try {
+          expect(hooks.after.called).to.be.true;
           expect(handler.calledTwice).to.be.true;
           expect(webHooks.callQueueAlertURL.calledOnce).to.be.true;
           expect(metrics.get().queued).to.equal(1);
@@ -69,27 +65,36 @@ describe(`Limiter`, () => {
     });
   }).timeout(5000);
 
-  it('passes through arguments', () => {
-    const args = ['one', 'two', 'three'];
-    const config = new Config();
-    const metrics = new Metrics();
-    const monitoring = new Monitoring(config);
-    config.setConcurrent(1);
-    config.setQueued(0);
-    config.setTimeout(-1);
+  it('passes through arguments', () =>
+    new Promise((resolve, reject) => {
+      const args = ['one', 'two', 'three'];
+      const config = new Config();
+      const metrics = new Metrics();
+      const monitoring = new Monitoring(config);
+      config.setConcurrent(1);
+      config.setQueued(0);
+      config.setTimeout(-1);
 
-    const limiter = new Limiter(
-      config,
-      metrics,
-      monitoring,
-      webHooks as unknown as WebHooks,
-    );
-    const handler = spy();
-    const job = limiter.limit(handler, asyncNoop, asyncNoop, noop);
-    // @ts-ignore will fix later
-    job(...args);
-    expect(handler.args[0]).to.eql(args);
-  });
+      const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
+      const handler = spy();
+      const job = limiter.limit(handler, asyncNoop, asyncNoop, noop);
+      // @ts-ignore will fix later
+      job(...args);
+      expect(handler.args[0]).to.eql(args);
+
+      limiter.addEventListener('end', () => {
+        try {
+          expect(hooks.after.args[0][0]).to.have.property('start');
+          expect(hooks.after.args[0][0]).to.have.property(
+            'status',
+            'successful',
+          );
+        } catch (e) {
+          return reject(e);
+        }
+        resolve(undefined);
+      });
+    }));
 
   it('waits to run jobs until the first are done', async () => {
     const config = new Config();
@@ -99,12 +104,7 @@ describe(`Limiter`, () => {
     config.setQueued(1);
     config.setTimeout(-1);
 
-    const limiter = new Limiter(
-      config,
-      metrics,
-      monitoring,
-      webHooks as unknown as WebHooks,
-    );
+    const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
     const handlerOne = () => new Promise((r) => setTimeout(r, 50));
     const handlerTwo = spy();
 
@@ -129,12 +129,7 @@ describe(`Limiter`, () => {
     config.setQueued(1);
     config.setTimeout(-1);
 
-    const limiter = new Limiter(
-      config,
-      metrics,
-      monitoring,
-      webHooks as unknown as WebHooks,
-    );
+    const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
     const spy = () => new Promise((_r, rej) => rej(error));
 
     const job = limiter.limit(spy, asyncNoop, asyncNoop, noop);
@@ -150,6 +145,7 @@ describe(`Limiter`, () => {
           error,
         );
         expect(webHooks.callErrorAlertURL.calledOnce).to.be.true;
+        expect(hooks.after.args[0][0]).to.have.property('status', 'error');
       });
     });
   });
@@ -163,12 +159,7 @@ describe(`Limiter`, () => {
     config.setQueued(0);
     config.setTimeout(-1);
 
-    const limiter = new Limiter(
-      config,
-      metrics,
-      monitoring,
-      webHooks as unknown as WebHooks,
-    );
+    const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
 
     const handler = spy();
     const onError = spy();
@@ -184,37 +175,41 @@ describe(`Limiter`, () => {
     expect(onError.args[0]).to.eql(args);
   });
 
-  it('calls a timeout handler with arguments if a job takes too long', (r) => {
-    const args = ['one', 'two', 'three'];
-    const config = new Config();
-    const metrics = new Metrics();
-    const monitoring = new Monitoring(config);
-    config.setConcurrent(1);
-    config.setQueued(0);
-    config.setTimeout(10);
-
-    let timer: NodeJS.Timer;
-    const limiter = new Limiter(
-      config,
-      metrics,
-      monitoring,
-      webHooks as unknown as WebHooks,
-    );
-    const handler = () =>
-      new Promise((d) => (timer = global.setTimeout(d, 1000)));
-
-    const onTimeout = (...calledArgs: unknown[]) => {
-      clearTimeout(timer as unknown as number);
-      expect(calledArgs).to.eql(args);
-      expect(webHooks.callTimeoutAlertURL.calledOnce).to.be.true;
-      r(null);
-    };
+  it('calls a timeout handler with arguments if a job takes too long', () =>
+    new Promise((resolve, reject) => {
+      const args = ['one', 'two', 'three'];
+      const config = new Config();
+      const metrics = new Metrics();
+      const monitoring = new Monitoring(config);
+      config.setConcurrent(1);
+      config.setQueued(0);
+      config.setTimeout(10);
 
-    const job = limiter.limit(handler, noop, onTimeout, noop);
+      let timer: NodeJS.Timer;
+      const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
+      const handler = () =>
+        new Promise((d) => (timer = global.setTimeout(d, 1000)));
 
-    // @ts-ignore
-    job(...args);
-  });
+      const onTimeout = (...calledArgs: unknown[]) => {
+        clearTimeout(timer as unknown as number);
+        expect(calledArgs).to.eql(args);
+        expect(webHooks.callTimeoutAlertURL.calledOnce).to.be.true;
+      };
+
+      const job = limiter.limit(handler, noop, onTimeout, noop);
+
+      // @ts-ignore
+      job(...args);
+
+      limiter.addEventListener('end', () => {
+        try {
+          expect(hooks.after.args[0][0]).to.have.property('status', 'timedout');
+        } catch (e) {
+          return reject(e);
+        }
+        resolve(undefined);
+      });
+    }));
 
   it('allows overriding the timeouts', async () => {
     const config = new Config();
@@ -224,12 +219,7 @@ describe(`Limiter`, () => {
     config.setQueued(0);
     config.setTimeout(1);
 
-    const limiter = new Limiter(
-      config,
-      metrics,
-      monitoring,
-      webHooks as unknown as WebHooks,
-    );
+    const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
     const onTimeout = spy();
     const handler = async () => new Promise((r) => setTimeout(r, 10));
 
@@ -252,12 +242,7 @@ describe(`Limiter`, () => {
     config.setQueued(0);
     config.setTimeout(20);
 
-    const limiter = new Limiter(
-      config,
-      metrics,
-      monitoring,
-      webHooks as unknown as WebHooks,
-    );
+    const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
     const handler = () => new Promise((r) => setTimeout(r, 1));
     const timeout = spy();
     const job = limiter.limit(handler, noop, timeout, noop);
@@ -276,12 +261,7 @@ describe(`Limiter`, () => {
       config.setQueued(0);
       config.setTimeout(-1);
 
-      const limiter = new Limiter(
-        config,
-        metrics,
-        monitoring,
-        webHooks as unknown as WebHooks,
-      );
+      const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
 
       const handler = spy();
       const job = limiter.limit(handler, noop, noop, noop);
@@ -312,12 +292,7 @@ describe(`Limiter`, () => {
       config.setQueued(10);
       config.setTimeout(-1);
 
-      const limiter = new Limiter(
-        config,
-        metrics,
-        monitoring,
-        webHooks as unknown as WebHooks,
-      );
+      const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
 
       const handler = spy();
       const job = limiter.limit(handler, noop, noop, noop);
@@ -343,12 +318,7 @@ describe(`Limiter`, () => {
       config.setQueued(10);
       config.setTimeout(-1);
 
-      const limiter = new Limiter(
-        config,
-        metrics,
-        monitoring,
-        webHooks as unknown as WebHooks,
-      );
+      const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
 
       const handler = spy();
       const job = limiter.limit(handler, noop, noop, noop);