@browserless.io/browserless 2.4.0 → 2.5.0-beta-2

package/build/exports.d.ts

@@ -9,6 +9,7 @@ export * from './metrics.js';
 export * from './mime-types.js';
 export * from './monitoring.js';
 export * from './router.js';
+export * from './sdk-utils.js';
 export * from './server.js';
 export * from './shim.js';
 export * from './token.js';

package/build/exports.js

@@ -10,6 +10,7 @@ export * from './metrics.js';
 export * from './mime-types.js';
 export * from './monitoring.js';
 export * from './router.js';
+export * from './sdk-utils.js';
 export * from './server.js';
 export * from './shim.js';
 export * from './token.js';

package/build/file-system.d.ts

@@ -9,7 +9,6 @@ export declare class FileSystem extends EventEmitter {
     protected currentAESKey: Buffer;
     protected log: import("debug").Debugger;
     constructor(config: Config);
-    private handleTokenChange;
     /**
      * Appends contents to a file-path for persistance. File contents are
      * encrypted before being saved to disk. Reads happen via the in-memory
@@ -19,7 +18,7 @@ export declare class FileSystem extends EventEmitter {
      * @param newContent A string of new content to add to the file
      * @returns void
      */
-    append: (path: string, newContent: string) => Promise<void>;
+    append: (path: string, newContent: string, shouldEncode: boolean) => Promise<void>;
     /**
      * Reads contents from the local map, if any exist, or loads
      * from the file system and hydrates the cache for the particular filepath
@@ -27,7 +26,7 @@ export declare class FileSystem extends EventEmitter {
      * @param path The filepath of the contents to read
      * @returns Promise of the contents separated by newlines
      */
-    read: (path: string) => Promise<string[]>;
+    read: (path: string, encoded: boolean) => Promise<string[]>;
     /**
      * Implement any browserless-core-specific shutdown logic here.
      * Calls the empty-SDK stop method for downstream implementations.

package/build/file-system.js

@@ -10,21 +10,7 @@ export class FileSystem extends EventEmitter {
         super();
         this.config = config;
         this.currentAESKey = config.getAESKey();
-        this.config.on('token', this.handleTokenChange);
     }
-    handleTokenChange = async () => {
-        this.log(`Token has changed, updating file-system contents`);
-        const start = Date.now();
-        const newAESKey = this.config.getAESKey();
-        await Promise.all(Array.from(this.fsMap).map(async ([filePath, contents]) => {
-            const newlyEncoded = encrypt(contents.join('\n'), Buffer.from(newAESKey));
-            return writeFile(filePath, newlyEncoded);
-        })).catch((e) => {
-            this.log(`Error in setting new token: "${e}"`);
-        });
-        this.log(`Successfully updated file encodings in ${Date.now() - start}ms`);
-        this.currentAESKey = this.config.getAESKey();
-    };
     /**
      * Appends contents to a file-path for persistance. File contents are
      * encrypted before being saved to disk. Reads happen via the in-memory
@@ -34,11 +20,13 @@ export class FileSystem extends EventEmitter {
      * @param newContent A string of new content to add to the file
      * @returns void
      */
-    append = async (path, newContent) => {
-        const contents = await this.read(path);
+    append = async (path, newContent, shouldEncode) => {
+        const contents = await this.read(path, shouldEncode);
         contents.push(newContent);
         this.fsMap.set(path, contents);
-        const encoded = await encrypt(contents.join('\n'), Buffer.from(this.currentAESKey));
+        const encoded = shouldEncode
+            ? await encrypt(contents.join('\n'), this.currentAESKey)
+            : contents.join('\n');
         return writeFile(path, encoded.toString());
     };
     /**
@@ -48,15 +36,16 @@ export class FileSystem extends EventEmitter {
      * @param path The filepath of the contents to read
      * @returns Promise of the contents separated by newlines
      */
-    read = async (path) => {
+    read = async (path, encoded) => {
         const hasKey = this.fsMap.has(path);
         if (hasKey) {
             return this.fsMap.get(path);
         }
         const contents = (await readFile(path).catch(() => '')).toString();
-        const splitContents = contents.length
-            ? (await decrypt(contents, Buffer.from(this.currentAESKey))).split('\n')
-            : [];
+        const decoded = encoded && contents.length
+            ? await decrypt(contents, this.currentAESKey)
+            : contents;
+        const splitContents = decoded.length ? decoded.split('\n') : [];
         this.fsMap.set(path, splitContents);
         return splitContents;
     };

package/build/file-system.spec.js

@@ -1,44 +1,61 @@
-import { Config, FileSystem, sleep } from '@browserless.io/browserless';
+import { Config, FileSystem, noop } from '@browserless.io/browserless';
 import { readFile, unlink } from 'fs/promises';
 import { expect } from 'chai';
 const filePath = '/tmp/_browserless_test_fs_';
 describe('File-System', () => {
-    afterEach(async () => unlink(filePath));
+    afterEach(async () => unlink(filePath).catch(noop));
     it('saves and encodes files', async () => {
         const mySecretContents = 'pony-foo';
         const config = new Config();
         config.setToken('browserless.io');
         const f = new FileSystem(config);
-        await f.append(filePath, mySecretContents);
-        expect(await f.read(filePath)).to.eql([mySecretContents]);
+        await f.append(filePath, mySecretContents, true);
+        expect(await f.read(filePath, true)).to.eql([mySecretContents]);
         const rawText = (await readFile(filePath)).toString();
         expect(rawText.toString()).to.not.include(mySecretContents);
     });
-    it('appends newlines to files', async () => {
+    it('saves files without encoding', async () => {
+        const mySecretContents = 'pony-foo';
+        const config = new Config();
+        config.setToken('browserless.io');
+        const f = new FileSystem(config);
+        await f.append(filePath, mySecretContents, false);
+        expect(await f.read(filePath, false)).to.eql([mySecretContents]);
+        const rawText = (await readFile(filePath)).toString();
+        expect(rawText.toString()).to.include(mySecretContents);
+    });
+    it('appends newlines to files and encodes them', async () => {
         const mySecretContents = 'pony-foo';
         const moreSecretContents = 'pony-pony-foo-foo';
         const config = new Config();
         config.setToken('browserless.io');
         const f = new FileSystem(config);
-        await f.append(filePath, mySecretContents);
-        expect(await f.read(filePath)).to.eql([mySecretContents]);
-        await f.append(filePath, moreSecretContents);
-        expect(await f.read(filePath)).to.eql([
+        await f.append(filePath, mySecretContents, true);
+        expect(await f.read(filePath, true)).to.eql([mySecretContents]);
+        await f.append(filePath, moreSecretContents, true);
+        expect(await f.read(filePath, true)).to.eql([
             mySecretContents,
             moreSecretContents,
         ]);
+        const rawText = (await readFile(filePath)).toString();
+        expect(rawText).to.not.include(mySecretContents);
+        expect(rawText).to.not.include(moreSecretContents);
     });
-    it('re-encodes files on token changes', async () => {
+    it('appends newlines to files and does not encode them', async () => {
+        const mySecretContents = 'pony-foo';
+        const moreSecretContents = 'pony-pony-foo-foo';
         const config = new Config();
         config.setToken('browserless.io');
         const f = new FileSystem(config);
-        const mySecretContents = 'pony-foo';
-        await f.append(filePath, mySecretContents);
-        const oldText = (await readFile(filePath)).toString();
-        config.setToken('super-browserless-64');
-        await sleep(200);
-        const newText = (await readFile(filePath)).toString();
-        expect(oldText).to.not.equal(newText);
-        expect(await f.read(filePath)).to.eql([mySecretContents]);
+        await f.append(filePath, mySecretContents, false);
+        expect(await f.read(filePath, false)).to.eql([mySecretContents]);
+        await f.append(filePath, moreSecretContents, false);
+        expect(await f.read(filePath, false)).to.eql([
+            mySecretContents,
+            moreSecretContents,
+        ]);
+        const rawText = (await readFile(filePath)).toString();
+        expect(rawText).to.include(mySecretContents);
+        expect(rawText).to.include(moreSecretContents);
     });
 });

package/build/hooks.d.ts

@@ -1,4 +1,18 @@
-export { default as beforeRequest } from '../external/before.js';
-export { default as afterRequest } from '../external/after.js';
-export { default as pageHook } from '../external/page.js';
-export { default as browserHook } from '../external/browser.js';
+/// <reference types="node" />
+import { AfterResponse, BeforeRequest, BrowserHook, PageHook } from '@browserless.io/browserless';
+import { EventEmitter } from 'events';
+export declare class Hooks extends EventEmitter {
+    before(args: BeforeRequest): Promise<boolean>;
+    after(args: AfterResponse): Promise<unknown>;
+    page(args: PageHook): Promise<unknown>;
+    browser(args: BrowserHook): Promise<unknown>;
+    /**
+     * Implement any browserless-core-specific shutdown logic here.
+     * Calls the empty-SDK stop method for downstream implementations.
+     */
+    shutdown: () => Promise<void>;
+    /**
+     * Left blank for downstream SDK modules to optionally implement.
+     */
+    stop: () => void;
+}

package/build/hooks.js

@@ -1,8 +1,37 @@
+import { EventEmitter } from 'events';
+// KEPT for backwards compatibility reasons since some downstream
+// docker images will override these files to inject their own hook
+// behaviors
 // @ts-ignore
-export { default as beforeRequest } from '../external/before.js';
+import { default as afterRequest } from '../external/after.js';
 // @ts-ignore
-export { default as afterRequest } from '../external/after.js';
+import { default as beforeRequest } from '../external/before.js';
 // @ts-ignore
-export { default as pageHook } from '../external/page.js';
+import { default as browserHook } from '../external/browser.js';
 // @ts-ignore
-export { default as browserHook } from '../external/browser.js';
+import { default as pageHook } from '../external/page.js';
+export class Hooks extends EventEmitter {
+    before(args) {
+        return beforeRequest(args);
+    }
+    after(args) {
+        return afterRequest(args);
+    }
+    page(args) {
+        return pageHook(args);
+    }
+    browser(args) {
+        return browserHook(args);
+    }
+    /**
+     * Implement any browserless-core-specific shutdown logic here.
+     * Calls the empty-SDK stop method for downstream implementations.
+     */
+    shutdown = async () => {
+        await this.stop();
+    };
+    /**
+     * Left blank for downstream SDK modules to optionally implement.
+     */
+    stop = () => { };
+}

package/build/limiter.d.ts

@@ -1,5 +1,5 @@
 /// <reference types="debug" />
-import { AfterResponse, Config, Metrics, Monitoring, WebHooks } from '@browserless.io/browserless';
+import { AfterResponse, Config, Hooks, Metrics, Monitoring, WebHooks } from '@browserless.io/browserless';
 import q from 'queue';
 export type LimitFn<TArgs extends unknown[], TResult> = (...args: TArgs) => Promise<TResult>;
 export type ErrorFn<TArgs extends unknown[]> = (...args: TArgs) => void;
@@ -15,9 +15,10 @@ export declare class Limiter extends q {
     protected metrics: Metrics;
     protected monitor: Monitoring;
     protected webhooks: WebHooks;
+    protected hooks: Hooks;
     protected queued: number;
     protected debug: import("debug").Debugger;
-    constructor(config: Config, metrics: Metrics, monitor: Monitoring, webhooks: WebHooks);
+    constructor(config: Config, metrics: Metrics, monitor: Monitoring, webhooks: WebHooks, hooks: Hooks);
     protected handleEnd(): void;
     protected jobEnd(jobInfo: AfterResponse): void;
     protected handleSuccess({ detail: { job } }: {

package/build/limiter.js

@@ -1,13 +1,14 @@
-import { TooManyRequests, afterRequest, createLogger, } from '@browserless.io/browserless';
+import { TooManyRequests, createLogger, } from '@browserless.io/browserless';
 import q from 'queue';
 export class Limiter extends q {
     config;
     metrics;
     monitor;
     webhooks;
+    hooks;
     queued;
     debug = createLogger('limiter');
-    constructor(config, metrics, monitor, webhooks) {
+    constructor(config, metrics, monitor, webhooks, hooks) {
         super({
             autostart: true,
             concurrency: config.getConcurrent(),
@@ -17,6 +18,7 @@ export class Limiter extends q {
         this.metrics = metrics;
         this.monitor = monitor;
         this.webhooks = webhooks;
+        this.hooks = hooks;
         this.queued = config.getQueued();
         this.debug(`Concurrency: ${this.concurrency} queue: ${this.queued} timeout: ${this.timeout}ms`);
         config.on('concurrent', (concurrency) => {
@@ -43,7 +45,7 @@ export class Limiter extends q {
         this.logQueue('All jobs complete.');
     }
     jobEnd(jobInfo) {
-        afterRequest(jobInfo);
+        this.hooks.after(jobInfo);
     }
     handleSuccess({ detail: { job } }) {
         const timeUsed = Date.now() - job.start;

package/build/limiter.spec.js

@@ -1,15 +1,10 @@
-import { Config, Limiter, Metrics, Monitoring, sleep, } from '@browserless.io/browserless';
+import { Config, Hooks, Limiter, Metrics, Monitoring, WebHooks, sleep, } from '@browserless.io/browserless';
+import Sinon, { spy } from 'sinon';
 import { expect } from 'chai';
-import { spy } from 'sinon';
 const asyncNoop = () => Promise.resolve(undefined);
 const noop = () => undefined;
-const webHooks = {
-    callErrorAlertURL: spy(),
-    callFailedHealthURL: spy(),
-    callQueueAlertURL: spy(),
-    callRejectAlertURL: spy(),
-    callTimeoutAlertURL: spy(),
-};
+const webHooks = Sinon.createStubInstance(WebHooks);
+const hooks = Sinon.createStubInstance(Hooks);
 describe(`Limiter`, () => {
     afterEach(() => {
         webHooks.callFailedHealthURL.resetHistory();
@@ -17,8 +12,12 @@ describe(`Limiter`, () => {
         webHooks.callRejectAlertURL.resetHistory();
         webHooks.callTimeoutAlertURL.resetHistory();
         webHooks.callErrorAlertURL.resetHistory();
+        hooks.before.resetHistory();
+        hooks.after.resetHistory();
+        hooks.browser.resetHistory();
+        hooks.page.resetHistory();
     });
-    it('limits and queues function calls and calls queue alert urls', async () => {
+    it('limits and queues function calls, calls hooks, and calls queue alert urls', async () => {
         return new Promise((resolve, reject) => {
             const config = new Config();
             config.setQueueAlertURL('https://example.com');
@@ -27,7 +26,7 @@ describe(`Limiter`, () => {
             config.setConcurrent(1);
             config.setQueued(1);
             config.setTimeout(-1);
-            const limiter = new Limiter(config, metrics, monitoring, webHooks);
+            const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
             const handler = spy();
             const job = limiter.limit(handler, asyncNoop, asyncNoop, noop);
             job();
@@ -35,6 +34,7 @@ describe(`Limiter`, () => {
             expect(handler.calledOnce).to.be.true;
             limiter.addEventListener('end', () => {
                 try {
+                    expect(hooks.after.called).to.be.true;
                     expect(handler.calledTwice).to.be.true;
                     expect(webHooks.callQueueAlertURL.calledOnce).to.be.true;
                     expect(metrics.get().queued).to.equal(1);
@@ -48,7 +48,7 @@ describe(`Limiter`, () => {
             });
         });
     }).timeout(5000);
-    it('passes through arguments', () => {
+    it('passes through arguments', () => new Promise((resolve, reject) => {
         const args = ['one', 'two', 'three'];
         const config = new Config();
         const metrics = new Metrics();
@@ -56,13 +56,23 @@ describe(`Limiter`, () => {
         config.setConcurrent(1);
         config.setQueued(0);
         config.setTimeout(-1);
-        const limiter = new Limiter(config, metrics, monitoring, webHooks);
+        const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
         const handler = spy();
         const job = limiter.limit(handler, asyncNoop, asyncNoop, noop);
         // @ts-ignore will fix later
         job(...args);
         expect(handler.args[0]).to.eql(args);
-    });
+        limiter.addEventListener('end', () => {
+            try {
+                expect(hooks.after.args[0][0]).to.have.property('start');
+                expect(hooks.after.args[0][0]).to.have.property('status', 'successful');
+            }
+            catch (e) {
+                return reject(e);
+            }
+            resolve(undefined);
+        });
+    }));
     it('waits to run jobs until the first are done', async () => {
         const config = new Config();
         const monitoring = new Monitoring(config);
@@ -70,7 +80,7 @@ describe(`Limiter`, () => {
         config.setConcurrent(1);
         config.setQueued(1);
         config.setTimeout(-1);
-        const limiter = new Limiter(config, metrics, monitoring, webHooks);
+        const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
         const handlerOne = () => new Promise((r) => setTimeout(r, 50));
         const handlerTwo = spy();
         const jobOne = limiter.limit(handlerOne, asyncNoop, asyncNoop, noop);
@@ -89,7 +99,7 @@ describe(`Limiter`, () => {
         config.setConcurrent(1);
         config.setQueued(1);
         config.setTimeout(-1);
-        const limiter = new Limiter(config, metrics, monitoring, webHooks);
+        const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
         const spy = () => new Promise((_r, rej) => rej(error));
         const job = limiter.limit(spy, asyncNoop, asyncNoop, noop);
         await job().catch(noop);
@@ -101,6 +111,7 @@ describe(`Limiter`, () => {
                 expect(metrics.get().error).to.eql(1);
                 expect(webHooks.callErrorAlertURL.getCalls()[0].firstArg).to.include(error);
                 expect(webHooks.callErrorAlertURL.calledOnce).to.be.true;
+                expect(hooks.after.args[0][0]).to.have.property('status', 'error');
             });
         });
     });
@@ -112,7 +123,7 @@ describe(`Limiter`, () => {
         config.setConcurrent(1);
         config.setQueued(0);
         config.setTimeout(-1);
-        const limiter = new Limiter(config, metrics, monitoring, webHooks);
+        const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
         const handler = spy();
         const onError = spy();
         const job = limiter.limit(handler, onError, noop, noop);
@@ -124,7 +135,7 @@ describe(`Limiter`, () => {
         expect(onError.calledOnce).to.be.true;
         expect(onError.args[0]).to.eql(args);
     });
-    it('calls a timeout handler with arguments if a job takes too long', (r) => {
+    it('calls a timeout handler with arguments if a job takes too long', () => new Promise((resolve, reject) => {
         const args = ['one', 'two', 'three'];
         const config = new Config();
         const metrics = new Metrics();
@@ -133,18 +144,26 @@ describe(`Limiter`, () => {
         config.setQueued(0);
         config.setTimeout(10);
         let timer;
-        const limiter = new Limiter(config, metrics, monitoring, webHooks);
+        const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
         const handler = () => new Promise((d) => (timer = global.setTimeout(d, 1000)));
         const onTimeout = (...calledArgs) => {
             clearTimeout(timer);
             expect(calledArgs).to.eql(args);
             expect(webHooks.callTimeoutAlertURL.calledOnce).to.be.true;
-            r(null);
         };
         const job = limiter.limit(handler, noop, onTimeout, noop);
         // @ts-ignore
         job(...args);
-    });
+        limiter.addEventListener('end', () => {
+            try {
+                expect(hooks.after.args[0][0]).to.have.property('status', 'timedout');
+            }
+            catch (e) {
+                return reject(e);
+            }
+            resolve(undefined);
+        });
+    }));
     it('allows overriding the timeouts', async () => {
         const config = new Config();
         const metrics = new Metrics();
@@ -152,7 +171,7 @@ describe(`Limiter`, () => {
         config.setConcurrent(2);
         config.setQueued(0);
         config.setTimeout(1);
-        const limiter = new Limiter(config, metrics, monitoring, webHooks);
+        const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
         const onTimeout = spy();
         const handler = async () => new Promise((r) => setTimeout(r, 10));
         const job = limiter.limit(handler, noop, onTimeout, noop);
@@ -169,7 +188,7 @@ describe(`Limiter`, () => {
         config.setConcurrent(1);
         config.setQueued(0);
         config.setTimeout(20);
-        const limiter = new Limiter(config, metrics, monitoring, webHooks);
+        const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
         const handler = () => new Promise((r) => setTimeout(r, 1));
         const timeout = spy();
         const job = limiter.limit(handler, noop, timeout, noop);
@@ -184,7 +203,7 @@ describe(`Limiter`, () => {
         config.setConcurrent(1);
         config.setQueued(0);
         config.setTimeout(-1);
-        const limiter = new Limiter(config, metrics, monitoring, webHooks);
+        const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
         const handler = spy();
         const job = limiter.limit(handler, noop, noop, noop);
         job();
@@ -207,7 +226,7 @@ describe(`Limiter`, () => {
         config.setConcurrent(10);
         config.setQueued(10);
         config.setTimeout(-1);
-        const limiter = new Limiter(config, metrics, monitoring, webHooks);
+        const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
         const handler = spy();
         const job = limiter.limit(handler, noop, noop, noop);
         job().catch(() => {
@@ -227,7 +246,7 @@ describe(`Limiter`, () => {
         config.setConcurrent(10);
         config.setQueued(10);
         config.setTimeout(-1);
-        const limiter = new Limiter(config, metrics, monitoring, webHooks);
+        const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
         const handler = spy();
         const job = limiter.limit(handler, noop, noop, noop);
         job();

package/build/routes/chrome/http/content.post.body.json

@@ -394,14 +394,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@10915": {
+                "__@toStringTag@10918": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@10915",
+                "__@toStringTag@10918",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -436,13 +436,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@10915": {
+        "__@toStringTag@10918": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@10915",
+        "__@toStringTag@10918",
         "byteLength"
       ]
     },
@@ -452,18 +452,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@11016": {
+        "__@species@11019": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@10915": {
+        "__@toStringTag@10918": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@11016",
-        "__@toStringTag@10915",
+        "__@species@11019",
+        "__@toStringTag@10918",
         "byteLength"
       ]
     },

package/build/routes/chrome/http/pdf.post.body.json

@@ -535,14 +535,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@86785": {
+                "__@toStringTag@75984": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@86785",
+                "__@toStringTag@75984",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -577,13 +577,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@86785": {
+        "__@toStringTag@75984": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@86785",
+        "__@toStringTag@75984",
         "byteLength"
       ]
     },
@@ -593,18 +593,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@86886": {
+        "__@species@76085": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@86785": {
+        "__@toStringTag@75984": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@86886",
-        "__@toStringTag@86785",
+        "__@species@76085",
+        "__@toStringTag@75984",
         "byteLength"
       ]
     },