@browserless.io/browserless 2.4.0-beta-3 → 2.5.0-beta-1

package/src/limiter.spec.ts

@@ -1,23 +1,19 @@
 import {
   Config,
+  Hooks,
   Limiter,
   Metrics,
   Monitoring,
   WebHooks,
   sleep,
 } from '@browserless.io/browserless';
+import Sinon, { spy } from 'sinon';
 import { expect } from 'chai';
-import { spy } from 'sinon';
 
 const asyncNoop = () => Promise.resolve(undefined);
 const noop = () => undefined;
-const webHooks = {
-  callErrorAlertURL: spy(),
-  callFailedHealthURL: spy(),
-  callQueueAlertURL: spy(),
-  callRejectAlertURL: spy(),
-  callTimeoutAlertURL: spy(),
-};
+const webHooks = Sinon.createStubInstance(WebHooks);
+const hooks = Sinon.createStubInstance(Hooks);
 
 describe(`Limiter`, () => {
   afterEach(() => {
@@ -26,9 +22,13 @@ describe(`Limiter`, () => {
     webHooks.callRejectAlertURL.resetHistory();
     webHooks.callTimeoutAlertURL.resetHistory();
     webHooks.callErrorAlertURL.resetHistory();
+    hooks.before.resetHistory();
+    hooks.after.resetHistory();
+    hooks.browser.resetHistory();
+    hooks.page.resetHistory();
   });
 
-  it('limits and queues function calls and calls queue alert urls', async () => {
+  it('limits and queues function calls, calls hooks, and calls queue alert urls', async () => {
     return new Promise((resolve, reject) => {
       const config = new Config();
       config.setQueueAlertURL('https://example.com');
@@ -40,12 +40,7 @@ describe(`Limiter`, () => {
       config.setQueued(1);
       config.setTimeout(-1);
 
-      const limiter = new Limiter(
-        config,
-        metrics,
-        monitoring,
-        webHooks as unknown as WebHooks,
-      );
+      const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
       const handler = spy();
       const job = limiter.limit(handler, asyncNoop, asyncNoop, noop);
 
@@ -56,6 +51,7 @@ describe(`Limiter`, () => {
 
       limiter.addEventListener('end', () => {
         try {
+          expect(hooks.after.called).to.be.true;
           expect(handler.calledTwice).to.be.true;
           expect(webHooks.callQueueAlertURL.calledOnce).to.be.true;
           expect(metrics.get().queued).to.equal(1);
@@ -69,27 +65,36 @@ describe(`Limiter`, () => {
     });
   }).timeout(5000);
 
-  it('passes through arguments', () => {
-    const args = ['one', 'two', 'three'];
-    const config = new Config();
-    const metrics = new Metrics();
-    const monitoring = new Monitoring(config);
-    config.setConcurrent(1);
-    config.setQueued(0);
-    config.setTimeout(-1);
+  it('passes through arguments', () =>
+    new Promise((resolve, reject) => {
+      const args = ['one', 'two', 'three'];
+      const config = new Config();
+      const metrics = new Metrics();
+      const monitoring = new Monitoring(config);
+      config.setConcurrent(1);
+      config.setQueued(0);
+      config.setTimeout(-1);
 
-    const limiter = new Limiter(
-      config,
-      metrics,
-      monitoring,
-      webHooks as unknown as WebHooks,
-    );
-    const handler = spy();
-    const job = limiter.limit(handler, asyncNoop, asyncNoop, noop);
-    // @ts-ignore will fix later
-    job(...args);
-    expect(handler.args[0]).to.eql(args);
-  });
+      const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
+      const handler = spy();
+      const job = limiter.limit(handler, asyncNoop, asyncNoop, noop);
+      // @ts-ignore will fix later
+      job(...args);
+      expect(handler.args[0]).to.eql(args);
+
+      limiter.addEventListener('end', () => {
+        try {
+          expect(hooks.after.args[0][0]).to.have.property('start');
+          expect(hooks.after.args[0][0]).to.have.property(
+            'status',
+            'successful',
+          );
+        } catch (e) {
+          return reject(e);
+        }
+        resolve(undefined);
+      });
+    }));
 
   it('waits to run jobs until the first are done', async () => {
     const config = new Config();
@@ -99,12 +104,7 @@ describe(`Limiter`, () => {
     config.setQueued(1);
     config.setTimeout(-1);
 
-    const limiter = new Limiter(
-      config,
-      metrics,
-      monitoring,
-      webHooks as unknown as WebHooks,
-    );
+    const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
     const handlerOne = () => new Promise((r) => setTimeout(r, 50));
     const handlerTwo = spy();
 
@@ -129,12 +129,7 @@ describe(`Limiter`, () => {
     config.setQueued(1);
     config.setTimeout(-1);
 
-    const limiter = new Limiter(
-      config,
-      metrics,
-      monitoring,
-      webHooks as unknown as WebHooks,
-    );
+    const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
     const spy = () => new Promise((_r, rej) => rej(error));
 
     const job = limiter.limit(spy, asyncNoop, asyncNoop, noop);
@@ -150,6 +145,7 @@ describe(`Limiter`, () => {
           error,
         );
         expect(webHooks.callErrorAlertURL.calledOnce).to.be.true;
+        expect(hooks.after.args[0][0]).to.have.property('status', 'error');
       });
     });
   });
@@ -163,12 +159,7 @@ describe(`Limiter`, () => {
     config.setQueued(0);
     config.setTimeout(-1);
 
-    const limiter = new Limiter(
-      config,
-      metrics,
-      monitoring,
-      webHooks as unknown as WebHooks,
-    );
+    const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
 
     const handler = spy();
     const onError = spy();
@@ -184,37 +175,41 @@ describe(`Limiter`, () => {
     expect(onError.args[0]).to.eql(args);
   });
 
-  it('calls a timeout handler with arguments if a job takes too long', (r) => {
-    const args = ['one', 'two', 'three'];
-    const config = new Config();
-    const metrics = new Metrics();
-    const monitoring = new Monitoring(config);
-    config.setConcurrent(1);
-    config.setQueued(0);
-    config.setTimeout(10);
-
-    let timer: NodeJS.Timer;
-    const limiter = new Limiter(
-      config,
-      metrics,
-      monitoring,
-      webHooks as unknown as WebHooks,
-    );
-    const handler = () =>
-      new Promise((d) => (timer = global.setTimeout(d, 1000)));
-
-    const onTimeout = (...calledArgs: unknown[]) => {
-      clearTimeout(timer as unknown as number);
-      expect(calledArgs).to.eql(args);
-      expect(webHooks.callTimeoutAlertURL.calledOnce).to.be.true;
-      r(null);
-    };
+  it('calls a timeout handler with arguments if a job takes too long', () =>
+    new Promise((resolve, reject) => {
+      const args = ['one', 'two', 'three'];
+      const config = new Config();
+      const metrics = new Metrics();
+      const monitoring = new Monitoring(config);
+      config.setConcurrent(1);
+      config.setQueued(0);
+      config.setTimeout(10);
 
-    const job = limiter.limit(handler, noop, onTimeout, noop);
+      let timer: NodeJS.Timer;
+      const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
+      const handler = () =>
+        new Promise((d) => (timer = global.setTimeout(d, 1000)));
 
-    // @ts-ignore
-    job(...args);
-  });
+      const onTimeout = (...calledArgs: unknown[]) => {
+        clearTimeout(timer as unknown as number);
+        expect(calledArgs).to.eql(args);
+        expect(webHooks.callTimeoutAlertURL.calledOnce).to.be.true;
+      };
+
+      const job = limiter.limit(handler, noop, onTimeout, noop);
+
+      // @ts-ignore
+      job(...args);
+
+      limiter.addEventListener('end', () => {
+        try {
+          expect(hooks.after.args[0][0]).to.have.property('status', 'timedout');
+        } catch (e) {
+          return reject(e);
+        }
+        resolve(undefined);
+      });
+    }));
 
   it('allows overriding the timeouts', async () => {
     const config = new Config();
@@ -224,12 +219,7 @@ describe(`Limiter`, () => {
     config.setQueued(0);
     config.setTimeout(1);
 
-    const limiter = new Limiter(
-      config,
-      metrics,
-      monitoring,
-      webHooks as unknown as WebHooks,
-    );
+    const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
     const onTimeout = spy();
     const handler = async () => new Promise((r) => setTimeout(r, 10));
 
@@ -252,12 +242,7 @@ describe(`Limiter`, () => {
     config.setQueued(0);
     config.setTimeout(20);
 
-    const limiter = new Limiter(
-      config,
-      metrics,
-      monitoring,
-      webHooks as unknown as WebHooks,
-    );
+    const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
     const handler = () => new Promise((r) => setTimeout(r, 1));
     const timeout = spy();
     const job = limiter.limit(handler, noop, timeout, noop);
@@ -276,12 +261,7 @@ describe(`Limiter`, () => {
       config.setQueued(0);
       config.setTimeout(-1);
 
-      const limiter = new Limiter(
-        config,
-        metrics,
-        monitoring,
-        webHooks as unknown as WebHooks,
-      );
+      const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
 
       const handler = spy();
       const job = limiter.limit(handler, noop, noop, noop);
@@ -312,12 +292,7 @@ describe(`Limiter`, () => {
       config.setQueued(10);
       config.setTimeout(-1);
 
-      const limiter = new Limiter(
-        config,
-        metrics,
-        monitoring,
-        webHooks as unknown as WebHooks,
-      );
+      const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
 
       const handler = spy();
       const job = limiter.limit(handler, noop, noop, noop);
@@ -343,12 +318,7 @@ describe(`Limiter`, () => {
       config.setQueued(10);
       config.setTimeout(-1);
 
-      const limiter = new Limiter(
-        config,
-        metrics,
-        monitoring,
-        webHooks as unknown as WebHooks,
-      );
+      const limiter = new Limiter(config, metrics, monitoring, webHooks, hooks);
 
       const handler = spy();
       const job = limiter.limit(handler, noop, noop, noop);

package/src/limiter.ts

@@ -1,11 +1,11 @@
 import {
   AfterResponse,
   Config,
+  Hooks,
   Metrics,
   Monitoring,
   TooManyRequests,
   WebHooks,
-  afterRequest,
   createLogger,
 } from '@browserless.io/browserless';
 import q from 'queue';
@@ -33,13 +33,13 @@ export class Limiter extends q {
     protected metrics: Metrics,
     protected monitor: Monitoring,
     protected webhooks: WebHooks,
+    protected hooks: Hooks,
   ) {
     super({
       autostart: true,
       concurrency: config.getConcurrent(),
       timeout: config.getTimeout(),
     });
-
     this.queued = config.getQueued();
 
     this.debug(
@@ -78,7 +78,7 @@ export class Limiter extends q {
   }
 
   protected jobEnd(jobInfo: AfterResponse) {
-    afterRequest(jobInfo);
+    this.hooks.after(jobInfo);
   }
 
   protected handleSuccess({ detail: { job } }: { detail: { job: Job } }) {

package/src/routes/management/http/metrics-total.get.ts

@@ -29,9 +29,9 @@ export default class MetricsTotalGetRoute extends HTTPRoute {
   handler = async (_req: Request, res: ServerResponse): Promise<void> => {
     const fileSystem = this.fileSystem();
     const config = this.config();
-    const metrics = (await fileSystem.read(config.getMetricsJSONPath())).map(
-      (m) => JSON.parse(m),
-    );
+    const metrics = (
+      await fileSystem.read(config.getMetricsJSONPath(), false)
+    ).map((m) => JSON.parse(m));
     const availableMetrics = metrics.length;
     const totals: IBrowserlessMetricTotals = metrics.reduce(
       (accum, metric) => ({

package/src/routes/management/http/metrics.get.ts

@@ -20,7 +20,7 @@ export default class MetricsGetRoute extends HTTPRoute {
   browser = null;
   concurrency = false;
   contentTypes = [contentTypes.json];
-  description = `Gets total metric details from the time the server started.`;
+  description = `Returns a list of metric details as far back as possible.`;
   method = Methods.get;
   path = HTTPManagementRoutes.metrics;
   tags = [APITags.management];
@@ -28,7 +28,7 @@ export default class MetricsGetRoute extends HTTPRoute {
     const fileSystem = this.fileSystem();
     const config = this.config();
 
-    const stats = await fileSystem.read(config.getMetricsJSONPath());
+    const stats = await fileSystem.read(config.getMetricsJSONPath(), false);
     const response = `[${stats.join(',')}]`;
 
     return writeResponse(res, 200, response, contentTypes.json);

package/src/sdk-utils.ts

@@ -0,0 +1,136 @@
+import { createInterface } from 'readline';
+import debug from 'debug';
+import fs from 'fs/promises';
+import path from 'path';
+import { spawn } from 'child_process';
+
+export const getArgSwitches = () => {
+  return process.argv.reduce(
+    (accum, arg, idx) => {
+      if (!arg.startsWith('--')) {
+        return accum;
+      }
+
+      if (arg.includes('=')) {
+        const [parameter, value] = arg.split('=');
+        accum[parameter.replace(/-/gi, '')] = value || true;
+        return accum;
+      }
+
+      const nextSwitchOrParameter = process.argv[idx + 1];
+      const param = arg.replace(/-/gi, '');
+
+      if (
+        typeof nextSwitchOrParameter === 'undefined' ||
+        nextSwitchOrParameter?.startsWith('--')
+      ) {
+        accum[param] = true;
+        return accum;
+      }
+
+      accum[param] = nextSwitchOrParameter;
+
+      return accum;
+    },
+    {} as { [key: string]: string | true },
+  );
+};
+
+export const getSourceFiles = async (cwd: string) => {
+  const buildDir = path.join(cwd, 'build');
+  const files = await fs.readdir(buildDir, { recursive: true });
+  const [httpRoutes, webSocketRoutes] = files.reduce(
+    ([httpRoutes, webSocketRoutes], file) => {
+      const parsed = path.parse(file);
+      if (parsed.name.endsWith('http')) {
+        httpRoutes.push(path.join(buildDir, file));
+      }
+
+      if (parsed.name.endsWith('ws')) {
+        webSocketRoutes.push(path.join(buildDir, file));
+      }
+
+      return [httpRoutes, webSocketRoutes];
+    },
+    [[] as string[], [] as string[]],
+  );
+
+  return {
+    files,
+    httpRoutes,
+    webSocketRoutes,
+  };
+};
+
+export const camelCase = (str: string) =>
+  str.replace(/-([a-z])/g, (_, w) => w.toUpperCase());
+
+export const prompt = async (question: string) => {
+  const promptLog = debug('browserless.io:prompt');
+  const rl = createInterface({
+    input: process.stdin,
+    output: process.stdout,
+  });
+
+  return new Promise((resolve) => {
+    promptLog(question);
+    rl.question('  > ', (a) => {
+      rl.close();
+      resolve(a.trim());
+    });
+  });
+};
+
+export const installDependencies = async (
+  workingDirectory: string,
+): Promise<void> =>
+  new Promise((resolve, reject) => {
+    spawn('npm', ['i'], {
+      cwd: workingDirectory,
+      stdio: 'inherit',
+    }).once('close', (code) => {
+      if (code === 0) {
+        return resolve();
+      }
+      return reject(
+        `Error when installing dependencies, see output for more details`,
+      );
+    });
+  });
+
+export const buildDockerImage = async (
+  cmd: string,
+  projectDir: string,
+): Promise<void> =>
+  new Promise((resolve, reject) => {
+    const [docker, ...args] = cmd.split(' ');
+    spawn(docker, args, {
+      cwd: projectDir,
+      stdio: 'inherit',
+    }).once('close', (code) => {
+      if (code === 0) {
+        return resolve();
+      }
+      return reject(
+        `Error when building Docker image, see output for more details`,
+      );
+    });
+  });
+
+export const buildTypeScript = async (
+  buildDir: string,
+  projectDir: string,
+): Promise<void> =>
+  new Promise((resolve, reject) => {
+    spawn('npx', ['tsc', '--outDir', buildDir], {
+      cwd: projectDir,
+      stdio: 'inherit',
+    }).once('close', (code) => {
+      if (code === 0) {
+        return resolve();
+      }
+      return reject(
+        `Error in building TypeScript, see output for more details`,
+      );
+    });
+  });

package/src/server.ts

@@ -4,6 +4,7 @@ import {
   BadRequest,
   Config,
   HTTPRoute,
+  Hooks,
   Metrics,
   NotFound,
   Request,
@@ -14,7 +15,6 @@ import {
   TooManyRequests,
   Unauthorized,
   WebSocketRoute,
-  beforeRequest,
   contentTypes,
   convertPathToURL,
   createLogger,
@@ -48,6 +48,7 @@ export class HTTPServer extends EventEmitter {
     protected metrics: Metrics,
     protected token: Token,
     protected router: Router,
+    protected hooks: Hooks,
   ) {
     super();
     this.host = config.getHost();
@@ -111,7 +112,7 @@ export class HTTPServer extends EventEmitter {
     );
 
     const req = request as Request;
-    const proceed = await beforeRequest({ req, res });
+    const proceed = await this.hooks.before({ req, res });
     req.parsed = convertPathToURL(request.url || '', this.config);
     shimLegacyRequests(req.parsed);
 
@@ -297,7 +298,7 @@ export class HTTPServer extends EventEmitter {
     this.verbose(`Handling inbound WebSocket request on "${request.url}"`);
 
     const req = request as Request;
-    const proceed = await beforeRequest({ head, req, socket });
+    const proceed = await this.hooks.before({ head, req, socket });
     req.parsed = convertPathToURL(request.url || '', this.config);
     shimLegacyRequests(req.parsed);
 

package/src/shared/content.http.ts

@@ -176,6 +176,10 @@ export default class ChromiumContentPostRoute extends BrowserHTTPRoute {
       });
     }
 
+    const gotoResponse = await gotoCall(content, gotoOptions).catch(
+      bestAttemptCatch(bestAttempt),
+    );
+
     if (addStyleTag.length) {
       for (const tag in addStyleTag) {
         await page.addStyleTag(addStyleTag[tag]);
@@ -188,10 +192,6 @@ export default class ChromiumContentPostRoute extends BrowserHTTPRoute {
       }
     }
 
-    const gotoResponse = await gotoCall(content, gotoOptions).catch(
-      bestAttemptCatch(bestAttempt),
-    );
-
     if (waitForTimeout) {
       await sleep(waitForTimeout).catch(bestAttemptCatch(bestAttempt));
     }

package/src/shared/pdf.http.ts

@@ -36,7 +36,6 @@ export interface BodySchema {
   addStyleTag?: Array<Parameters<Page['addStyleTag']>[0]>;
   authenticate?: Parameters<Page['authenticate']>[0];
   bestAttempt?: bestAttempt;
-  blockModals?: boolean;
   cookies?: Array<Parameters<Page['setCookie']>[0]>;
   emulateMediaType?: Parameters<Page['emulateMediaType']>[0];
   gotoOptions?: Parameters<Page['goto']>[1];
@@ -185,6 +184,10 @@ export default class ChromiumPDFPostRoute extends BrowserHTTPRoute {
       });
     }
 
+    const gotoResponse = await gotoCall(content, gotoOptions).catch(
+      bestAttemptCatch(bestAttempt),
+    );
+
     if (addStyleTag.length) {
       for (const tag in addStyleTag) {
         await page.addStyleTag(addStyleTag[tag]);
@@ -197,10 +200,6 @@ export default class ChromiumPDFPostRoute extends BrowserHTTPRoute {
       }
     }
 
-    const gotoResponse = await gotoCall(content, gotoOptions).catch(
-      bestAttemptCatch(bestAttempt),
-    );
-
     if (waitForTimeout) {
       await sleep(waitForTimeout).catch(bestAttemptCatch(bestAttempt));
     }

package/src/shared/screenshot.http.ts

@@ -191,6 +191,10 @@ export default class ScreenshotPost extends BrowserHTTPRoute {
       });
     }
 
+    const gotoResponse = await gotoCall(content, gotoOptions).catch(
+      bestAttemptCatch(bestAttempt),
+    );
+
     if (addStyleTag.length) {
       for (const tag in addStyleTag) {
         await page.addStyleTag(addStyleTag[tag]);
@@ -203,10 +207,6 @@ export default class ScreenshotPost extends BrowserHTTPRoute {
       }
     }
 
-    const gotoResponse = await gotoCall(content, gotoOptions).catch(
-      bestAttemptCatch(bestAttempt),
-    );
-
     if (waitForTimeout) {
       await sleep(waitForTimeout).catch(bestAttemptCatch(bestAttempt));
     }

package/src/utils.ts

@@ -745,7 +745,8 @@ export const encrypt = (text: string, secret: Buffer) => {
 };
 
 export const decrypt = (encryptedText: string, secret: Buffer) => {
-  const [encrypted, iv] = encryptedText.split(encryptionSep);
+  const [encrypted, iv] = encryptedText.toString().split(encryptionSep);
+  console.log('>>', encryptedText.toString());
   if (!iv) throw new ServerError('Bad or invalid encrypted format');
   const decipher = crypto.createDecipheriv(
     encryptionAlgo,