@browserless.io/browserless 2.0.0-beta-1

package/src/http.ts

@@ -0,0 +1,157 @@
+import {
+  BrowserServerOptions,
+  CDPLaunchOptions,
+} from '@browserless.io/browserless';
+
+import http from 'http';
+
+export const errorCodes = {
+  400: {
+    code: 400,
+    description: `The request contains errors or didn't properly encode content.`,
+    message: 'HTTP/1.1 400 Bad Request',
+  },
+  401: {
+    code: 401,
+    description: `The request is missing, or contains bad, authorization credentials.`,
+    message: 'HTTP/1.1 401 Unauthorized',
+  },
+  404: {
+    code: 404,
+    description: `Resource couldn't be found.`,
+    message: 'HTTP/1.1 404 Not Found',
+  },
+  408: {
+    code: 408,
+    description: `The request took has taken too long to process.`,
+    message: 'HTTP/1.1 408 Request Timeout',
+  },
+  429: {
+    code: 429,
+    description: `Too many requests are currently being processed.`,
+    message: 'HTTP/1.1 429 Too Many Requests',
+  },
+  500: {
+    code: 500,
+    description: `An internal error occurred when handling the request.`,
+    message: 'HTTP/1.1 500 Internal Server Error',
+  },
+} as const;
+
+export const okCodes = {
+  200: {
+    code: 200,
+    description: `The request ran successfully and returned an OK response.`,
+    message: 'HTTP/1.1 200 OK',
+  },
+  204: {
+    code: 204,
+    description: `The request ran successfully, but no response was necessary.`,
+    message: 'HTTP/1.1 204 No Content',
+  },
+} as const;
+
+export const codes = {
+  ...errorCodes,
+  ...okCodes,
+} as const;
+
+export enum contentTypes {
+  any = '*/*',
+  html = 'text/html',
+  javascript = 'application/javascript',
+  jpeg = 'image/jpeg',
+  json = 'application/json',
+  pdf = 'application/pdf',
+  png = 'image/png',
+  text = 'text/plain',
+  zip = 'application/zip',
+}
+
+export enum encodings {
+  utf8 = 'UTF-8',
+}
+
+export enum Methods {
+  delete = 'delete',
+  get = 'get',
+  post = 'post',
+  put = 'put',
+}
+
+export enum WebsocketRoutes {
+  '/' = '?(/)',
+  browser = '/devtools/browser/*',
+  page = '/devtools/page/*',
+  playwrightChromium = '/playwright/chromium',
+  playwrightFirefox = '/playwright/firefox',
+  playwrightWebkit = '/playwright/webkit',
+}
+
+export enum HTTPRoutes {
+  content = '/content',
+  download = '/download',
+  function = '/function',
+  pdf = '/pdf',
+  performance = '/performance',
+  scrape = '/scrape',
+  screenshot = '/screenshot',
+}
+
+export enum HTTPManagementRoutes {
+  config = '/config',
+  metrics = '/metrics',
+  metricsTotal = '/metrics/total',
+  sessions = '/sessions',
+  static = '/',
+}
+
+export enum APITags {
+  'browserAPI' = 'Browser APIs',
+  'browserWS' = 'Browser WebSockets',
+  'management' = 'Management APIs',
+}
+
+export interface Request extends http.IncomingMessage {
+  body: unknown;
+  parsed: URL;
+  queryParams: Record<string, unknown>;
+}
+
+export type Response = http.ServerResponse;
+
+export interface SystemQueryParameters {
+  /**
+   * Whether or nor to load ad-blocking extensions for the session.
+   * This currently uses uBlock Origin and may cause certain sites
+   * to not load properly.
+   */
+  blockAds?: boolean;
+
+  /**
+   * Launch options, which can be either an object
+   * of puppeteer.launch options or playwright.launchServer
+   * options, depending on the API. Must be either JSON
+   * object, or a base64-encoded JSON object.
+   */
+  launch?: CDPLaunchOptions | BrowserServerOptions | string;
+
+  /**
+   * Whether or nor to record the session. The browser will run
+   * in "head-full" mode, and recording is started and closed
+   * via the embedded browserless API. Please refer to the "Recording"
+   * section in the live documentation site for more details.
+   */
+  record?: boolean;
+
+  /**
+   * Override the system-level timeout for this request.
+   * Accepts a value in milliseconds.
+   */
+  timeout?: number;
+
+  /**
+   * The authorization token
+   */
+  token?: string;
+}

package/src/index.ts

@@ -0,0 +1,41 @@
+import { Browserless, createLogger } from '@browserless.io/browserless';
+
+(async () => {
+  const browserless = new Browserless();
+  const debug = createLogger('index.js');
+  browserless.start();
+
+  process
+    .on('unhandledRejection', async (reason, promise) => {
+      console.error('Unhandled Rejection at:', promise, 'reason:', reason);
+    })
+    .once('uncaughtException', async (err, origin) => {
+      console.error('Unhandled exception at:', origin, 'error:', err);
+      await browserless.stop();
+      process.exit(1);
+    })
+    .once('SIGTERM', async () => {
+      debug(`SIGTERM received, saving and closing down`);
+      await browserless.stop();
+      process.exit(0);
+    })
+    .once('SIGINT', async () => {
+      debug(`SIGINT received, saving and closing down`);
+      await browserless.stop();
+      process.exit(0);
+    })
+    .once('SIGHUP', async () => {
+      debug(`SIGHUP received, saving and closing down`);
+      await browserless.stop();
+      process.exit(0);
+    })
+    .once('SIGUSR2', async () => {
+      debug(`SIGUSR2 received, saving and closing down`);
+      await browserless.stop();
+      process.exit(0);
+    })
+    .once('exit', () => {
+      debug(`Process is finished, exiting`);
+      process.exit(0);
+    });
+})();

package/src/limiter.spec.ts

@@ -0,0 +1,365 @@
+import {
+  Config,
+  Limiter,
+  Metrics,
+  Monitoring,
+  WebHooks,
+  sleep,
+} from '@browserless.io/browserless';
+import { expect } from 'chai';
+import { spy } from 'sinon';
+
+const asyncNoop = () => Promise.resolve(undefined);
+const noop = () => undefined;
+const webHooks = {
+  callErrorAlertURL: spy(),
+  callFailedHealthURL: spy(),
+  callQueueAlertURL: spy(),
+  callRejectAlertURL: spy(),
+  callTimeoutAlertURL: spy(),
+};
+
+describe(`Limiter`, () => {
+  afterEach(() => {
+    webHooks.callFailedHealthURL.resetHistory();
+    webHooks.callQueueAlertURL.resetHistory();
+    webHooks.callRejectAlertURL.resetHistory();
+    webHooks.callTimeoutAlertURL.resetHistory();
+    webHooks.callErrorAlertURL.resetHistory();
+  });
+
+  it('limits and queues function calls and calls queue alert urls', async () => {
+    return new Promise((resolve, reject) => {
+      const config = new Config();
+      config.setQueueAlertURL('https://example.com');
+
+      const monitoring = new Monitoring(config);
+      const metrics = new Metrics();
+
+      config.setConcurrent(1);
+      config.setQueued(1);
+      config.setTimeout(-1);
+
+      const limiter = new Limiter(
+        config,
+        metrics,
+        monitoring,
+        webHooks as unknown as WebHooks,
+      );
+      const handler = spy();
+      const job = limiter.limit(handler, asyncNoop, asyncNoop, noop);
+
+      job();
+      job();
+
+      expect(handler.calledOnce).to.be.true;
+
+      limiter.addEventListener('end', () => {
+        try {
+          expect(handler.calledTwice).to.be.true;
+          expect(webHooks.callQueueAlertURL.calledOnce).to.be.true;
+          expect(metrics.get().queued).to.equal(1);
+          expect(metrics.get().maxConcurrent).to.equal(1);
+          expect(metrics.get().successful).to.equal(2);
+        } catch (e) {
+          return reject(e);
+        }
+        resolve(undefined);
+      });
+    });
+  }).timeout(5000);
+
+  it('passes through arguments', () => {
+    const args = ['one', 'two', 'three'];
+    const config = new Config();
+    const metrics = new Metrics();
+    const monitoring = new Monitoring(config);
+    config.setConcurrent(1);
+    config.setQueued(0);
+    config.setTimeout(-1);
+
+    const limiter = new Limiter(
+      config,
+      metrics,
+      monitoring,
+      webHooks as unknown as WebHooks,
+    );
+    const handler = spy();
+    const job = limiter.limit(handler, asyncNoop, asyncNoop, noop);
+    // @ts-ignore will fix later
+    job(...args);
+    expect(handler.args[0]).to.eql(args);
+  });
+
+  it('waits to run jobs until the first are done', async () => {
+    const config = new Config();
+    const monitoring = new Monitoring(config);
+    const metrics = new Metrics();
+    config.setConcurrent(1);
+    config.setQueued(1);
+    config.setTimeout(-1);
+
+    const limiter = new Limiter(
+      config,
+      metrics,
+      monitoring,
+      webHooks as unknown as WebHooks,
+    );
+    const handlerOne = () => new Promise((r) => setTimeout(r, 50));
+    const handlerTwo = spy();
+
+    const jobOne = limiter.limit(handlerOne, asyncNoop, asyncNoop, noop);
+    const jobTwo = limiter.limit(handlerTwo, asyncNoop, asyncNoop, noop);
+
+    const wait = jobOne();
+    jobTwo();
+
+    expect(handlerTwo.calledOnce).to.be.false;
+    await wait;
+    expect(handlerTwo.calledOnce).to.be.true;
+  });
+
+  it('bubbles up errors', async () => {
+    const config = new Config();
+    const monitoring = new Monitoring(config);
+    const metrics = new Metrics();
+    const error = new Error('WOW');
+
+    config.setConcurrent(1);
+    config.setQueued(1);
+    config.setTimeout(-1);
+
+    const limiter = new Limiter(
+      config,
+      metrics,
+      monitoring,
+      webHooks as unknown as WebHooks,
+    );
+    const spy = () => new Promise((_r, rej) => rej(error));
+
+    const job = limiter.limit(spy, asyncNoop, asyncNoop, noop);
+    await job().catch(noop);
+
+    limiter.addEventListener('end', (res) => {
+      // 'end' callback fires before the failed callback hooks do, so nextTick to wait
+      process.nextTick(() => {
+        expect(res.detail.error).to.eql(error);
+        expect(metrics.get().successful).to.eql(0);
+        expect(metrics.get().error).to.eql(1);
+        expect(webHooks.callErrorAlertURL.getCalls()[0].firstArg).to.include(
+          error,
+        );
+        expect(webHooks.callErrorAlertURL.calledOnce).to.be.true;
+      });
+    });
+  });
+
+  it('calls an error handler with arguments if there are too many function calls', () => {
+    const args = ['one', 'two', 'three'];
+    const config = new Config();
+    const monitoring = new Monitoring(config);
+    const metrics = new Metrics();
+    config.setConcurrent(1);
+    config.setQueued(0);
+    config.setTimeout(-1);
+
+    const limiter = new Limiter(
+      config,
+      metrics,
+      monitoring,
+      webHooks as unknown as WebHooks,
+    );
+
+    const handler = spy();
+    const onError = spy();
+    const job = limiter.limit(handler, onError, noop, noop);
+
+    job();
+    // @ts-ignore
+    job(...args);
+
+    expect(webHooks.callRejectAlertURL.calledOnce).to.be.true;
+    expect(handler.calledOnce).to.be.true;
+    expect(onError.calledOnce).to.be.true;
+    expect(onError.args[0]).to.eql(args);
+  });
+
+  it('calls a timeout handler with arguments if a job takes too long', (r) => {
+    const args = ['one', 'two', 'three'];
+    const config = new Config();
+    const metrics = new Metrics();
+    const monitoring = new Monitoring(config);
+    config.setConcurrent(1);
+    config.setQueued(0);
+    config.setTimeout(10);
+
+    let timer: NodeJS.Timer;
+    const limiter = new Limiter(
+      config,
+      metrics,
+      monitoring,
+      webHooks as unknown as WebHooks,
+    );
+    const handler = () =>
+      new Promise((d) => (timer = global.setTimeout(d, 1000)));
+
+    const onTimeout = (...calledArgs: unknown[]) => {
+      clearTimeout(timer as unknown as number);
+      expect(calledArgs).to.eql(args);
+      expect(webHooks.callTimeoutAlertURL.calledOnce).to.be.true;
+      r(null);
+    };
+
+    const job = limiter.limit(handler, noop, onTimeout, noop);
+
+    // @ts-ignore
+    job(...args);
+  });
+
+  it('allows overriding the timeouts', async () => {
+    const config = new Config();
+    const metrics = new Metrics();
+    const monitoring = new Monitoring(config);
+    config.setConcurrent(2);
+    config.setQueued(0);
+    config.setTimeout(1);
+
+    const limiter = new Limiter(
+      config,
+      metrics,
+      monitoring,
+      webHooks as unknown as WebHooks,
+    );
+    const onTimeout = spy();
+    const handler = async () => new Promise((r) => setTimeout(r, 10));
+
+    const job = limiter.limit(handler, noop, onTimeout, noop);
+    const jobTwo = limiter.limit(handler, noop, onTimeout, () => 20);
+
+    job();
+    jobTwo();
+
+    await sleep(25);
+
+    expect(onTimeout.calledOnce).to.be.true;
+  });
+
+  it(`doesn't call a timeout handler if the job finishes in time`, async () => {
+    const config = new Config();
+    const metrics = new Metrics();
+    const monitoring = new Monitoring(config);
+    config.setConcurrent(1);
+    config.setQueued(0);
+    config.setTimeout(20);
+
+    const limiter = new Limiter(
+      config,
+      metrics,
+      monitoring,
+      webHooks as unknown as WebHooks,
+    );
+    const handler = () => new Promise((r) => setTimeout(r, 1));
+    const timeout = spy();
+    const job = limiter.limit(handler, noop, timeout, noop);
+    await job();
+
+    expect(timeout.called).not.to.be.true;
+    expect(webHooks.callTimeoutAlertURL.calledOnce).not.to.be.true;
+  });
+
+  it(`won't add items to the queue when reached`, () =>
+    new Promise((r) => {
+      const config = new Config();
+      const monitoring = new Monitoring(config);
+      const metrics = new Metrics();
+      config.setConcurrent(1);
+      config.setQueued(0);
+      config.setTimeout(-1);
+
+      const limiter = new Limiter(
+        config,
+        metrics,
+        monitoring,
+        webHooks as unknown as WebHooks,
+      );
+
+      const handler = spy();
+      const job = limiter.limit(handler, noop, noop, noop);
+
+      job();
+      job();
+
+      expect(limiter.length).to.equal(1);
+
+      limiter.addEventListener('end', () => {
+        expect(limiter.length).to.equal(0);
+        expect(handler.calledOnce).to.be.true;
+        expect(webHooks.callRejectAlertURL.calledOnce).to.be.true;
+        r(undefined);
+      });
+    }));
+
+  it(`won't add items when health is bad`, () =>
+    new Promise((r) => {
+      const config = new Config();
+      config.enableHealthChecks(true);
+      config.setCPULimit(1);
+      config.setMemoryLimit(1);
+
+      const monitoring = new Monitoring(config);
+      const metrics = new Metrics();
+      config.setConcurrent(10);
+      config.setQueued(10);
+      config.setTimeout(-1);
+
+      const limiter = new Limiter(
+        config,
+        metrics,
+        monitoring,
+        webHooks as unknown as WebHooks,
+      );
+
+      const handler = spy();
+      const job = limiter.limit(handler, noop, noop, noop);
+
+      job().catch(() => {
+        expect(limiter.length).to.equal(0);
+        expect(handler.calledOnce).to.be.false;
+        expect(webHooks.callFailedHealthURL.calledOnce).to.be.true;
+        r(undefined);
+      });
+    }));
+
+  it(`will add items when health is good`, () =>
+    new Promise((r) => {
+      const config = new Config();
+      config.enableHealthChecks(true);
+      config.setCPULimit(100);
+      config.setMemoryLimit(100);
+
+      const metrics = new Metrics();
+      const monitoring = new Monitoring(config);
+      config.setConcurrent(10);
+      config.setQueued(10);
+      config.setTimeout(-1);
+
+      const limiter = new Limiter(
+        config,
+        metrics,
+        monitoring,
+        webHooks as unknown as WebHooks,
+      );
+
+      const handler = spy();
+      const job = limiter.limit(handler, noop, noop, noop);
+
+      job();
+
+      limiter.addEventListener('end', () => {
+        expect(limiter.length).to.equal(0);
+        expect(handler.calledOnce).to.be.true;
+        expect(webHooks.callFailedHealthURL.calledOnce).to.be.false;
+        r(undefined);
+      });
+    }));
+});