@browserless.io/browserless 2.0.0-beta-1

package/src/routes/chromium/utils/performance/main.ts

@@ -0,0 +1,90 @@
+import { Message, mainOptions } from './types.js';
+import { createLogger } from '@browserless.io/browserless';
+import { fork } from 'child_process';
+import path from 'path';
+
+const DEFAULT_AUDIT_CONFIG = {
+  extends: 'lighthouse:default',
+};
+
+const debug = createLogger('http:performance:main');
+
+export default async ({
+  browser,
+  context,
+  timeout,
+}: mainOptions): Promise<unknown> => {
+  return new Promise((resolve, reject) => {
+    const childPath = path.join(
+      './',
+      'build',
+      'routes',
+      'chromium',
+      'utils',
+      'performance',
+      'child.js',
+    );
+
+    debug(`Starting up child at ${childPath}`);
+
+    const child = fork(childPath);
+    const port = new URL(browser.wsEndpoint() || '').port;
+
+    let closed = false;
+    let timeoutId =
+      timeout !== -1
+        ? setTimeout(() => {
+            close(child.pid);
+          }, timeout)
+        : null;
+
+    const close = (pid?: number) => {
+      if (closed) return;
+      if (pid) process.kill(pid, 'SIGINT');
+      timeoutId && clearTimeout(timeoutId);
+      closed = true;
+      timeoutId = null;
+    };
+
+    const { url, config = DEFAULT_AUDIT_CONFIG, budgets } = context;
+
+    const options = {
+      budgets,
+      logLevel: 'info',
+      output: 'json',
+      port,
+    };
+
+    child.on('error', (err) => {
+      debug(`Error in child process`, err);
+      reject('Performance run error: ' + err.message);
+      close(child.pid);
+    });
+
+    child.on('message', (payload: Message) => {
+      if (payload.event === 'created') {
+        debug(`Child process is up, sending performance request`);
+        return child.send({
+          config,
+          event: 'start',
+          options,
+          url,
+        });
+      }
+
+      if (payload.event === 'complete') {
+        debug(`Performance gathered, closing and resolving request`);
+        close(child.pid);
+        return resolve({
+          data: payload.data,
+          type: 'json',
+        });
+      }
+
+      if (payload.event === 'error') {
+        close(child.pid);
+        reject(new Error('Error running performance metrics ' + payload.error));
+      }
+    });
+  });
+};

package/src/routes/chromium/utils/performance/types.ts

@@ -0,0 +1,24 @@
+import { Config, Flags } from 'lighthouse';
+import { BrowserInstance } from '@browserless.io/browserless';
+
+export interface Message {
+  data?: unknown;
+  error?: unknown;
+  event: string;
+}
+
+export interface mainOptions {
+  browser: BrowserInstance;
+  context: {
+    budgets?: Array<unknown>;
+    config?: unknown;
+    url: string;
+  };
+  timeout: number;
+}
+
+export interface start {
+  config?: Config;
+  options?: Flags;
+  url: string;
+}

package/src/routes/chromium/ws/browser.ts

@@ -0,0 +1,31 @@
+import {
+  APITags,
+  BrowserWebsocketRoute,
+  CDPChromium,
+  CDPLaunchOptions,
+  Request,
+  SystemQueryParameters,
+  WebsocketRoutes,
+} from '@browserless.io/browserless';
+import { Duplex } from 'stream';
+
+export interface QuerySchema extends SystemQueryParameters {
+  launch?: CDPLaunchOptions | string;
+}
+
+const route: BrowserWebsocketRoute = {
+  auth: true,
+  browser: CDPChromium,
+  concurrency: false,
+  description: `Connect to an already-running Chromium with a library like puppeteer, or others, that work over chrome-devtools-protocol.`,
+  handler: async (
+    req: Request,
+    socket: Duplex,
+    head: Buffer,
+    chrome: CDPChromium,
+  ): Promise<void> => chrome.proxyWebSocket(req, socket, head),
+  path: WebsocketRoutes.browser,
+  tags: [APITags.browserWS],
+};
+
+export default route;

package/src/routes/chromium/ws/cdp-chromium.ts

@@ -0,0 +1,31 @@
+import {
+  APITags,
+  BrowserWebsocketRoute,
+  CDPChromium,
+  CDPLaunchOptions,
+  Request,
+  SystemQueryParameters,
+  WebsocketRoutes,
+} from '@browserless.io/browserless';
+import { Duplex } from 'stream';
+
+export interface QuerySchema extends SystemQueryParameters {
+  launch?: CDPLaunchOptions | string;
+}
+
+const route: BrowserWebsocketRoute = {
+  auth: true,
+  browser: CDPChromium,
+  concurrency: true,
+  description: `Launch and connect to Chromium with a library like puppeteer or others that work over chrome-devtools-protocol.`,
+  handler: async (
+    req: Request,
+    socket: Duplex,
+    head: Buffer,
+    chrome: CDPChromium,
+  ): Promise<void> => chrome.proxyWebSocket(req, socket, head),
+  path: WebsocketRoutes['/'],
+  tags: [APITags.browserWS],
+};
+
+export default route;

package/src/routes/chromium/ws/page.ts

@@ -0,0 +1,31 @@
+import {
+  APITags,
+  BrowserWebsocketRoute,
+  CDPChromium,
+  CDPLaunchOptions,
+  Request,
+  SystemQueryParameters,
+  WebsocketRoutes,
+} from '@browserless.io/browserless';
+import { Duplex } from 'stream';
+
+export interface QuerySchema extends SystemQueryParameters {
+  launch?: CDPLaunchOptions | string;
+}
+
+const route: BrowserWebsocketRoute = {
+  auth: true,
+  browser: CDPChromium,
+  concurrency: false,
+  description: `Connect to Chromium with a library like chrome-remote-interface or others that work over JSON chrome-devtools-protocol.`,
+  handler: async (
+    req: Request,
+    socket: Duplex,
+    head: Buffer,
+    chrome: CDPChromium,
+  ): Promise<void> => chrome.proxyPageWebSocket(req, socket, head),
+  path: WebsocketRoutes.page,
+  tags: [APITags.browserWS],
+};
+
+export default route;

package/src/routes/chromium/ws/playwright-chromium.ts

@@ -0,0 +1,44 @@
+import {
+  APITags,
+  BadRequest,
+  BrowserServerOptions,
+  BrowserWebsocketRoute,
+  PlaywrightChromium,
+  Request,
+  SystemQueryParameters,
+  WebsocketRoutes,
+} from '@browserless.io/browserless';
+import { Duplex } from 'stream';
+
+export interface QuerySchema extends SystemQueryParameters {
+  launch?: BrowserServerOptions | string;
+}
+
+const route: BrowserWebsocketRoute = {
+  auth: true,
+  browser: PlaywrightChromium,
+  concurrency: true,
+  description: `Connect to Chromium with any playwright-compliant library.`,
+  handler: async (
+    req: Request,
+    socket: Duplex,
+    head: Buffer,
+    browser: PlaywrightChromium,
+  ): Promise<void> => {
+    const isPlaywright = req.headers['user-agent']
+      ?.toLowerCase()
+      .includes('playwright');
+
+    if (!isPlaywright) {
+      throw new BadRequest(
+        `Only playwright is allowed to work with this route`,
+      );
+    }
+
+    return browser.proxyWebSocket(req, socket, head);
+  },
+  path: WebsocketRoutes.playwrightChromium,
+  tags: [APITags.browserWS],
+};
+
+export default route;

package/src/routes/firefox/tests/websocket.spec.ts

@@ -0,0 +1,146 @@
+import {
+  Browserless,
+  Config,
+  Metrics,
+  sleep,
+} from '@browserless.io/browserless';
+import { expect } from 'chai';
+import { firefox } from 'playwright-core';
+
+describe('Firefox Websocket API', function () {
+  // Server shutdown can take a few seconds
+  // and so can these tests :/
+  this.timeout(5000);
+
+  let browserless: Browserless;
+
+  const start = ({
+    config = new Config(),
+    metrics = new Metrics(),
+  }: { config?: Config; metrics?: Metrics } = {}) => {
+    browserless = new Browserless({ config, metrics });
+    return browserless.start();
+  };
+
+  afterEach(async () => {
+    await browserless.stop();
+  });
+
+  it('runs firefox websocket requests', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const browser = await firefox.connect(
+      `ws://localhost:3000/playwright/firefox?token=browserless`,
+    );
+
+    await browser.close();
+  });
+
+  it('rejects playwright requests', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const didError = await firefox
+      .connect(`ws://localhost:3000/playwright/firefox?token=bad`)
+      .then(() => false)
+      .catch(() => true);
+
+    expect(didError).to.be.true;
+  });
+
+  it('runs with job-based timeouts', async () => {
+    const config = new Config();
+    const metrics = new Metrics();
+    config.setTimeout(-1); // No timeout
+    config.setToken('browserless');
+    await start({ config, metrics });
+
+    const browser = await firefox
+      .connect(
+        `ws://localhost:3000/playwright/firefox?timeout=500&token=browserless`,
+      )
+      .catch(() => null);
+
+    await sleep(750);
+    browser && browser.close();
+    expect(metrics.get().timedout).to.equal(1);
+    expect(metrics.get().successful).to.equal(0);
+  });
+
+  it('queues requests', async () => {
+    const config = new Config();
+    const metrics = new Metrics();
+    config.setConcurrent(1);
+    config.setToken('browserless');
+    await start({ config, metrics });
+
+    const job = async () => {
+      const browser = await firefox.connect(
+        `ws://localhost:3000/playwright/firefox?token=browserless`,
+      );
+      await sleep(100);
+
+      return browser.close();
+    };
+
+    await Promise.all([job(), job()]);
+
+    await sleep(100);
+
+    const results = metrics.get();
+    expect(results.successful).to.equal(2);
+    expect(results.rejected).to.equal(0);
+    expect(results.queued).to.equal(1);
+  });
+
+  it('fails requests', async () => {
+    const config = new Config();
+    config.setConcurrent(0);
+    config.setQueued(0);
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    return firefox
+      .connect(`ws://localhost:3000/playwright/firefox?token=browserless`)
+      .catch((error: Error) => {
+        const results = metrics.get();
+        expect(results.successful).to.equal(0);
+        expect(results.rejected).to.equal(1);
+        expect(results.queued).to.equal(0);
+        expect(error.message).to.contain.oneOf([`400`, `429`]);
+      });
+  });
+
+  it('fails requests without tokens', async () => {
+    const metrics = new Metrics();
+    const config = new Config();
+    config.setToken('browserless');
+    await start({ config, metrics });
+
+    return firefox
+      .connect(`ws://localhost:3000/playwright/firefox`)
+      .catch((error: Error) => {
+        const results = metrics.get();
+        expect(results.successful).to.equal(0);
+        expect(results.rejected).to.equal(0);
+        expect(results.queued).to.equal(0);
+        expect(error.message).to.contain(`401`);
+      });
+  });
+
+  it('allows requests without token when auth token is not set', async () => {
+    await start();
+
+    const browser = await firefox.connect(
+      `ws://localhost:3000/playwright/firefox`,
+    );
+
+    await browser.close();
+  });
+});

package/src/routes/firefox/ws/playwright-firefox.ts

@@ -0,0 +1,44 @@
+import {
+  APITags,
+  BadRequest,
+  BrowserServerOptions,
+  BrowserWebsocketRoute,
+  PlaywrightFirefox,
+  Request,
+  SystemQueryParameters,
+  WebsocketRoutes,
+} from '@browserless.io/browserless';
+import { Duplex } from 'stream';
+
+export interface QuerySchema extends SystemQueryParameters {
+  launch?: BrowserServerOptions | string;
+}
+
+const route: BrowserWebsocketRoute = {
+  auth: true,
+  browser: PlaywrightFirefox,
+  concurrency: true,
+  description: `Connect to Firefox with any playwright-compliant library.`,
+  handler: async (
+    req: Request,
+    socket: Duplex,
+    head: Buffer,
+    browser: PlaywrightFirefox,
+  ): Promise<void> => {
+    const isPlaywright = req.headers['user-agent']
+      ?.toLowerCase()
+      .includes('playwright');
+
+    if (!isPlaywright) {
+      throw new BadRequest(
+        `Only playwright is allowed to work with this route`,
+      );
+    }
+
+    return browser.proxyWebSocket(req, socket, head);
+  },
+  path: WebsocketRoutes.playwrightFirefox,
+  tags: [APITags.browserWS],
+};
+
+export default route;

package/src/routes/management/http/config-get.ts

@@ -0,0 +1,82 @@
+import {
+  APITags,
+  HTTPManagementRoutes,
+  HTTPRoute,
+  Methods,
+  Request,
+  ServerError,
+  contentTypes,
+  jsonResponse,
+} from '@browserless.io/browserless';
+import { ServerResponse } from 'http';
+
+export interface ResponseSchema {
+  allowCORS: boolean;
+  allowFileProtocol: boolean;
+  allowGetCalls: boolean;
+  concurrent: number;
+  data: string;
+  debug: string;
+  errorAlertURL: string | null;
+  healthFailureURL: string | null;
+  host: string;
+  maxCPU: number;
+  maxMemory: number;
+  metricsJSONPath: string;
+  port: number;
+  queued: number;
+  queuedAlertURL: string | null;
+  rejectAlertURL: string | null;
+  retries: number;
+  timeout: number;
+  timeoutAlertURL: string | null;
+  token: string | null;
+}
+
+const route: HTTPRoute = {
+  accepts: [contentTypes.any],
+  auth: true,
+  browser: null,
+  concurrency: false,
+  contentTypes: [contentTypes.json],
+  description: `Returns a JSON payload of the current system configuration.`,
+  handler: async (_req: Request, res: ServerResponse): Promise<void> => {
+    const { _config: getConfig } = route;
+
+    if (!getConfig) {
+      throw new ServerError(`Couldn't locate the config object`);
+    }
+
+    const config = getConfig();
+
+    const response: ResponseSchema = {
+      allowCORS: config.getAllowCORS(),
+      allowFileProtocol: config.getAllowFileProtocol(),
+      allowGetCalls: config.getAllowGetCalls(),
+      concurrent: config.getConcurrent(),
+      data: await config.getDataDir(),
+      debug: config.getDebug(),
+      errorAlertURL: config.getErrorAlertURL(),
+      healthFailureURL: config.getFailedHealthURL(),
+      host: config.getHost(),
+      maxCPU: config.getCPULimit(),
+      maxMemory: config.getMemoryLimit(),
+      metricsJSONPath: config.getMetricsJSONPath(),
+      port: config.getPort(),
+      queued: config.getQueued(),
+      queuedAlertURL: config.getQueueAlertURL(),
+      rejectAlertURL: config.getRejectAlertURL(),
+      retries: config.getRetries(),
+      timeout: config.getTimeout(),
+      timeoutAlertURL: config.getTimeoutAlertURL(),
+      token: config.getToken(),
+    };
+
+    return jsonResponse(res, 200, response);
+  },
+  method: Methods.get,
+  path: HTTPManagementRoutes.config,
+  tags: [APITags.management],
+};
+
+export default route;

package/src/routes/management/http/metrics-get.ts

@@ -0,0 +1,43 @@
+import {
+  APITags,
+  HTTPManagementRoutes,
+  HTTPRoute,
+  IBrowserlessStats,
+  Methods,
+  Request,
+  ServerError,
+  contentTypes,
+  writeResponse,
+} from '@browserless.io/browserless';
+import { ServerResponse } from 'http';
+
+export type ResponseSchema = Array<IBrowserlessStats>;
+
+const route: HTTPRoute = {
+  accepts: [contentTypes.any],
+  auth: true,
+  browser: null,
+  concurrency: false,
+  contentTypes: [contentTypes.json],
+  description: `Gets total metric details from the time the server started.`,
+  handler: async (_req: Request, res: ServerResponse): Promise<void> => {
+    const { _fileSystem, _config } = route;
+
+    if (!_fileSystem || !_config) {
+      throw new ServerError(`Couldn't locate the file-system or config module`);
+    }
+
+    const fileSystem = _fileSystem();
+    const config = _config();
+
+    const stats = await fileSystem.read(config.getMetricsJSONPath());
+    const response = `[${stats.join(',')}]`;
+
+    return writeResponse(res, 200, response, contentTypes.json);
+  },
+  method: Methods.get,
+  path: HTTPManagementRoutes.metrics,
+  tags: [APITags.management],
+};
+
+export default route;

package/src/routes/management/http/metrics-total-get.ts

@@ -0,0 +1,89 @@
+import {
+  APITags,
+  HTTPManagementRoutes,
+  HTTPRoute,
+  IBrowserlessMetricTotals,
+  Methods,
+  Request,
+  ServerError,
+  contentTypes,
+  writeResponse,
+} from '@browserless.io/browserless';
+import { ServerResponse } from 'http';
+
+export type ResponseSchema = IBrowserlessMetricTotals;
+
+const fiveMinuteIntervalsInAMonth = 8640;
+
+const route: HTTPRoute = {
+  accepts: [contentTypes.any],
+  auth: true,
+  browser: null,
+  concurrency: false,
+  contentTypes: [contentTypes.json],
+  description: `Gets total metric details summed from the time the server started.`,
+  handler: async (_req: Request, res: ServerResponse): Promise<void> => {
+    const { _fileSystem, _config } = route;
+
+    if (!_fileSystem || !_config) {
+      throw new ServerError(`Couldn't locate the file-system or config module`);
+    }
+
+    const fileSystem = _fileSystem();
+    const config = _config();
+
+    const metrics = (await fileSystem.read(config.getMetricsJSONPath())).map(
+      (m) => JSON.parse(m),
+    );
+    const availableMetrics = metrics.length;
+
+    const totals: IBrowserlessMetricTotals = metrics.reduce(
+      (accum, metric) => ({
+        error: accum.error + metric.error,
+        estimatedMonthlyUnits: accum.estimatedMonthlyUnits,
+        maxConcurrent: Math.max(accum.maxConcurrent, metric.maxConcurrent),
+        maxTime: Math.max(accum.maxTime, metric.maxTime),
+        meanTime: accum.meanTime + metric.meanTime,
+        minTime: Math.min(accum.minTime, metric.minTime),
+        minutesOfMetricsAvailable: accum.minutesOfMetricsAvailable + 5,
+        queued: accum.queued + metric.queued,
+        rejected: accum.rejected + metric.rejected,
+        sessionTimes: [...accum.sessionTimes, ...metric.sessionTimes],
+        successful: accum.successful + metric.successful,
+        timedout: accum.timedout + metric.timedout,
+        totalTime: accum.totalTime + metric.totalTime,
+        unhealthy: accum.unhealthy + metric.unhealthy,
+        units: accum.units + metric.units,
+      }),
+      {
+        error: 0,
+        estimatedMonthlyUnits: 0,
+        maxConcurrent: 0,
+        maxTime: 0,
+        meanTime: 0,
+        minTime: 0,
+        minutesOfMetricsAvailable: 0,
+        queued: 0,
+        rejected: 0,
+        sessionTimes: [],
+        successful: 0,
+        timedout: 0,
+        totalTime: 0,
+        unhealthy: 0,
+        units: 0,
+      },
+    );
+
+    totals.meanTime = totals.meanTime / metrics.length;
+    totals.estimatedMonthlyUnits = Math.round(
+      totals.units / (availableMetrics / fiveMinuteIntervalsInAMonth),
+    );
+
+    return writeResponse(res, 200, JSON.stringify(totals), contentTypes.json);
+  },
+  method: Methods.get,
+  path: HTTPManagementRoutes.metricsTotal,
+  tags: [APITags.management],
+};
+
+export default route;

package/src/routes/management/http/sessions-get.ts

@@ -0,0 +1,39 @@
+import {
+  APITags,
+  BadRequest,
+  BrowserlessSessionJSON,
+  HTTPManagementRoutes,
+  HTTPRoute,
+  Methods,
+  Request,
+  contentTypes,
+  jsonResponse,
+} from '@browserless.io/browserless';
+import { ServerResponse } from 'http';
+
+export type ResponseSchema = BrowserlessSessionJSON[];
+
+const route: HTTPRoute = {
+  accepts: [contentTypes.any],
+  auth: true,
+  browser: null,
+  concurrency: false,
+  contentTypes: [contentTypes.json],
+  description: `Lists all currently running sessions and relevant meta-data excluding potentially open pages.`,
+  handler: async (req: Request, res: ServerResponse): Promise<void> => {
+    const { _browserManager: browserManager } = route;
+
+    if (!browserManager) {
+      throw new BadRequest(`Couldn't load browsers running`);
+    }
+
+    const response: ResponseSchema = await browserManager().getAllSessions(req);
+
+    return jsonResponse(res, 200, response);
+  },
+  method: Methods.get,
+  path: HTTPManagementRoutes.sessions,
+  tags: [APITags.management],
+};
+
+export default route;