@browserless.io/browserless 2.0.0-beta-1

package/src/routes/chromium/tests/websocket.spec.ts

@@ -0,0 +1,407 @@
+import {
+  Browserless,
+  Config,
+  Metrics,
+  exists,
+  sleep,
+} from '@browserless.io/browserless';
+import { chromium } from 'playwright-core';
+import { deleteAsync } from 'del';
+import { expect } from 'chai';
+import puppeteer from 'puppeteer-core';
+
+describe('WebSocket API', function () {
+  // Server shutdown can take a few seconds
+  // and so can these tests :/
+  this.timeout(5000);
+
+  let browserless: Browserless;
+
+  const start = ({
+    config = new Config(),
+    metrics = new Metrics(),
+  }: { config?: Config; metrics?: Metrics } = {}) => {
+    browserless = new Browserless({ config, metrics });
+    return browserless.start();
+  };
+
+  afterEach(async () => {
+    await browserless.stop();
+  });
+
+  it('runs chromium websocket requests', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const browser = await puppeteer.connect({
+      browserWSEndpoint: `ws://localhost:3000?token=browserless`,
+    });
+
+    await browser.disconnect();
+  });
+
+  it('runs chromium CDP requests', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const browser = await chromium.connectOverCDP(
+      `ws://localhost:3000?token=browserless`,
+    );
+
+    await browser.close();
+  });
+
+  it('runs chromium websocket requests', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const browser = await puppeteer.connect({
+      browserWSEndpoint: `ws://localhost:3000?token=browserless`,
+    });
+
+    await browser.disconnect();
+  });
+
+  it('runs multiple websocket requests', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const browser = await puppeteer.connect({
+      browserWSEndpoint: `ws://localhost:3000?token=browserless`,
+    });
+
+    const browserTwo = await puppeteer.connect({
+      browserWSEndpoint: `ws://localhost:3000?token=browserless`,
+    });
+
+    await Promise.all([browser.disconnect(), browserTwo.disconnect()]);
+  });
+
+  it('rejects websocket requests', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const didError = await puppeteer
+      .connect({
+        browserWSEndpoint: `ws://localhost:3000?token=bad`,
+      })
+      .then(() => false)
+      .catch(() => true);
+
+    expect(didError).to.be.true;
+  });
+
+  it('rejects file protocol requests', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const didError = await puppeteer
+      .connect({
+        browserWSEndpoint: `ws://localhost:3000?token=browserless`,
+      })
+      .then(async (b) => {
+        const page = await b.newPage();
+        await page.goto('file:///etc/passwd');
+        await page.content();
+        await b.disconnect();
+        return false;
+      })
+      .catch(() => true);
+
+    expect(didError).to.be.true;
+  });
+
+  it('runs with ignored arguments', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+    const args = {
+      ignoreDefaultArgs: true,
+    };
+
+    const success = await puppeteer
+      .connect({
+        browserWSEndpoint: `ws://localhost:3000?token=browserless&launch=${JSON.stringify(
+          args,
+        )}`,
+      })
+      .then(async (b) => {
+        const page = await b.newPage();
+        await page.close();
+        await b.disconnect();
+        return true;
+      })
+      .catch(() => false);
+
+    expect(success).to.be.true;
+  });
+
+  it('deletes user-data-dirs when not specified', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const browser = await puppeteer.connect({
+      browserWSEndpoint: `ws://localhost:3000?token=browserless`,
+    });
+
+    const [{ userDataDir }] = await fetch(
+      'http://localhost:3000/sessions?token=browserless',
+    ).then((r) => r.json());
+    expect(await exists(userDataDir)).to.be.true;
+
+    await browser.disconnect();
+    await sleep(1000);
+
+    expect(await exists(userDataDir)).to.be.false;
+  });
+
+  it('creates user-data-dirs with userDataDir options', async () => {
+    const dataDirLocation = '/tmp/browserless-test-dir';
+    const launch = JSON.stringify({
+      userDataDir: dataDirLocation,
+    });
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const browser = await puppeteer.connect({
+      browserWSEndpoint: `ws://localhost:3000?token=browserless&launch=${launch}`,
+    });
+
+    const [{ userDataDir }] = await fetch(
+      'http://localhost:3000/sessions?token=browserless',
+    ).then((r) => r.json());
+
+    expect(userDataDir === dataDirLocation).to.be.true;
+    expect(await exists(userDataDir)).to.be.true;
+
+    await browser.disconnect();
+    await sleep(500);
+
+    expect(await exists(userDataDir)).to.be.true;
+    await deleteAsync(userDataDir, { force: true });
+  });
+
+  it('creates user-data-dirs with CLI flags', async () => {
+    const dataDirLocation = '/tmp/browserless-test-dir';
+    const launch = JSON.stringify({
+      args: [`--user-data-dir==${dataDirLocation}`],
+    });
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const browser = await puppeteer.connect({
+      browserWSEndpoint: `ws://localhost:3000?token=browserless&launch=${launch}`,
+    });
+
+    const [{ userDataDir }] = await fetch(
+      'http://localhost:3000/sessions?token=browserless',
+    ).then((r) => r.json());
+
+    expect(userDataDir === dataDirLocation).to.be.true;
+    expect(await exists(userDataDir)).to.be.true;
+
+    await browser.disconnect();
+    await sleep(500);
+
+    expect(await exists(userDataDir)).to.be.true;
+    await deleteAsync(userDataDir, { force: true });
+  });
+
+  it('runs with job-based timeouts', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    config.setTimeout(-1); // No timeout
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const browser = await puppeteer.connect({
+      browserWSEndpoint: `ws://localhost:3000?timeout=500&token=browserless`,
+    });
+
+    await sleep(750);
+    browser.disconnect();
+    expect(metrics.get().timedout).to.equal(1);
+    expect(metrics.get().successful).to.equal(0);
+  });
+
+  it('allows the file-chooser', async () =>
+    new Promise(async (done) => {
+      const config = new Config();
+      config.setToken('browserless');
+      const metrics = new Metrics();
+      await start({ config, metrics });
+      const job = async () => {
+        const browser = await puppeteer.connect({
+          browserWSEndpoint: `ws://localhost:3000?token=browserless`,
+        });
+
+        const page = await browser.newPage();
+
+        await page.setContent(`<div class="output" style="height: 62%;"><label for="avatar">Choose a profile picture:</label>
+      <input type="file" id="avatar" name="avatar" accept="image/png, image/jpeg">
+    </div>`);
+
+        if (page.waitForFileChooser) {
+          const [fileChooser] = await Promise.all([
+            page.waitForFileChooser(),
+            page.click('#avatar'),
+          ]);
+          expect(fileChooser).to.not.be.undefined;
+          expect(fileChooser).to.not.be.null;
+        }
+        browser.disconnect();
+        done();
+      };
+
+      job();
+    }));
+
+  it('queues requests', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    config.setConcurrent(1);
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const job = async () => {
+      const browser = await puppeteer.connect({
+        browserWSEndpoint: `ws://localhost:3000?token=browserless`,
+      });
+      await sleep(100);
+
+      return browser.disconnect();
+    };
+
+    await Promise.all([job(), job()]);
+
+    await sleep(100);
+
+    const results = metrics.get();
+    expect(results.successful).to.equal(2);
+    expect(results.rejected).to.equal(0);
+    expect(results.queued).to.equal(1);
+  });
+
+  it('fails requests', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    config.setConcurrent(0);
+    config.setQueued(0);
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    return puppeteer
+      .connect({ browserWSEndpoint: `ws://localhost:3000?token=browserless` })
+      .catch((error) => {
+        const results = metrics.get();
+        expect(results.successful).to.equal(0);
+        expect(results.rejected).to.equal(1);
+        expect(results.queued).to.equal(0);
+        expect(error.message).to.contain.oneOf([`400`, `429`]);
+      });
+  });
+
+  it('fails requests without tokens', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    return puppeteer
+      .connect({ browserWSEndpoint: `ws://localhost:3000` })
+      .catch((error: Error) => {
+        const results = metrics.get();
+        expect(results.successful).to.equal(0);
+        expect(results.rejected).to.equal(0);
+        expect(results.queued).to.equal(0);
+        expect(error.message).to.contain(`401`);
+      });
+  });
+
+  it('runs playwright', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const browser = await chromium.connect(
+      `ws://localhost:3000/playwright/chromium?token=browserless`,
+    );
+
+    await browser.close();
+    await sleep(100);
+
+    const results = metrics.get();
+    expect(results.timedout).to.equal(0);
+    expect(results.successful).to.equal(1);
+    expect(results.rejected).to.equal(0);
+    expect(results.queued).to.equal(0);
+  });
+
+  it('runs playwright over CDP', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const browser = await chromium.connectOverCDP(
+      `ws://localhost:3000?token=browserless`,
+    );
+
+    await browser.close();
+    await sleep(100);
+
+    const results = metrics.get();
+    expect(results.timedout).to.equal(0);
+    expect(results.successful).to.equal(1);
+    expect(results.rejected).to.equal(0);
+    expect(results.queued).to.equal(0);
+  });
+
+  it('rejects playwright without tokens', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    await chromium
+      .connect(`ws://localhost:3000/playwright/chromium`)
+      .catch((e) => {
+        const results = metrics.get();
+        expect(e.message).to.include('Bad or missing authentication');
+        expect(results.timedout).to.equal(0);
+        expect(results.successful).to.equal(0);
+        expect(results.unauthorized).to.equal(1);
+        expect(results.queued).to.equal(0);
+      });
+  });
+
+  it('allows requests without token when auth token is not set', async () => {
+    await start();
+
+    const browser = await puppeteer.connect({
+      browserWSEndpoint: `ws://localhost:3000`,
+    });
+
+    await browser.disconnect();
+  });
+});

package/src/routes/chromium/utils/function/client.ts

@@ -0,0 +1,99 @@
+import { Browser, Page } from 'puppeteer-core';
+import { BrowserWebSocketTransport } from 'puppeteer-core/lib/esm/puppeteer/common/BrowserWebSocketTransport.js';
+import { _connectToCdpBrowser as connect } from 'puppeteer-core/lib/esm/puppeteer/cdp/BrowserConnector.js';
+
+type codeHandler = (params: {
+  context: unknown;
+  page: Page;
+}) => Promise<unknown>;
+
+export class FunctionRunner {
+  private browser?: Browser;
+  private page?: Page;
+
+  public log = () => console.log.bind(console);
+
+  public async start(data: {
+    browserWSEndpoint: string;
+    code: codeHandler;
+    context: unknown;
+    options: {
+      downloadPath?: string;
+    };
+  }) {
+    console.log(`/function.js: Got endpoint: "${data.browserWSEndpoint}"`);
+    const { browserWSEndpoint, code, context, options } = data;
+    const connectionTransport =
+      await BrowserWebSocketTransport.create(browserWSEndpoint);
+    const cdpOptions = {
+      headers: {
+        Host: '127.0.0.1',
+      },
+    };
+
+    this.browser = (await connect(
+      connectionTransport,
+      browserWSEndpoint,
+      cdpOptions,
+    )) as unknown as Browser;
+    this.browser.once('disconnected', () => this.stop());
+    this.page = await this.browser.newPage();
+
+    if (options.downloadPath) {
+      console.debug(
+        `_browserless_function_client_: Setting downloads for page to "${options.downloadPath}"`,
+      );
+      // @ts-ignore
+      const client = this.page._client.call(this.page);
+      await client.send('Page.setDownloadBehavior', {
+        behavior: 'allow',
+        downloadPath: options.downloadPath,
+      });
+    }
+
+    const response = await code({ context, page: this.page });
+    console.debug(
+      `_browserless_function_client_: Code is finished executing, closing page.`,
+    );
+    this.page.close().catch(this.log);
+
+    if (response instanceof Uint8Array) {
+      return {
+        contentType: 'uint8array',
+        payload: Array.from(response),
+      };
+    }
+
+    if (typeof response === 'string') {
+      return {
+        contentType: response.startsWith('<') ? 'text/html' : 'text/plain',
+        payload: response,
+      };
+    }
+
+    if (typeof response === 'object') {
+      return {
+        contentType: 'application/json',
+        payload: JSON.stringify(response, null, '  '),
+      };
+    }
+
+    return {
+      contentType: 'text/plain',
+      payload: response,
+    };
+  }
+
+  public stop() {
+    if (this.browser) this.browser.disconnect();
+  }
+}
+
+// Set this as an immutable property on window so our handler's
+// can call it downstream
+Object.defineProperty(window, 'BrowserlessFunctionRunner', {
+  configurable: false,
+  enumerable: false,
+  value: FunctionRunner,
+  writable: false,
+});

package/src/routes/chromium/utils/function/handler.ts

@@ -0,0 +1,172 @@
+import {
+  BadRequest,
+  BrowserInstance,
+  CDPChromium,
+  Config,
+  HTTPRoutes,
+  Request,
+  UnwrapPromise,
+  contentTypes,
+  convertIfBase64,
+  exists,
+  id,
+  makeExternalURL,
+  mimeTypes,
+} from '@browserless.io/browserless';
+import { FunctionRunner } from './client.js';
+import { Page } from 'puppeteer-core';
+import debug from 'debug';
+import fs from 'fs';
+import path from 'path';
+
+declare global {
+  interface Window {
+    BrowserlessFunctionRunner: typeof FunctionRunner;
+  }
+}
+
+interface JSONSchema {
+  code: string;
+  context?: Record<string, string | number>;
+}
+
+interface HandlerOptions {
+  downloadPath?: string;
+}
+
+export default (
+    config: Config,
+    debug: debug.Debugger,
+    options: HandlerOptions = {},
+  ) =>
+  async (
+    req: Request,
+    browser: BrowserInstance,
+  ): Promise<{ contentType: string; page: Page; payload: unknown }> => {
+    const isJson = req.headers['content-type']?.includes('json');
+
+    const functionAssetLocation = path.join(config.getStatic(), 'function');
+    const functionRequestPath = makeExternalURL(
+      config.getExternalAddress(),
+      HTTPRoutes.function,
+    );
+    const functionIndexHTML = makeExternalURL(
+      config.getExternalAddress(),
+      HTTPRoutes.function,
+      '/index.html',
+    );
+
+    const { code: rawCode, context: rawContext } = isJson
+      ? (req.body as JSONSchema)
+      : {
+          code: req.body as string,
+          context: {},
+        };
+
+    const context = JSON.stringify(rawContext);
+    const code = convertIfBase64(rawCode);
+    const browserWSEndpoint = browser.publicWSEndpoint(
+      req.parsed.searchParams.get('token') ?? '',
+    );
+
+    if (!browserWSEndpoint) {
+      throw new Error(`No browser endpoint was found, is the browser running?`);
+    }
+
+    const functionCodeJS = `browserless-function-${id()}.js`;
+    const page = (await browser.newPage()) as UnwrapPromise<
+      ReturnType<CDPChromium['newPage']>
+    >;
+    await page.setRequestInterception(true);
+
+    /**
+     * We serve static files to the function api by injecting
+     * request responses. This is done because users can use
+     * a proxy, which might not have access or abilities to
+     * request the function index/js from this server.
+     */
+    page.on('request', async (request) => {
+      const requestUrl = request.url();
+      debug(`/function.js: Page Request: "${requestUrl}"`);
+      if (requestUrl.startsWith(functionRequestPath)) {
+        const filename = path.basename(requestUrl);
+        if (filename === functionCodeJS) {
+          return request.respond({
+            body: code,
+            contentType: contentTypes.javascript,
+            status: 200,
+          });
+        }
+        const filePath = path.join(functionAssetLocation, filename);
+        if (await exists(filePath)) {
+          const contentType = mimeTypes.get(path.extname(filePath));
+          return request.respond({
+            body: await fs.readFileSync(filePath).toString(),
+            contentType: contentType,
+            status: 200,
+          });
+        }
+        debug(
+          `Static asset request to "${requestUrl}" couldn't be found, 404-ing`,
+        );
+        return request.respond({
+          body: code,
+          contentType: `Couldn't locate this file "${filename}" request "${requestUrl}" in "${functionAssetLocation}"`,
+          status: 404,
+        });
+      }
+      return request.continue();
+    });
+
+    page.on('response', (res) => {
+      debug(`/function.js: Page Response: "${res.url()}"`);
+      if (res.status() !== 200) {
+        debug(`Received a non-200 response for request "${res.url()}"`);
+      }
+    });
+
+    page.on('console', (event) => {
+      debug(`${event.type()}: ${event.text()}`);
+    });
+
+    await page.goto(functionIndexHTML);
+
+    const { contentType, payload } = await page
+      .evaluate(
+        async (
+          browserWSEndpoint,
+          context,
+          functionCodeJS,
+          serializedOptions,
+        ) => {
+          const [{ default: code }] = await Promise.all([
+            import('./' + functionCodeJS),
+          ]);
+          console.log('/function.js: imported successfully.');
+          const helper = new window.BrowserlessFunctionRunner();
+          const options = JSON.parse(serializedOptions);
+          console.log('/function.js: executing puppeteer code.');
+
+          return helper.start({
+            browserWSEndpoint,
+            code,
+            context: JSON.parse(context || `{}`),
+            options,
+          });
+        },
+        browserWSEndpoint,
+        context,
+        functionCodeJS,
+        JSON.stringify(options),
+      )
+      .catch((e) => {
+        debug(`Error running code: ${e}`);
+        throw new BadRequest(e.message);
+      });
+
+    return {
+      contentType,
+      page,
+      payload,
+    };
+  };

package/src/routes/chromium/utils/performance/child.ts

@@ -0,0 +1,38 @@
+import { Message, start } from './types.js';
+import { createLogger } from '@browserless.io/browserless';
+import lighthouse from 'lighthouse';
+
+const debug = createLogger('http:performance:child');
+
+debug(`Child init`);
+
+const send = (msg: Message) => process.send && process.send(msg);
+
+const start = async ({ url, config, options }: start) => {
+  try {
+    debug(`Child got payload, starting lighthouse`);
+    const results = await lighthouse(url, options, config);
+
+    send({
+      data: results?.lhr,
+      event: 'complete',
+    });
+  } catch (error: unknown) {
+    send({
+      error,
+      event: 'error',
+    });
+  }
+};
+
+process.on('message', (payload) => {
+  const { event } = payload as { event: string };
+
+  if (event === 'start') {
+    return start(payload as start);
+  }
+
+  return;
+});
+
+send({ event: 'created' });