@browserless.io/browserless 2.0.0-beta-1

package/src/routes/chromium/http/download-post.ts

@@ -0,0 +1,162 @@
+import {
+  APITags,
+  BrowserHTTPRoute,
+  BrowserInstance,
+  CDPChromium,
+  CDPLaunchOptions,
+  HTTPRoutes,
+  Methods,
+  NotFound,
+  Request,
+  ServerError,
+  SystemQueryParameters,
+  contentTypes,
+  dedent,
+  id,
+  mimeTypes,
+  once,
+  sleep,
+} from '@browserless.io/browserless';
+import { mkdir, readdir } from 'fs/promises';
+import { ServerResponse } from 'http';
+import { createReadStream } from 'fs';
+import { deleteAsync } from 'del';
+import functionHandler from '../utils/function/handler.js';
+import path from 'path';
+
+interface JSONSchema {
+  code: string;
+  context?: Record<string, string | number>;
+}
+
+export type BodySchema = JSONSchema | string;
+
+export interface QuerySchema extends SystemQueryParameters {
+  launch?: CDPLaunchOptions | string;
+}
+
+/**
+ * Responses are determined by the returned value of the downloads
+ * themselves, so there isn't a static response type for this API.
+ */
+export type ResponseSchema = unknown;
+
+const route: BrowserHTTPRoute = {
+  accepts: [contentTypes.json, contentTypes.javascript],
+  auth: true,
+  browser: CDPChromium,
+  concurrency: true,
+  contentTypes: [contentTypes.any],
+  description: dedent(`
+  A JSON or JavaScript content-type API for returning files Chrome has downloaded during
+  the execution of puppeteer code, which is ran inside context of the browser.
+  Browserless sets up a blank page, a fresh download directory, injects your puppeteer code, and then executes it.
+  You can load external libraries via the "import" syntax, and import ESM-style modules
+  that are written for execution inside of the browser. Once your script is finished, any
+  downloaded files from Chromium are returned back with the appropriate content-type header.`),
+  handler: async (
+    req: Request,
+    res: ServerResponse,
+    browser: BrowserInstance,
+  ): Promise<void> =>
+    new Promise(async (resolve, reject) => {
+      const { _config: getConfig, _debug: getDebug } = route;
+
+      if (!getConfig || !getDebug) {
+        return reject(
+          new ServerError(`Couldn't load configuration for request`),
+        );
+      }
+
+      const debug = getDebug();
+      const config = getConfig();
+      const downloadPath = path.join(
+        await config.getDownloadsDir(),
+        `.browserless.download.${id()}`,
+      );
+
+      debug(`Generating a download directory at "${downloadPath}"`);
+      await mkdir(downloadPath);
+      const handler = functionHandler(config, debug, { downloadPath });
+      const response = await handler(req, browser).catch((e) => {
+        debug(`Error running download code handler: "${e}"`);
+        reject(e);
+        return null;
+      });
+
+      if (!response) {
+        return;
+      }
+
+      const { page } = response;
+      debug(`Download function has returned, finding downloads...`);
+      async function checkIfDownloadComplete(): Promise<string | null> {
+        if (res.headersSent) {
+          debug(`Request headers have been sent, terminating download watch.`);
+          return null;
+        }
+        const [fileName] = await readdir(downloadPath);
+        if (!fileName || fileName.endsWith('.crdownload')) {
+          await sleep(500);
+          return checkIfDownloadComplete();
+        }
+
+        debug(`All files have finished downloading`);
+
+        return path.join(downloadPath, fileName);
+      }
+
+      const filePath = await checkIfDownloadComplete();
+      debug(`Closing pages.`);
+      page.close();
+      page.removeAllListeners();
+
+      const rmDownload = once(
+        () =>
+          filePath &&
+          deleteAsync(filePath, { force: true })
+            .then(() => {
+              debug(
+                `Successfully deleted downloads from disk at "${filePath}"`,
+              );
+            })
+            .catch((err) => {
+              debug(
+                `Error cleaning up downloaded files: "${err}" at "${filePath}"`,
+              );
+            }),
+      );
+
+      if (res.headersSent || !filePath) {
+        rmDownload();
+        return;
+      }
+      const contentType = mimeTypes.get(path.extname(filePath));
+      if (contentType) {
+        res.setHeader('Content-Type', contentType);
+      }
+
+      return createReadStream(filePath)
+        .on('error', (error) => {
+          if (error) {
+            rmDownload();
+            return reject(
+              new NotFound(
+                `Couldn't locate or send downloads in "${downloadPath}"`,
+              ),
+            );
+          }
+        })
+        .on('end', () => {
+          debug(`Downloads successfully sent`);
+          rmDownload();
+          return resolve();
+        })
+        .pipe(res);
+    }),
+  method: Methods.post,
+  path: HTTPRoutes.download,
+  tags: [APITags.browserAPI],
+};
+
+export default route;

package/src/routes/chromium/http/function-post.ts

@@ -0,0 +1,98 @@
+import {
+  APITags,
+  BadRequest,
+  BrowserHTTPRoute,
+  BrowserInstance,
+  CDPChromium,
+  CDPLaunchOptions,
+  HTTPRoutes,
+  Methods,
+  Request,
+  ServerError,
+  SystemQueryParameters,
+  contentTypes,
+  dedent,
+  writeResponse,
+} from '@browserless.io/browserless';
+import { ServerResponse } from 'http';
+import Stream from 'stream';
+import { fileTypeFromBuffer } from 'file-type';
+import functionHandler from '../utils/function/handler.js';
+
+interface JSONSchema {
+  code: string;
+  context?: Record<string, string | number>;
+}
+
+export type BodySchema = JSONSchema | string;
+
+export interface QuerySchema extends SystemQueryParameters {
+  launch?: CDPLaunchOptions | string;
+}
+
+/**
+ * Responses are determined by the returned value of the function
+ * itself. Binary responses (PDF's, screenshots) are returned back
+ * as binary data, and primitive JavaScript values are returned back
+ * by type (HTML data is "text/html", Objects are "application/json")
+ */
+export type ResponseSchema = unknown;
+
+const route: BrowserHTTPRoute = {
+  accepts: [contentTypes.json, contentTypes.javascript],
+  auth: true,
+  browser: CDPChromium,
+  concurrency: true,
+  contentTypes: [contentTypes.any],
+  description: dedent(`
+  A JSON or JavaScript content-type API for running puppeteer code in the browser's context.
+  Browserless sets up a blank page, injects your puppeteer code, and runs it.
+  You can optionally load external libraries via the "import" module that are meant for browser usage.
+  Values returned from the function are checked and an appropriate content-type and response is sent back
+  to your HTTP call.`),
+  handler: async (
+    req: Request,
+    res: ServerResponse,
+    browser: BrowserInstance,
+  ): Promise<void> => {
+    const { _config: getConfig, _debug: getDebug } = route;
+
+    if (!getConfig || !getDebug) {
+      throw new ServerError(`Couldn't load configuration for request`);
+    }
+
+    const debug = getDebug();
+    const config = getConfig();
+    const handler = functionHandler(config, debug);
+    const { contentType, payload, page } = await handler(req, browser);
+
+    debug(`Got function response of "${contentType}"`);
+    page.close();
+    page.removeAllListeners();
+
+    if (contentType === 'uint8array') {
+      const response = new Uint8Array(payload as Buffer);
+      const type = ((await fileTypeFromBuffer(response)) || { mime: undefined })
+        .mime;
+
+      if (!type) {
+        throw new BadRequest(`Couldn't determine function's response type.`);
+      } else {
+        debug(`Sending file-type response of "${type}"`);
+        const readStream = new Stream.PassThrough();
+        readStream.end(response);
+        res.setHeader('Content-Type', type);
+        return new Promise((r) => readStream.pipe(res).once('close', r));
+      }
+    } else {
+      writeResponse(res, 200, payload as string, contentType as contentTypes);
+    }
+
+    return;
+  },
+  method: Methods.post,
+  path: HTTPRoutes.function,
+  tags: [APITags.browserAPI],
+};
+
+export default route;

package/src/routes/chromium/http/pdf-post.ts

@@ -0,0 +1,234 @@
+import {
+  APITags,
+  BadRequest,
+  BrowserHTTPRoute,
+  BrowserInstance,
+  CDPChromium,
+  CDPLaunchOptions,
+  HTTPRoutes,
+  Methods,
+  Request,
+  SystemQueryParameters,
+  UnwrapPromise,
+  WaitForEventOptions,
+  WaitForFunctionOptions,
+  WaitForSelectorOptions,
+  bestAttempt,
+  bestAttemptCatch,
+  contentTypes,
+  noop,
+  rejectRequestPattern,
+  rejectResourceTypes,
+  requestInterceptors,
+  waitForEvent as waitForEvt,
+  waitForFunction as waitForFn,
+  writeResponse,
+} from '@browserless.io/browserless';
+import { Page } from 'puppeteer-core';
+import { ServerResponse } from 'http';
+
+export interface BodySchema {
+  addScriptTag?: Array<Parameters<Page['addScriptTag']>[0]>;
+  addStyleTag?: Array<Parameters<Page['addStyleTag']>[0]>;
+  authenticate?: Parameters<Page['authenticate']>[0];
+  bestAttempt?: bestAttempt;
+  blockModals?: boolean;
+  cookies?: Array<Parameters<Page['setCookie']>[0]>;
+  emulateMediaType?: Parameters<Page['emulateMediaType']>[0];
+  gotoOptions?: Parameters<Page['goto']>[1];
+  html?: Parameters<Page['setContent']>[0];
+  options?: Parameters<Page['pdf']>[0];
+  rejectRequestPattern?: rejectRequestPattern[];
+  rejectResourceTypes?: rejectResourceTypes[];
+  requestInterceptors?: Array<requestInterceptors>;
+  setExtraHTTPHeaders?: Parameters<Page['setExtraHTTPHeaders']>[0];
+  setJavaScriptEnabled?: boolean;
+  url?: Parameters<Page['goto']>[0];
+  userAgent?: Parameters<Page['setUserAgent']>[0];
+  viewport?: Parameters<Page['setViewport']>[0];
+  waitForEvent?: WaitForEventOptions;
+  waitForFunction?: WaitForFunctionOptions;
+  waitForSelector?: WaitForSelectorOptions;
+  waitForTimeout?: Parameters<Page['waitForTimeout']>[0];
+}
+
+export interface QuerySchema extends SystemQueryParameters {
+  launch?: CDPLaunchOptions | string;
+}
+
+/**
+ * Responds with an application/pdf content-type and a binary PDF
+ */
+export type ResponseSchema = string;
+
+const route: BrowserHTTPRoute = {
+  accepts: [contentTypes.json],
+  auth: true,
+  browser: CDPChromium,
+  concurrency: true,
+  contentTypes: [contentTypes.pdf],
+  description: `A JSON-based API for getting a PDF binary from either a supplied "url" or "html" payload in your request.`,
+  handler: async (
+    req: Request,
+    res: ServerResponse,
+    browser: BrowserInstance,
+  ): Promise<void> => {
+    const contentType =
+      !req.headers.accept || req.headers.accept?.includes('*')
+        ? 'application/pdf'
+        : req.headers.accept;
+
+    if (!req.body) {
+      writeResponse(res, 400, `Couldn't parse JSON body`);
+      return;
+    }
+
+    res.setHeader('Content-Type', contentType);
+
+    const {
+      url,
+      gotoOptions,
+      authenticate,
+      html,
+      addScriptTag = [],
+      addStyleTag = [],
+      cookies = [],
+      emulateMediaType,
+      rejectRequestPattern = [],
+      requestInterceptors = [],
+      rejectResourceTypes = [],
+      options,
+      setExtraHTTPHeaders,
+      setJavaScriptEnabled,
+      userAgent,
+      viewport,
+      waitForFunction,
+      waitForSelector,
+      waitForEvent,
+      bestAttempt = false,
+    } = req.body as BodySchema;
+
+    const content = url || html;
+
+    if (!content) {
+      throw new BadRequest(`One of "url" or "html" properties are required.`);
+    }
+
+    const page = (await browser.newPage()) as UnwrapPromise<
+      ReturnType<CDPChromium['newPage']>
+    >;
+    const gotoCall = url ? page.goto.bind(page) : page.setContent.bind(page);
+
+    if (emulateMediaType) {
+      await page.emulateMediaType(emulateMediaType);
+    }
+
+    if (cookies.length) {
+      await page.setCookie(...cookies);
+    }
+
+    if (viewport) {
+      await page.setViewport(viewport);
+    }
+
+    if (userAgent) {
+      await page.setUserAgent(userAgent);
+    }
+
+    if (authenticate) {
+      await page.authenticate(authenticate);
+    }
+
+    if (setExtraHTTPHeaders) {
+      await page.setExtraHTTPHeaders(setExtraHTTPHeaders);
+    }
+
+    if (setJavaScriptEnabled) {
+      await page.setJavaScriptEnabled(setJavaScriptEnabled);
+    }
+
+    if (
+      rejectRequestPattern.length ||
+      requestInterceptors.length ||
+      rejectResourceTypes.length
+    ) {
+      await page.setRequestInterception(true);
+
+      page.on('request', (req) => {
+        if (
+          !!rejectRequestPattern.find((pattern) => req.url().match(pattern)) ||
+          rejectResourceTypes.includes(req.resourceType())
+        ) {
+          return req.abort();
+        }
+        const interceptor = requestInterceptors.find((r) =>
+          req.url().match(r.pattern),
+        );
+        if (interceptor) {
+          return req.respond(interceptor.response);
+        }
+        return req.continue();
+      });
+    }
+
+    if (addStyleTag.length) {
+      for (const tag in addStyleTag) {
+        await page.addStyleTag(addStyleTag[tag]);
+      }
+    }
+
+    if (addScriptTag.length) {
+      for (const tag in addScriptTag) {
+        await page.addScriptTag(addScriptTag[tag]);
+      }
+    }
+
+    const gotoResponse = await gotoCall(content, gotoOptions).catch(
+      bestAttemptCatch(bestAttempt),
+    );
+
+    if (waitForFunction) {
+      await waitForFn(page, waitForFunction).catch(
+        bestAttemptCatch(bestAttempt),
+      );
+    }
+
+    if (waitForSelector) {
+      const { selector, hidden, timeout, visible } = waitForSelector;
+      await page
+        .waitForSelector(selector, { hidden, timeout, visible })
+        .catch(bestAttemptCatch(bestAttempt));
+    }
+
+    if (waitForEvent) {
+      await waitForEvt(page, waitForEvent).catch(bestAttemptCatch(bestAttempt));
+    }
+
+    const headers = {
+      'X-Response-Code': gotoResponse?.status(),
+      'X-Response-IP': gotoResponse?.remoteAddress().ip,
+      'X-Response-Port': gotoResponse?.remoteAddress().port,
+      'X-Response-Status': gotoResponse?.statusText(),
+      'X-Response-URL': gotoResponse?.url().substring(0, 1000),
+    };
+
+    for (const [key, value] of Object.entries(headers)) {
+      if (value !== undefined) {
+        res.setHeader(key, value);
+      }
+    }
+
+    const pdfStream = await page.createPDFStream(options);
+
+    await new Promise((resolve, reject) => {
+      return pdfStream.pipe(res).once('finish', resolve).once('error', reject);
+    });
+
+    page.close().catch(noop);
+  },
+  method: Methods.post,
+  path: HTTPRoutes.pdf,
+  tags: [APITags.browserAPI],
+};
+
+export default route;

package/src/routes/chromium/http/performance.ts

@@ -0,0 +1,71 @@
+import {
+  APITags,
+  BadRequest,
+  BrowserHTTPRoute,
+  BrowserInstance,
+  CDPChromium,
+  CDPLaunchOptions,
+  HTTPRoutes,
+  Methods,
+  Request,
+  ServerError,
+  SystemQueryParameters,
+  contentTypes,
+  jsonResponse,
+} from '@browserless.io/browserless';
+import { ServerResponse } from 'http';
+
+import main from '../utils/performance/main.js';
+
+export interface BodySchema {
+  budgets?: Array<object>;
+  config?: object;
+  url: string;
+}
+
+export interface QuerySchema extends SystemQueryParameters {
+  launch?: CDPLaunchOptions | string;
+}
+
+/**
+ * The response of the lighthouse stats. Response objects are
+ * determined by the type of budgets and config in the POST
+ * JSON body
+ */
+export type ResponseSchema = object;
+
+const route: BrowserHTTPRoute = {
+  accepts: [contentTypes.json],
+  auth: true,
+  browser: CDPChromium,
+  concurrency: true,
+  contentTypes: [contentTypes.json],
+  description: `Run lighthouse performance audits with a supplied "url" in your JSON payload.`,
+  handler: async (
+    req: Request,
+    res: ServerResponse,
+    browser: BrowserInstance,
+  ): Promise<void> => {
+    const { _config: getConfig } = route;
+    if (!req.body) {
+      throw new BadRequest(`No JSON body present`);
+    }
+
+    if (!getConfig) {
+      throw new ServerError(`Couldn't load configuration for timeouts`);
+    }
+    const config = getConfig();
+    const response = await main({
+      browser,
+      context: req.body as BodySchema,
+      timeout: config.getTimeout(),
+    });
+
+    return jsonResponse(res, 200, response);
+  },
+  method: Methods.post,
+  path: HTTPRoutes.performance,
+  tags: [APITags.browserAPI],
+};
+
+export default route;