@browserless.io/browserless 2.0.0-beta-1

package/src/mime-types.ts

@@ -0,0 +1,332 @@
+const mimeMap = {
+  'application/andrew-inset': ['ez'],
+  'application/applixware': ['aw'],
+  'application/atom+xml': ['atom'],
+  'application/atomcat+xml': ['atomcat'],
+  'application/atomdeleted+xml': ['atomdeleted'],
+  'application/atomsvc+xml': ['atomsvc'],
+  'application/atsc-dwd+xml': ['dwd'],
+  'application/atsc-held+xml': ['held'],
+  'application/atsc-rsat+xml': ['rsat'],
+  'application/bdoc': ['bdoc'],
+  'application/calendar+xml': ['xcs'],
+  'application/ccxml+xml': ['ccxml'],
+  'application/cdfx+xml': ['cdfx'],
+  'application/cdmi-capability': ['cdmia'],
+  'application/cdmi-container': ['cdmic'],
+  'application/cdmi-domain': ['cdmid'],
+  'application/cdmi-object': ['cdmio'],
+  'application/cdmi-queue': ['cdmiq'],
+  'application/cu-seeme': ['cu'],
+  'application/dash+xml': ['mpd'],
+  'application/davmount+xml': ['davmount'],
+  'application/docbook+xml': ['dbk'],
+  'application/dssc+der': ['dssc'],
+  'application/dssc+xml': ['xdssc'],
+  'application/ecmascript': ['ecma', 'es'],
+  'application/emma+xml': ['emma'],
+  'application/emotionml+xml': ['emotionml'],
+  'application/epub+zip': ['epub'],
+  'application/exi': ['exi'],
+  'application/fdt+xml': ['fdt'],
+  'application/font-tdpfr': ['pfr'],
+  'application/geo+json': ['geojson'],
+  'application/gml+xml': ['gml'],
+  'application/gpx+xml': ['gpx'],
+  'application/gxf': ['gxf'],
+  'application/gzip': ['gz'],
+  'application/hjson': ['hjson'],
+  'application/hyperstudio': ['stk'],
+  'application/inkml+xml': ['ink', 'inkml'],
+  'application/ipfix': ['ipfix'],
+  'application/its+xml': ['its'],
+  'application/java-archive': ['jar', 'war', 'ear'],
+  'application/java-serialized-object': ['ser'],
+  'application/java-vm': ['class'],
+  'application/javascript': ['js', 'mjs'],
+  'application/json': ['json', 'map'],
+  'application/json5': ['json5'],
+  'application/jsonml+json': ['jsonml'],
+  'application/ld+json': ['jsonld'],
+  'application/lgr+xml': ['lgr'],
+  'application/lost+xml': ['lostxml'],
+  'application/mac-binhex40': ['hqx'],
+  'application/mac-compactpro': ['cpt'],
+  'application/mads+xml': ['mads'],
+  'application/manifest+json': ['webmanifest'],
+  'application/marc': ['mrc'],
+  'application/marcxml+xml': ['mrcx'],
+  'application/mathematica': ['ma', 'nb', 'mb'],
+  'application/mathml+xml': ['mathml'],
+  'application/mbox': ['mbox'],
+  'application/mediaservercontrol+xml': ['mscml'],
+  'application/metalink+xml': ['metalink'],
+  'application/metalink4+xml': ['meta4'],
+  'application/mets+xml': ['mets'],
+  'application/mmt-aei+xml': ['maei'],
+  'application/mmt-usd+xml': ['musd'],
+  'application/mods+xml': ['mods'],
+  'application/mp21': ['m21', 'mp21'],
+  'application/mp4': ['mp4s', 'm4p'],
+  'application/mrb-consumer+xml': ['*xdf'],
+  'application/mrb-publish+xml': ['*xdf'],
+  'application/msword': ['doc', 'dot'],
+  'application/mxf': ['mxf'],
+  'application/n-quads': ['nq'],
+  'application/n-triples': ['nt'],
+  'application/node': ['cjs'],
+  'application/octet-stream': [
+    'bin',
+    'dms',
+    'lrf',
+    'mar',
+    'so',
+    'dist',
+    'distz',
+    'pkg',
+    'bpk',
+    'dump',
+    'elc',
+    'deploy',
+    'exe',
+    'dll',
+    'deb',
+    'dmg',
+    'iso',
+    'img',
+    'msi',
+    'msp',
+    'msm',
+    'buffer',
+  ],
+  'application/oda': ['oda'],
+  'application/oebps-package+xml': ['opf'],
+  'application/ogg': ['ogx'],
+  'application/omdoc+xml': ['omdoc'],
+  'application/onenote': ['onetoc', 'onetoc2', 'onetmp', 'onepkg'],
+  'application/oxps': ['oxps'],
+  'application/p2p-overlay+xml': ['relo'],
+  'application/patch-ops-error+xml': ['*xer'],
+  'application/pdf': ['pdf'],
+  'application/pgp-encrypted': ['pgp'],
+  'application/pgp-signature': ['asc', 'sig'],
+  'application/pics-rules': ['prf'],
+  'application/pkcs10': ['p10'],
+  'application/pkcs7-mime': ['p7m', 'p7c'],
+  'application/pkcs7-signature': ['p7s'],
+  'application/pkcs8': ['p8'],
+  'application/pkix-attr-cert': ['ac'],
+  'application/pkix-cert': ['cer'],
+  'application/pkix-crl': ['crl'],
+  'application/pkix-pkipath': ['pkipath'],
+  'application/pkixcmp': ['pki'],
+  'application/pls+xml': ['pls'],
+  'application/postscript': ['ai', 'eps', 'ps'],
+  'application/provenance+xml': ['provx'],
+  'application/pskc+xml': ['pskcxml'],
+  'application/raml+yaml': ['raml'],
+  'application/rdf+xml': ['rdf', 'owl'],
+  'application/reginfo+xml': ['rif'],
+  'application/relax-ng-compact-syntax': ['rnc'],
+  'application/resource-lists+xml': ['rl'],
+  'application/resource-lists-diff+xml': ['rld'],
+  'application/rls-services+xml': ['rs'],
+  'application/route-apd+xml': ['rapd'],
+  'application/route-s-tsid+xml': ['sls'],
+  'application/route-usd+xml': ['rusd'],
+  'application/rpki-ghostbusters': ['gbr'],
+  'application/rpki-manifest': ['mft'],
+  'application/rpki-roa': ['roa'],
+  'application/rsd+xml': ['rsd'],
+  'application/rss+xml': ['rss'],
+  'application/rtf': ['rtf'],
+  'application/sbml+xml': ['sbml'],
+  'application/scvp-cv-request': ['scq'],
+  'application/scvp-cv-response': ['scs'],
+  'application/scvp-vp-request': ['spq'],
+  'application/scvp-vp-response': ['spp'],
+  'application/sdp': ['sdp'],
+  'application/senml+xml': ['senmlx'],
+  'application/sensml+xml': ['sensmlx'],
+  'application/set-payment-initiation': ['setpay'],
+  'application/set-registration-initiation': ['setreg'],
+  'application/shf+xml': ['shf'],
+  'application/sieve': ['siv', 'sieve'],
+  'application/smil+xml': ['smi', 'smil'],
+  'application/sparql-query': ['rq'],
+  'application/sparql-results+xml': ['srx'],
+  'application/srgs': ['gram'],
+  'application/srgs+xml': ['grxml'],
+  'application/sru+xml': ['sru'],
+  'application/ssdl+xml': ['ssdl'],
+  'application/ssml+xml': ['ssml'],
+  'application/swid+xml': ['swidtag'],
+  'application/tei+xml': ['tei', 'teicorpus'],
+  'application/thraud+xml': ['tfi'],
+  'application/timestamped-data': ['tsd'],
+  'application/toml': ['toml'],
+  'application/ttml+xml': ['ttml'],
+  'application/ubjson': ['ubj'],
+  'application/urc-ressheet+xml': ['rsheet'],
+  'application/urc-targetdesc+xml': ['td'],
+  'application/voicexml+xml': ['vxml'],
+  'application/wasm': ['wasm'],
+  'application/widget': ['wgt'],
+  'application/winhlp': ['hlp'],
+  'application/wsdl+xml': ['wsdl'],
+  'application/wspolicy+xml': ['wspolicy'],
+  'application/xaml+xml': ['xaml'],
+  'application/xcap-att+xml': ['xav'],
+  'application/xcap-caps+xml': ['xca'],
+  'application/xcap-diff+xml': ['xdf'],
+  'application/xcap-el+xml': ['xel'],
+  'application/xcap-error+xml': ['xer'],
+  'application/xcap-ns+xml': ['xns'],
+  'application/xenc+xml': ['xenc'],
+  'application/xhtml+xml': ['xhtml', 'xht'],
+  'application/xliff+xml': ['xlf'],
+  'application/xml': ['xml', 'xsl', 'xsd', 'rng'],
+  'application/xml-dtd': ['dtd'],
+  'application/xop+xml': ['xop'],
+  'application/xproc+xml': ['xpl'],
+  'application/xslt+xml': ['*xsl', 'xslt'],
+  'application/xspf+xml': ['xspf'],
+  'application/xv+xml': ['mxml', 'xhvml', 'xvml', 'xvm'],
+  'application/yang': ['yang'],
+  'application/yin+xml': ['yin'],
+  'application/zip': ['zip'],
+  'audio/3gpp': ['*3gpp'],
+  'audio/adpcm': ['adp'],
+  'audio/amr': ['amr'],
+  'audio/basic': ['au', 'snd'],
+  'audio/midi': ['mid', 'midi', 'kar', 'rmi'],
+  'audio/mobile-xmf': ['mxmf'],
+  'audio/mp3': ['*mp3'],
+  'audio/mp4': ['m4a', 'mp4a'],
+  'audio/mpeg': ['mpga', 'mp2', 'mp2a', 'mp3', 'm2a', 'm3a'],
+  'audio/ogg': ['oga', 'ogg', 'spx', 'opus'],
+  'audio/s3m': ['s3m'],
+  'audio/silk': ['sil'],
+  'audio/wav': ['wav'],
+  'audio/wave': ['*wav'],
+  'audio/webm': ['weba'],
+  'audio/xm': ['xm'],
+  'font/collection': ['ttc'],
+  'font/otf': ['otf'],
+  'font/ttf': ['ttf'],
+  'font/woff': ['woff'],
+  'font/woff2': ['woff2'],
+  'image/aces': ['exr'],
+  'image/apng': ['apng'],
+  'image/avif': ['avif'],
+  'image/bmp': ['bmp'],
+  'image/cgm': ['cgm'],
+  'image/dicom-rle': ['drle'],
+  'image/emf': ['emf'],
+  'image/fits': ['fits'],
+  'image/g3fax': ['g3'],
+  'image/gif': ['gif'],
+  'image/heic': ['heic'],
+  'image/heic-sequence': ['heics'],
+  'image/heif': ['heif'],
+  'image/heif-sequence': ['heifs'],
+  'image/hej2k': ['hej2'],
+  'image/hsj2': ['hsj2'],
+  'image/ief': ['ief'],
+  'image/jls': ['jls'],
+  'image/jp2': ['jp2', 'jpg2'],
+  'image/jpeg': ['jpeg', 'jpg', 'jpe'],
+  'image/jph': ['jph'],
+  'image/jphc': ['jhc'],
+  'image/jpm': ['jpm'],
+  'image/jpx': ['jpx', 'jpf'],
+  'image/jxr': ['jxr'],
+  'image/jxra': ['jxra'],
+  'image/jxrs': ['jxrs'],
+  'image/jxs': ['jxs'],
+  'image/jxsc': ['jxsc'],
+  'image/jxsi': ['jxsi'],
+  'image/jxss': ['jxss'],
+  'image/ktx': ['ktx'],
+  'image/ktx2': ['ktx2'],
+  'image/png': ['png'],
+  'image/sgi': ['sgi'],
+  'image/svg+xml': ['svg', 'svgz'],
+  'image/t38': ['t38'],
+  'image/tiff': ['tif', 'tiff'],
+  'image/tiff-fx': ['tfx'],
+  'image/webp': ['webp'],
+  'image/wmf': ['wmf'],
+  'message/disposition-notification': ['disposition-notification'],
+  'message/global': ['u8msg'],
+  'message/global-delivery-status': ['u8dsn'],
+  'message/global-disposition-notification': ['u8mdn'],
+  'message/global-headers': ['u8hdr'],
+  'message/rfc822': ['eml', 'mime'],
+  'model/3mf': ['3mf'],
+  'model/gltf+json': ['gltf'],
+  'model/gltf-binary': ['glb'],
+  'model/iges': ['igs', 'iges'],
+  'model/mesh': ['msh', 'mesh', 'silo'],
+  'model/mtl': ['mtl'],
+  'model/obj': ['obj'],
+  'model/stl': ['stl'],
+  'model/vrml': ['wrl', 'vrml'],
+  'model/x3d+binary': ['*x3db', 'x3dbz'],
+  'model/x3d+fastinfoset': ['x3db'],
+  'model/x3d+vrml': ['*x3dv', 'x3dvz'],
+  'model/x3d+xml': ['x3d', 'x3dz'],
+  'model/x3d-vrml': ['x3dv'],
+  'text/cache-manifest': ['appcache', 'manifest'],
+  'text/calendar': ['ics', 'ifb'],
+  'text/coffeescript': ['coffee', 'litcoffee'],
+  'text/css': ['css'],
+  'text/csv': ['csv'],
+  'text/html': ['html', 'htm', 'shtml'],
+  'text/jade': ['jade'],
+  'text/jsx': ['jsx'],
+  'text/less': ['less'],
+  'text/markdown': ['markdown', 'md'],
+  'text/mathml': ['mml'],
+  'text/mdx': ['mdx'],
+  'text/n3': ['n3'],
+  'text/plain': ['txt', 'text', 'conf', 'def', 'list', 'log', 'in', 'ini'],
+  'text/richtext': ['rtx'],
+  'text/rtf': ['*rtf'],
+  'text/sgml': ['sgml', 'sgm'],
+  'text/shex': ['shex'],
+  'text/slim': ['slim', 'slm'],
+  'text/spdx': ['spdx'],
+  'text/stylus': ['stylus', 'styl'],
+  'text/tab-separated-values': ['tsv'],
+  'text/troff': ['t', 'tr', 'roff', 'man', 'me', 'ms'],
+  'text/turtle': ['ttl'],
+  'text/uri-list': ['uri', 'uris', 'urls'],
+  'text/vcard': ['vcard'],
+  'text/vtt': ['vtt'],
+  'text/xml': ['*xml'],
+  'text/yaml': ['yaml', 'yml'],
+  'video/3gpp': ['3gp', '3gpp'],
+  'video/3gpp2': ['3g2'],
+  'video/h261': ['h261'],
+  'video/h263': ['h263'],
+  'video/h264': ['h264'],
+  'video/iso.segment': ['m4s'],
+  'video/jpeg': ['jpgv'],
+  'video/jpm': ['*jpm', 'jpgm'],
+  'video/mj2': ['mj2', 'mjp2'],
+  'video/mp2t': ['ts'],
+  'video/mp4': ['mp4', 'mp4v', 'mpg4'],
+  'video/mpeg': ['mpeg', 'mpg', 'mpe', 'm1v', 'm2v'],
+  'video/ogg': ['ogv'],
+  'video/quicktime': ['qt', 'mov'],
+  'video/webm': ['webm'],
+};
+
+export const mimeTypes: Map<string, string> = new Map();
+
+for (const [contentType, fileExtensions] of Object.entries(mimeMap)) {
+  fileExtensions.forEach((extension) =>
+    mimeTypes.set(`.${extension}`, contentType),
+  );
+}

package/src/monitoring.ts

@@ -0,0 +1,53 @@
+import {
+  Config,
+  IResourceLoad,
+  createLogger,
+} from '@browserless.io/browserless';
+import si from 'systeminformation';
+
+export class Monitoring {
+  private log = createLogger('hardware');
+  constructor(private config: Config) {}
+
+  public getMachineStats = async (): Promise<IResourceLoad> => {
+    const [cpuLoad, memLoad] = await Promise.all([
+      si.currentLoad(),
+      si.mem(),
+    ]).catch((err) => {
+      this.log(`Error checking machine stats`, err);
+      return [null, null];
+    });
+
+    const cpu = cpuLoad ? cpuLoad.currentLoadUser / 100 : null;
+    const memory = memLoad ? memLoad.active / memLoad.total : null;
+
+    return {
+      cpu,
+      memory,
+    };
+  };
+
+  public overloaded = async (): Promise<{
+    cpuInt: number | null;
+    cpuOverloaded: boolean;
+    memoryInt: number | null;
+    memoryOverloaded: boolean;
+  }> => {
+    const { cpu, memory } = await this.getMachineStats();
+    const cpuInt = cpu && Math.ceil(cpu * 100);
+    const memoryInt = memory && Math.ceil(memory * 100);
+
+    this.log(`Checking overload status: CPU ${cpuInt}% Memory ${memoryInt}%`);
+
+    const cpuOverloaded = !!(cpuInt && cpuInt >= this.config.getCPULimit());
+    const memoryOverloaded = !!(
+      memoryInt && memoryInt >= this.config.getMemoryLimit()
+    );
+    return {
+      cpuInt,
+      cpuOverloaded,
+      memoryInt,
+      memoryOverloaded,
+    };
+  };
+}

package/src/routes/chromium/http/content-post.ts

@@ -0,0 +1,238 @@
+import {
+  APITags,
+  BadRequest,
+  BrowserHTTPRoute,
+  BrowserInstance,
+  CDPChromium,
+  CDPLaunchOptions,
+  HTTPRoutes,
+  Methods,
+  Request,
+  SystemQueryParameters,
+  UnwrapPromise,
+  WaitForEventOptions,
+  WaitForFunctionOptions,
+  WaitForSelectorOptions,
+  bestAttempt,
+  bestAttemptCatch,
+  contentTypes,
+  noop,
+  rejectRequestPattern,
+  rejectResourceTypes,
+  requestInterceptors,
+  setJavaScriptEnabled,
+  waitForEvent as waitForEvt,
+  waitForFunction as waitForFn,
+  writeResponse,
+} from '@browserless.io/browserless';
+import { Page } from 'puppeteer-core';
+import { ServerResponse } from 'http';
+
+export interface BodySchema {
+  addScriptTag?: Array<Parameters<Page['addScriptTag']>[0]>;
+  addStyleTag?: Array<Parameters<Page['addStyleTag']>[0]>;
+  authenticate?: Parameters<Page['authenticate']>[0];
+  bestAttempt?: bestAttempt;
+  cookies?: Array<Parameters<Page['setCookie']>[0]>;
+  emulateMediaType?: Parameters<Page['emulateMediaType']>[0];
+  gotoOptions?: Parameters<Page['goto']>[1];
+  html?: Parameters<Page['setContent']>[0];
+  rejectRequestPattern?: rejectRequestPattern[];
+  rejectResourceTypes?: rejectResourceTypes[];
+  requestInterceptors?: Array<requestInterceptors>;
+  setExtraHTTPHeaders?: Parameters<Page['setExtraHTTPHeaders']>[0];
+  setJavaScriptEnabled?: setJavaScriptEnabled;
+  url?: Parameters<Page['goto']>[0];
+  userAgent?: Parameters<Page['setUserAgent']>[0];
+  viewport?: Parameters<Page['setViewport']>[0];
+  waitForEvent?: WaitForEventOptions;
+  waitForFunction?: WaitForFunctionOptions;
+  waitForSelector?: WaitForSelectorOptions;
+  waitForTimeout?: Parameters<Page['waitForTimeout']>[0];
+}
+
+/**
+ * An HTML payload of the website or HTML after JavaScript
+ * parsing and execution.
+ */
+export type ResponseSchema = string;
+
+export type QuerySchema = SystemQueryParameters & {
+  launch?: CDPLaunchOptions | string;
+};
+
+const route: BrowserHTTPRoute = {
+  accepts: [contentTypes.json],
+  auth: true,
+  browser: CDPChromium,
+  concurrency: true,
+  contentTypes: [contentTypes.html],
+  description: `A JSON-based API. Given a "url" or "html" field, runs and returns HTML content after the page has loaded and JavaScript has parsed.`,
+
+  handler: async (
+    req: Request,
+    res: ServerResponse,
+    browser: BrowserInstance,
+  ): Promise<void> => {
+    const contentType =
+      !req.headers.accept || req.headers.accept?.includes('*')
+        ? contentTypes.html
+        : req.headers.accept;
+
+    if (!req.body) {
+      throw new BadRequest(`Couldn't parse JSON body`);
+    }
+
+    res.setHeader('Content-Type', contentType);
+
+    const {
+      bestAttempt = false,
+      url,
+      gotoOptions,
+      html,
+      authenticate,
+      addScriptTag = [],
+      addStyleTag = [],
+      cookies = [],
+      emulateMediaType,
+      rejectRequestPattern = [],
+      requestInterceptors = [],
+      rejectResourceTypes = [],
+      setExtraHTTPHeaders,
+      setJavaScriptEnabled,
+      userAgent,
+      viewport,
+      waitForTimeout,
+      waitForFunction,
+      waitForSelector,
+      waitForEvent,
+    } = req.body as BodySchema;
+
+    const content = url || html;
+
+    if (!content) {
+      throw new BadRequest(`One of "url" or "html" properties are required.`);
+    }
+
+    const page = (await browser.newPage()) as UnwrapPromise<
+      ReturnType<CDPChromium['newPage']>
+    >;
+    const gotoCall = url ? page.goto.bind(page) : page.setContent.bind(page);
+
+    if (emulateMediaType) {
+      await page.emulateMediaType(emulateMediaType);
+    }
+
+    if (cookies.length) {
+      await page.setCookie(...cookies);
+    }
+
+    if (viewport) {
+      await page.setViewport(viewport);
+    }
+
+    if (userAgent) {
+      await page.setUserAgent(userAgent);
+    }
+
+    if (authenticate) {
+      await page.authenticate(authenticate);
+    }
+
+    if (setExtraHTTPHeaders) {
+      await page.setExtraHTTPHeaders(setExtraHTTPHeaders);
+    }
+
+    if (setJavaScriptEnabled) {
+      await page.setJavaScriptEnabled(setJavaScriptEnabled);
+    }
+
+    if (
+      rejectRequestPattern.length ||
+      requestInterceptors.length ||
+      rejectResourceTypes.length
+    ) {
+      await page.setRequestInterception(true);
+
+      page.on('request', (req) => {
+        if (
+          !!rejectRequestPattern.find((pattern) => req.url().match(pattern)) ||
+          rejectResourceTypes.includes(req.resourceType())
+        ) {
+          return req.abort();
+        }
+        const interceptor = requestInterceptors.find((r) =>
+          req.url().match(r.pattern),
+        );
+        if (interceptor) {
+          return req.respond(interceptor.response);
+        }
+        return req.continue();
+      });
+    }
+
+    if (addStyleTag.length) {
+      for (const tag in addStyleTag) {
+        await page.addStyleTag(addStyleTag[tag]);
+      }
+    }
+
+    if (addScriptTag.length) {
+      for (const tag in addScriptTag) {
+        await page.addScriptTag(addScriptTag[tag]);
+      }
+    }
+
+    const gotoResponse = await gotoCall(content, gotoOptions).catch(
+      bestAttemptCatch(bestAttempt),
+    );
+
+    if (waitForTimeout) {
+      await page
+        .waitForTimeout(waitForTimeout)
+        .catch(bestAttemptCatch(bestAttempt));
+    }
+
+    if (waitForFunction) {
+      await waitForFn(page, waitForFunction).catch(
+        bestAttemptCatch(bestAttempt),
+      );
+    }
+
+    if (waitForSelector) {
+      const { selector, hidden, timeout, visible } = waitForSelector;
+      await page
+        .waitForSelector(selector, { hidden, timeout, visible })
+        .catch(bestAttemptCatch(bestAttempt));
+    }
+
+    if (waitForEvent) {
+      await waitForEvt(page, waitForEvent).catch(bestAttemptCatch(bestAttempt));
+    }
+
+    const headers = {
+      'X-Response-Code': gotoResponse?.status(),
+      'X-Response-IP': gotoResponse?.remoteAddress().ip,
+      'X-Response-Port': gotoResponse?.remoteAddress().port,
+      'X-Response-Status': gotoResponse?.statusText(),
+      'X-Response-URL': gotoResponse?.url().substring(0, 1000),
+    };
+
+    for (const [key, value] of Object.entries(headers)) {
+      if (value !== undefined) {
+        res.setHeader(key, value);
+      }
+    }
+
+    const markup = await page.content();
+
+    page.close().catch(noop);
+
+    return writeResponse(res, 200, markup, contentTypes.html);
+  },
+  method: Methods.post,
+  path: HTTPRoutes.content,
+  tags: [APITags.browserAPI],
+};
+
+export default route;