@browserless.io/browserless 2.0.0-beta-1

package/scripts/build-open-api.js

@@ -0,0 +1,274 @@
+#!/usr/bin/env node
+/* global process */
+'use strict';
+
+import { join, parse } from 'path';
+import { dirname } from 'path';
+import { fileURLToPath } from 'url';
+import fs from 'fs/promises';
+import { marked } from 'marked';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const moduleMain = import.meta.url.endsWith(process.argv[1]);
+const swaggerJSONPath = join(__dirname, '..', 'static', 'docs', 'swagger.json');
+const packageJSONPath = join(__dirname, '..', 'package.json');
+
+const readFileOrNull = async (path) => {
+  if (!path) {
+    return 'null';
+  }
+
+  try {
+    const content = await fs.readFile(path);
+    return content.toString();
+  } catch (e) {
+    return 'null';
+  }
+};
+
+const sortSwaggerRequiredAlpha = (prop, otherProp) => {
+  if (prop.required === otherProp.required) {
+    if (prop.name < otherProp.name) {
+      return -1;
+    }
+    if (prop.name > otherProp.name) {
+      return 1;
+    }
+    return 0;
+  }
+  return Number(otherProp.required) - Number(prop.required);
+};
+
+const buildOpenAPI = async (
+  externalHTTPRoutes = [],
+  externalWebSocketRoutes = [],
+) => {
+  const [{ getRouteFiles }, { Config }, { errorCodes }, packageJSON] =
+    await Promise.all([
+      import('../build/utils.js'),
+      import('../build/config.js'),
+      import('../build/http.js'),
+      fs.readFile(packageJSONPath),
+    ]);
+
+  const isWin = process.platform === 'win32';
+  const readme = (await fs.readFile('README.md').catch(() => '')).toString();
+  const changelog = marked.parse(
+    (await fs.readFile('CHANGELOG.md').catch(() => '')).toString(),
+  );
+  const [httpRoutes, wsRoutes] = await getRouteFiles(new Config());
+  const swaggerJSON = {
+    customSiteTitle: 'Browserless Documentation',
+    definitions: {},
+    info: {
+      description: readme + changelog,
+      title: 'Browserless',
+      version: JSON.parse(packageJSON.toString()).version,
+      'x-logo': {
+        altText: 'browserless logo',
+        url: './docs/browserless-logo.png',
+      },
+    },
+    openapi: '3.0.0',
+    paths: {},
+    servers: [],
+    // Inject routes here...
+  };
+
+  const routeMetaData = await Promise.all(
+    [
+      ...httpRoutes,
+      ...wsRoutes,
+      ...externalHTTPRoutes,
+      ...externalWebSocketRoutes,
+    ]
+      .filter((r) => r.endsWith('.js'))
+      .sort()
+      .map(async (routeModule) => {
+        const routeImport = `${isWin ? 'file:///' : ''}${routeModule}`;
+        const { default: route } = await import(routeImport);
+        if (!route) {
+          throw new Error(`Invalid route file to import docs ${routeModule}`);
+        }
+        const { name } = parse(routeModule);
+        const body = routeModule.replace('.js', '.body.json');
+        const query = routeModule.replace('.js', '.query.json');
+        const response = routeModule.replace('.js', '.response.json');
+        const isWebSocket = routeModule.includes('/ws/') || name.endsWith('ws');
+
+        const {
+          tags,
+          description,
+          auth,
+          path,
+          method,
+          accepts,
+          contentTypes,
+          title,
+        } = route;
+
+        return {
+          accepts,
+          auth,
+          body: isWebSocket ? null : JSON.parse(await readFileOrNull(body)),
+          contentTypes,
+          description,
+          isWebSocket,
+          method,
+          path,
+          query: JSON.parse(await readFileOrNull(query)),
+          response: isWebSocket
+            ? null
+            : JSON.parse(await readFileOrNull(response)),
+          tags,
+          title,
+        };
+      }),
+  );
+
+  const paths = routeMetaData.reduce((accum, r) => {
+    const swaggerRoute = {
+      definitions: {},
+      description: r.description,
+      parameters: [],
+      requestBody: {
+        content: {},
+      },
+      responses: {
+        ...errorCodes,
+      },
+      summary: r.path,
+      tags: r.tags,
+    };
+
+    r.method = r.isWebSocket ? 'get' : r.method;
+
+    // Find all the swagger definitions and merge them into the
+    // definitions object
+    const allDefs = {
+      ...(r?.body?.definitions || {}),
+      ...(r?.query?.definitions || {}),
+      ...(r?.response?.definitions || {}),
+    };
+
+    Object.entries(allDefs).forEach(([defName, definition]) => {
+      // @ts-ignore
+      swaggerJSON.definitions[defName] =
+        // @ts-ignore
+        swaggerJSON.definitions[defName] ?? definition;
+    });
+
+    if (r.isWebSocket) {
+      swaggerRoute.responses['101'] = {
+        description: 'Indicates successful WebSocket upgrade.',
+      };
+    }
+
+    // Does a best-attempt at configuring multiple response types
+    // Won't figure out APIs that return mixed response types like
+    // JSON and binary blobs
+    if (r.response) {
+      if (r.contentTypes.length === 1) {
+        const [type] = r.contentTypes;
+        swaggerRoute.responses['200'] = {
+          content: {
+            [type]: {
+              schema: r.response,
+            },
+          },
+          description: r.response.description,
+        };
+      } else {
+        const okResponses = r.contentTypes.reduce(
+          (accum, c) => {
+            // @ts-ignore
+            accum.content[c] = {
+              schema: {
+                type: 'text',
+              },
+            };
+            return accum;
+          },
+          {
+            content: {},
+            description: r.response.description,
+          },
+        );
+        swaggerRoute.responses['200'] = okResponses;
+      }
+    }
+
+    // Does a best-attempt at configuring multiple body types and
+    // ignores the "accepts" properties on routes since we can't
+    // yet correlate the accepted types to the proper body
+    if (r.body) {
+      const { properties, type, anyOf } = r.body;
+      if (anyOf) {
+        // @ts-ignore
+        anyOf.forEach((anyType) => {
+          if (anyType.type === 'string') {
+            const type = r.accepts.filter(
+              // @ts-ignore
+              (accept) => accept !== 'application/json',
+            );
+            swaggerRoute.requestBody.content[type] = {
+              schema: {
+                type: 'string',
+              },
+            };
+          }
+
+          if (anyType['$ref']) {
+            swaggerRoute.requestBody.content['application/json'] = {
+              schema: {
+                $ref: anyType['$ref'],
+              },
+            };
+          }
+        });
+      }
+
+      // Handle JSON
+      if (type === 'object') {
+        swaggerRoute.requestBody.content['application/json'] = {
+          schema: {
+            properties,
+            type: 'object',
+          },
+        };
+      }
+    }
+
+    // Queries are easy in comparison, but still have to be iterated
+    // over and made open-api-able
+    if (r.query) {
+      const { properties, required } = r.query;
+      const props = Object.keys(properties || {});
+      if (props.length) {
+        swaggerRoute.parameters = props
+          .map((prop) => ({
+            in: 'query',
+            name: prop,
+            required: required?.includes(prop),
+            schema: properties[prop],
+          }))
+          .sort(sortSwaggerRequiredAlpha);
+      }
+    }
+
+    // @ts-ignore
+    accum[r.path] = accum[r.path] || {};
+    // @ts-ignore
+    accum[r.path][r.method] = swaggerRoute;
+
+    return accum;
+  }, {});
+  swaggerJSON.paths = paths;
+  fs.writeFile(swaggerJSONPath, JSON.stringify(swaggerJSON, null, '  '));
+};
+
+export default buildOpenAPI;
+
+if (moduleMain) {
+  buildOpenAPI();
+}

package/scripts/build-schemas.js

@@ -0,0 +1,86 @@
+#!/usr/bin/env node
+/* global console, process */
+'use strict';
+
+import TJS from 'typescript-json-schema';
+import fs from 'fs/promises';
+import path from 'path';
+
+const moduleMain = import.meta.url.endsWith(process.argv[1]);
+
+const buildSchemas = async (
+  externalHTTPRoutes = [],
+  externalWebSocketRoutes = [],
+) => {
+  const { getRouteFiles, tsExtension } = await import('../build/utils.js');
+
+  const schemas = ['BodySchema', 'QuerySchema', 'ResponseSchema'];
+  const settings = {
+    ignoreErrors: true,
+    noExtraProps: true,
+    required: true,
+  };
+
+  const { compilerOptions } = JSON.parse(
+    await fs.readFile('tsconfig.json', 'utf-8'),
+  );
+
+  const { Config } = await import('../build/config.js');
+  const [httpRoutes, wsRoutes] = await getRouteFiles(new Config());
+
+  // Depending on if we're parsing an external projects routes,
+  // skip the prebuilt ones. This makes it much faster to build
+  const routesToParse = moduleMain
+    ? [...httpRoutes, ...wsRoutes]
+    : [...externalHTTPRoutes, ...externalWebSocketRoutes];
+
+  await Promise.all(
+    routesToParse
+      .filter((r) => r.endsWith(tsExtension))
+      .map(async (route) => {
+        const routeFile = (await fs.readFile(route)).toString('utf-8');
+        if (!schemas.some((schemaName) => routeFile.includes(schemaName))) {
+          return;
+        }
+
+        const program = TJS.getProgramFromFiles([route], compilerOptions, './');
+
+        return Promise.all(
+          schemas.map((schemaName) => {
+            if (routeFile.includes(schemaName)) {
+              const routePath = path.parse(route);
+              const routeName = routePath.name.replace('.d', '');
+              const schemaSuffix = schemaName
+                .replace('Schema', '')
+                .toLocaleLowerCase();
+              routePath.base = `${routeName}.${schemaSuffix}.json`;
+              const jsonPath = path.format(routePath);
+              try {
+                const schema = TJS.generateSchema(
+                  program,
+                  schemaName,
+                  settings,
+                );
+                return fs.writeFile(
+                  jsonPath,
+                  JSON.stringify(schema, null, '  '),
+                );
+              } catch (e) {
+                console.error(
+                  `Error generating schema: (${routeName}) (${jsonPath}): ${e}`,
+                );
+                return null;
+              }
+            }
+            return;
+          }),
+        );
+      }),
+  );
+};
+
+export default buildSchemas;
+
+if (moduleMain) {
+  buildSchemas();
+}

package/scripts/build-selectors.js

@@ -0,0 +1,36 @@
+#!/usr/bin/env node
+/* global console, fetch */
+'use strict';
+
+import { existsSync } from 'fs';
+import fs from 'fs/promises';
+import path from 'path';
+
+(async () => {
+  const { buildDir } = await import('../build/utils.js');
+  const dataDir = path.join(buildDir, 'data');
+  const selectorsURL =
+    'https://raw.githubusercontent.com/wanhose/cookie-dialog-monster/main/data/elements.txt';
+  const classesURL =
+    'https://raw.githubusercontent.com/wanhose/cookie-dialog-monster/main/data/classes.txt';
+
+  const get = async (url, type) => {
+    try {
+      const res = await fetch(url);
+      const json = (await res.text()).split('\n');
+      const filename = path.join(dataDir, `${type}.json`);
+      await fs.writeFile(filename, JSON.stringify(json));
+    } catch (e) {
+      console.error(e);
+    }
+  };
+
+  if (!existsSync(dataDir)) {
+    await fs.mkdir(dataDir);
+  }
+
+  await Promise.all([
+    get(selectorsURL, 'selectors'),
+    get(classesURL, 'classes'),
+  ]);
+})();

package/scripts/clean.js

@@ -0,0 +1,7 @@
+#!/usr/bin/env node
+'use strict';
+import { deleteAsync } from 'del';
+
+(async () => {
+  await deleteAsync(['build', 'static/function/*js*']);
+})();

package/scripts/install-adblock.js

@@ -0,0 +1,45 @@
+#!/usr/bin/env node
+/* global fetch, console, process */
+'use strict';
+
+import { createWriteStream, existsSync } from 'fs';
+import path, { join } from 'path';
+import { Readable } from 'stream';
+import { deleteAsync } from 'del';
+import { moveFile } from 'move-file';
+import os from 'os';
+import unzip from 'extract-zip';
+
+(async () => {
+  const zipFile = os.tmpdir() + '/ublock.zip';
+  const tmpUblockPath = path.join(os.tmpdir(), 'uBlock0.chromium'); // uBlock0.chromium is always the prod name
+  const extensionsDir = join(process.cwd(), 'extensions');
+  const uBlockDir = join(extensionsDir, 'ublock');
+
+  const downloadUrlToDirectory = (url, dir) =>
+    fetch(url).then(
+      (response) =>
+        new Promise((resolve, reject) => {
+          // @ts-ignore
+          Readable.fromWeb(response.body)
+            .pipe(createWriteStream(dir))
+            .on('error', reject)
+            .on('finish', resolve);
+        }),
+    );
+
+  if (existsSync(uBlockDir)) {
+    await deleteAsync(uBlockDir);
+  }
+  const data = await fetch(
+    'https://api.github.com/repos/gorhill/uBlock/releases/latest',
+  );
+  const json = await data.json();
+
+  await downloadUrlToDirectory(json.assets[0].browser_download_url, zipFile);
+  await unzip(zipFile, { dir: os.tmpdir() });
+  await moveFile(join(tmpUblockPath), join(extensionsDir, 'ublock'));
+  await deleteAsync(zipFile, { force: true }).catch((err) => {
+    console.warn('Could not delete temporary download file: ' + err.message);
+  });
+})();

package/scripts/install-cdp-json.js

@@ -0,0 +1,37 @@
+import fs from 'fs/promises';
+import playwright from 'playwright-core';
+import puppeteer from 'puppeteer-core';
+
+(async () => {
+  const browser = await puppeteer.launch({
+    args: [`--no-sandbox`],
+    executablePath: playwright.chromium.executablePath(),
+    pipe: true,
+  });
+  const [page] = await browser.pages();
+  const client = await page.target().createCDPSession();
+  const res = await client.send('Browser.getVersion');
+  const webKitVersion = res.userAgent.match(/AppleWebKit\/(\d+(\.\d+)*) /);
+
+  if (!webKitVersion || !webKitVersion[1]) {
+    throw new Error(
+      `Error finding WebKit Version from user agent "${res.userAgent}"`,
+    );
+  }
+
+  const payload = JSON.stringify(
+    {
+      Browser: res.product,
+      'Protocol-Version': res.protocolVersion,
+      'User-Agent': res.userAgent,
+      'WebKit-Version': `${webKitVersion[1]} (${res.revision})`,
+      webSocketDebuggerUrl: 'ws://localhost:3000',
+    },
+    null,
+    '  ',
+  );
+
+  await fs.writeFile('browser.json', payload);
+
+  return browser.close();
+})();

package/scripts/start.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+set -e
+
+# Setup env variables
+export DISPLAY=:99
+
+# When docker restarts, this file is still there,
+# so we need to kill it just in case
+[ -f /tmp/.X99-lock ] && rm -f /tmp/.X99-lock
+
+_kill_procs() {
+  kill -TERM $node
+  kill -TERM $xvfb
+}
+
+# Relay quit commands to processes
+trap _kill_procs SIGTERM SIGINT
+
+Xvfb :99 -screen 0 1920x1080x16 -nolisten tcp -nolisten unix &
+xvfb=$!
+
+dumb-init -- node build/index.js $@ &
+node=$!
+
+wait $node
+
+if [ ! -z "$xvfb" ]
+then
+  wait $xvfb
+fi

package/scripts/test.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+# Install all dependencies for tests
+npm i --production=false
+
+# Setup env variables
+export DISPLAY=:1
+export DEBUG=-*
+export PUPPETEER_DISABLE_HEADLESS_WARNING=true
+export NODE_OPTIONS="--loader ts-node/esm"
+
+Xvfb :1 -screen 0 1024x768x16 -nolisten tcp -nolisten unix &
+xvfb=$!
+
+# Run the tests
+./node_modules/.bin/_mocha --timeout 30000 --slow 10000 --exit $@ && kill -TERM $xvfb