@browserless.io/browserless 2.1.0 → 2.2.0-beta-2

package/CHANGELOG.md

@@ -1,5 +1,22 @@
-# [Latest](https://github.com/browserless/chrome/compare/v2.0.0...master)
-- Dependency updates.
+# [Latest](https://github.com/browserless/chrome/compare/v2.1.0...main)
+- Dependency updates.
+- Add `clean` command to `@browserless.io/browserless` CLI.
+- Adds support for `/json/list`, `/json/new` and `json/protocol` APIs (Chrome only).
+- Removes legacy `browser.json` files since browserless now generates those and caches when requested.
+- Only verify that internally managed browsers are installed when starting.
+
+# [v2.1.0](https://github.com/browserless/browserless/compare/v2.0.0-beta-1...v2.1.0)
+- Dependency updates.
+- Name is now `@browserless.io/browserless` to reflect our npm package.
+- NEW: SDK is now live here: https://www.npmjs.com/package/@browserless.io/browserless.
+- Drops gulp and other gulp utilities in favor of our own. Move to modules in `scripts` dir.
+- Many private class properties now use `protected` so they can be referenced in SDK extensions.
+- Adds a `/json/version` route for older libraries that use it.
+- Merge startup/test scripts into `scripts` dir.
+- Moves installed browsers to `/usr/local/bin/playwright-browsers`.
+- Merge root files into package.json where possible.
+- README updates and fixes.
+- Numerous link and copyright fixes.
 
 # [v2.0.0](https://github.com/browserless/chrome/compare/master...feat/browserless-2.0)
 browserless 2.0.0 represents the best body of work after running browserless for over 5 years. It contains mostly the same functionality and more, and is rebuilt to be more modular and offer a NodeJS SDK. It's also much lighter and faster than prior versions and includes a lot of semantic changes.

package/bin/browserless.js

@@ -14,7 +14,9 @@ import fs from 'fs/promises';
 import path from 'path';
 import { spawn } from 'child_process';
 
-debug.enable('browserless*');
+if (typeof process.env.DEBUG === 'undefined') {
+  debug.enable('browserless*');
+}
 
 const log = debug('browserless.io:sdk:log');
 const promptLog = debug('browserless.io:prompt');
@@ -22,9 +24,17 @@ const promptLog = debug('browserless.io:prompt');
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const cmd = process.argv[2];
 const subCMD = process.argv[3];
-const cwd = process.cwd();
-const allowedCMDs = ['build', 'dev', 'docker', 'start', 'create', 'help'];
-const srcDir = path.join(cwd, 'build');
+const allowedCMDs = [
+  'build',
+  'dev',
+  'docker',
+  'start',
+  'create',
+  'help',
+  'clean',
+];
+const projectDir = process.cwd();
+const compiledDir = path.join(projectDir, 'build');
 const packageJSON = readFile(path.join(__dirname, '..', 'package.json')).then(
   (r) => JSON.parse(r.toString()),
 );
@@ -59,7 +69,7 @@ const importClassOverride = async (files, className) => {
     return;
   }
 
-  const classModuleFullFilePath = path.join(srcDir, classModuleFile);
+  const classModuleFullFilePath = path.join(compiledDir, classModuleFile);
 
   if (!classModuleFile) {
     return;
@@ -68,6 +78,12 @@ const importClassOverride = async (files, className) => {
   return (await import(classModuleFullFilePath)).default;
 };
 
+const clean = async () =>
+  fs.rm(path.join(compiledDir), {
+    force: true,
+    recursive: true,
+  });
+
 const installDependencies = async (workingDirectory) =>
   new Promise((resolve, reject) => {
     spawn('npm', ['i'], {
@@ -88,7 +104,7 @@ const buildDockerImage = async (cmd) => {
   new Promise((resolve, reject) => {
     const [docker, ...args] = cmd.split(' ');
     spawn(docker, args, {
-      cwd,
+      cwd: projectDir,
       stdio: 'inherit',
     }).once('close', (code) => {
       if (code === 0) {
@@ -105,7 +121,7 @@ const buildDockerImage = async (cmd) => {
 const buildTypeScript = async () =>
   new Promise((resolve, reject) => {
     spawn('npx', ['tsc', '--outDir', 'build'], {
-      cwd,
+      cwd: projectDir,
       stdio: 'inherit',
     }).once('close', (code) => {
       if (code === 0) {
@@ -118,16 +134,16 @@ const buildTypeScript = async () =>
   });
 
 const getSourceFiles = async () => {
-  const files = await fs.readdir(srcDir, { recursive: true });
+  const files = await fs.readdir(compiledDir, { recursive: true });
   const [httpRoutes, webSocketRoutes] = files.reduce(
     ([httpRoutes, webSocketRoutes], file) => {
       const parsed = path.parse(file);
       if (parsed.name.endsWith('http')) {
-        httpRoutes.push(path.join(srcDir, file));
+        httpRoutes.push(path.join(compiledDir, file));
       }
 
       if (parsed.name.endsWith('ws')) {
-        webSocketRoutes.push(path.join(srcDir, file));
+        webSocketRoutes.push(path.join(compiledDir, file));
       }
 
       return [httpRoutes, webSocketRoutes];
@@ -177,6 +193,9 @@ const getArgSwitches = () => {
  * and validation. Doesn't start the HTTP server.
  */
 const build = async () => {
+  log(`Cleaning build directory`);
+  await clean();
+
   log(`Compiling TypeScript`);
   await buildTypeScript();
 
@@ -309,7 +328,7 @@ const start = async (dev = false) => {
 };
 
 const buildDocker = async () => {
-  const finalDockerPath = path.join(cwd, 'build', 'Dockerfile');
+  const finalDockerPath = path.join(projectDir, 'build', 'Dockerfile');
   const argSwitches = getArgSwitches();
 
   await build();
@@ -320,7 +339,10 @@ const buildDocker = async () => {
 
   log(`Generating Dockerfile at "${finalDockerPath}"`);
 
-  await fs.writeFile(path.join(cwd, 'build', 'Dockerfile'), dockerContents);
+  await fs.writeFile(
+    path.join(projectDir, 'build', 'Dockerfile'),
+    dockerContents,
+  );
 
   const from =
     argSwitches.from ||
@@ -391,7 +413,7 @@ const create = async () => {
     throw new Error(`Name must not include special characters.`);
   }
 
-  const installPath = path.join(cwd, directory);
+  const installPath = path.join(projectDir, directory);
   log(`Creating folder "${installPath}"...`);
   await fs.mkdir(installPath);
 
@@ -440,6 +462,15 @@ const help = () => {
       `);
         break;
 
+      case 'clean':
+        console.log(dedent`
+        Usage: npx @browserless.io/browserless clean
+
+        Description: Cleans the TypeScript generated JavaScript found
+          in the "build" directory and any other temporary assets.
+      `);
+        break;
+
       case 'dev':
         console.log(dedent`
         Usage: npx @browserless.io/browserless dev
@@ -492,6 +523,7 @@ const help = () => {
     Usage: npx @browserless.io/browserless [command] [arguments]
 
     Options:
+      clean     Removes build artifacts and other temporary directories.
       create    Creates a new scaffold project, installs dependencies, and exits.
       dev       Compiles TypeScript, generates build assets and starts the server.
       build     Compiles TypeScript, generates build assets and exits.
@@ -501,6 +533,10 @@ const help = () => {
 };
 
 switch (cmd) {
+  case 'clean':
+    clean();
+    break;
+
   case 'start':
     start(false);
     break;

package/bin/scaffold/tsconfig.json

@@ -1,4 +1,4 @@
 {
   "extends": "./node_modules/@browserless.io/browserless/tsconfig.json",
-  "include": ["./src/**/*"],
+  "include": ["./src/**/*"]
 }

package/build/browserless.js

@@ -1,5 +1,5 @@
 import * as path from 'path';
-import { BrowserManager, Config, FileSystem, HTTPServer, Limiter, Metrics, Monitoring, Router, Token, WebHooks, availableBrowsers, createLogger, getRouteFiles, makeExternalURL, printLogo, safeParse, } from '@browserless.io/browserless';
+import { BrowserManager, CDPChromium, Config, FileSystem, HTTPServer, Limiter, Metrics, Monitoring, PlaywrightChromium, PlaywrightFirefox, PlaywrightWebkit, Router, Token, WebHooks, availableBrowsers, createLogger, getRouteFiles, makeExternalURL, printLogo, safeParse, } from '@browserless.io/browserless';
 import { readFile } from 'fs/promises';
 import { userInfo } from 'os';
 const routeSchemas = ['body', 'query'];
@@ -88,6 +88,12 @@ export class Browserless {
     async start() {
         const httpRoutes = [];
         const wsRoutes = [];
+        const internalBrowsers = [
+            CDPChromium,
+            PlaywrightFirefox,
+            PlaywrightChromium,
+            PlaywrightWebkit,
+        ];
         const [[httpRouteFiles, wsRouteFiles], installedBrowsers] = await Promise.all([getRouteFiles(this.config), availableBrowsers]);
         const docsLink = makeExternalURL(this.config.getExternalAddress(), '/docs');
         this.debug(printLogo(docsLink));
@@ -137,10 +143,11 @@ export class Browserless {
                 wsRoutes.push(route);
             }
         }
-        // Validate that browsers are installed and route paths are unique
+        // Validate that we have the browsers they are asking for
         [...httpRoutes, ...wsRoutes].forEach((route) => {
             if ('browser' in route &&
                 route.browser &&
+                internalBrowsers.includes(route.browser) &&
                 !installedBrowsers.some((b) => b.name === route.browser?.name)) {
                 throw new Error(`Couldn't load route "${route.path}" due to missing browser of "${route.browser?.name}"`);
             }

package/build/browsers/cdp-chromium.d.ts

@@ -38,7 +38,7 @@ export declare class CDPChromium extends EventEmitter {
     process: () => import("child_process").ChildProcess | null;
     launch: (options?: CDPLaunchOptions) => Promise<Browser>;
     wsEndpoint: () => string | null;
-    publicWSEndpoint: (token: string) => string | null;
+    publicWSEndpoint: (token: string | null) => string | null;
     proxyPageWebSocket: (req: Request, socket: Duplex, head: Buffer) => Promise<void>;
     proxyWebSocket: (req: Request, socket: Duplex, head: Buffer) => Promise<void>;
 }

package/build/browsers/cdp-chromium.js

@@ -243,14 +243,14 @@ export class CDPChromium extends EventEmitter {
             throw new ServerError(`No browserWSEndpoint found, did you launch first?`);
         }
         socket.once('close', resolve);
-        this.debug(`Proxying ${req.parsed.href} to browser ${this.browserWSEndpoint}`);
         const [page] = await this.browser.pages();
         const pageLocation = `/devtools/page/${this.getPageId(page)}`;
         this.debug(`Proxying ${req.parsed.href} to page "${pageLocation}"`);
-        req.url = pageLocation;
+        const target = new URL(pageLocation, this.browserWSEndpoint).href;
+        req.url = '';
         this.proxy.ws(req, socket, head, {
             changeOrigin: true,
-            target: this.browserWSEndpoint,
+            target,
         }, (error) => {
             this.debug(`Error proxying session: ${error}`);
             this.close();

package/build/browsers/index.d.ts

@@ -6,18 +6,10 @@ export declare class BrowserManager {
     protected launching: Map<string, Promise<unknown>>;
     protected timers: Map<string, number>;
     protected debug: import("debug").Debugger;
+    protected playwrightBrowserNames: string[];
     constructor(config: Config);
     protected removeUserDataDir: (userDataDir: string | null) => Promise<void>;
-    /**
-     * Generates a directory for the user-data-dir contents to be saved in. Uses
-     * the provided sessionId, or creates one when omitted,
-     * and appends it to the name of the directory. If the
-     * directory already exists then no action is taken, verified by run `stat`
-     *
-     * @param sessionId The ID of the session
-     * @returns Promise<string> of the fully-qualified path of the directory
-     */
-    protected generateDataDir: (sessionId?: string) => Promise<string>;
+    getProtocolJSON: () => Promise<object>;
     getVersionJSON: () => Promise<{
         Browser: string;
         'Debugger-Version': string;

package/build/browsers/index.js

@@ -1,13 +1,17 @@
-import { BadRequest, CDPChromium, HTTPManagementRoutes, NotFound, ServerError, browserHook, convertIfBase64, createLogger, exists, id, makeExternalURL, noop, pageHook, parseBooleanParam, } from '@browserless.io/browserless';
-import path, { join } from 'path';
+import { BadRequest, CDPChromium, HTTPManagementRoutes, NotFound, PlaywrightChromium, PlaywrightFirefox, PlaywrightWebkit, browserHook, convertIfBase64, createLogger, exists, generateDataDir, makeExternalURL, noop, pageHook, parseBooleanParam, } from '@browserless.io/browserless';
 import { deleteAsync } from 'del';
-import { mkdir } from 'fs/promises';
+import path from 'path';
 export class BrowserManager {
     config;
     browsers = new Map();
     launching = new Map();
     timers = new Map();
     debug = createLogger('browser-manager');
+    playwrightBrowserNames = [
+        PlaywrightChromium.name,
+        PlaywrightFirefox.name,
+        PlaywrightWebkit.name,
+    ];
     constructor(config) {
         this.config = config;
     }
@@ -19,29 +23,27 @@ export class BrowserManager {
             });
         }
     };
-    /**
-     * Generates a directory for the user-data-dir contents to be saved in. Uses
-     * the provided sessionId, or creates one when omitted,
-     * and appends it to the name of the directory. If the
-     * directory already exists then no action is taken, verified by run `stat`
-     *
-     * @param sessionId The ID of the session
-     * @returns Promise<string> of the fully-qualified path of the directory
-     */
-    generateDataDir = async (sessionId = id()) => {
-        const baseDirectory = await this.config.getDataDir();
-        const dataDirPath = join(baseDirectory, `browserless-data-dir-${sessionId}`);
-        if (await exists(dataDirPath)) {
-            this.debug(`Data directory already exists, not creating "${dataDirPath}"`);
-            return dataDirPath;
-        }
-        this.debug(`Generating user-data-dir at ${dataDirPath}`);
-        await mkdir(dataDirPath, { recursive: true }).catch((err) => {
-            throw new ServerError(`Error creating data-directory "${dataDirPath}": ${err}`);
+    getProtocolJSON = async () => {
+        this.debug(`Launching Chrome to generate /json/protocol results`);
+        const browser = new CDPChromium({
+            blockAds: false,
+            config: this.config,
+            record: false,
+            userDataDir: null,
         });
-        return dataDirPath;
+        await browser.launch();
+        const wsEndpoint = browser.wsEndpoint();
+        if (!wsEndpoint) {
+            throw new Error('There was an error launching the browser');
+        }
+        const { port } = new URL(wsEndpoint);
+        const res = await fetch(`http://127.0.0.1:${port}/json/protocol`);
+        const protocolJSON = await res.json();
+        browser.close();
+        return protocolJSON;
     };
     getVersionJSON = async () => {
+        this.debug(`Launching Chrome to generate /json/version results`);
         const browser = new CDPChromium({
             blockAds: false,
             config: this.config,
@@ -174,7 +176,9 @@ export class BrowserManager {
         // Always specify a user-data-dir since plugins can "inject" their own
         // unless it's playwright which takes care of its own data-dirs
         const userDataDir = manualUserDataDir ||
-            (Browser.name === CDPChromium.name ? await this.generateDataDir() : null);
+            (!this.playwrightBrowserNames.includes(Browser.name)
+                ? await generateDataDir(undefined, this.config)
+                : null);
         const proxyServerArg = launchOptions.args?.find((arg) => arg.includes('--proxy-server='));
         if (launchOptions.args &&
             proxyServerArg &&

package/build/browsers/playwright-chromium.d.ts

@@ -31,7 +31,7 @@ export declare class PlaywrightChromium extends EventEmitter {
     newPage: () => Promise<Page>;
     launch: (options?: BrowserServerOptions) => Promise<playwright.BrowserServer>;
     wsEndpoint: () => string | null;
-    publicWSEndpoint: (token: string) => string | null;
+    publicWSEndpoint: (token: string | null) => string | null;
     proxyPageWebSocket: () => Promise<void>;
     proxyWebSocket: (req: Request, socket: Duplex, head: Buffer) => Promise<void>;
 }

package/build/browsers/playwright-firefox.d.ts

@@ -31,7 +31,7 @@ export declare class PlaywrightFirefox extends EventEmitter {
     newPage: () => Promise<Page>;
     launch: (options?: BrowserServerOptions) => Promise<playwright.BrowserServer>;
     wsEndpoint: () => string | null;
-    publicWSEndpoint: (token: string) => string | null;
+    publicWSEndpoint: (token: string | null) => string | null;
     proxyPageWebSocket: () => Promise<void>;
     proxyWebSocket: (req: Request, socket: Duplex, head: Buffer) => Promise<void>;
 }

package/build/browsers/playwright-webkit.d.ts

@@ -31,7 +31,7 @@ export declare class PlaywrightWebkit extends EventEmitter {
     newPage: () => Promise<Page>;
     launch: (options?: BrowserServerOptions) => Promise<playwright.BrowserServer>;
     wsEndpoint: () => string | null;
-    publicWSEndpoint: (token: string) => string | null;
+    publicWSEndpoint: (token: string | null) => string | null;
     proxyPageWebSocket: () => Promise<void>;
     proxyWebSocket: (req: Request, socket: Duplex, head: Buffer) => Promise<void>;
 }

package/build/constants.d.ts

@@ -2,3 +2,4 @@ export declare const encryptionAlgo = "aes-192-cbc";
 export declare const encryptionSep = ".";
 export declare const liveURLSep = ":";
 export declare const keyLength = 24;
+export declare const BLESS_PAGE_IDENTIFIER = "BLESS";

package/build/constants.js

@@ -2,3 +2,4 @@ export const encryptionAlgo = 'aes-192-cbc';
 export const encryptionSep = '.';
 export const liveURLSep = ':';
 export const keyLength = 24;
+export const BLESS_PAGE_IDENTIFIER = 'BLESS';

package/build/data/classes.json

@@ -1 +1 @@
-["b-modal-banner--disabled","blocked","bodyBlocked","c24-cc-visible","cbar","cmp-open","consent-modal-open","consent-modal-overflow","CookiePolicy--show","cookies","cu_k_cookie_consent_modal_open","gcdc-locked","gdpr-infobar-visible","gdprbanner_consent_gdpr_consent","gdprCookieBanner-acceptedAll","hasCookieBanner","ibeugdpr-disabled","modal--is-open","modal-open","no_scroll","no-scroll","shopee-no-scroll","show-cookie-policy-info","sp-message-open","ta-cc-modal-open","touchevents-false","wt-cli-eu-country","wt-cli-geoip-on"]
+["b-modal-banner--disabled","blocked","bodyBlocked","c24-cc-visible","cbar","ccm-blocked","cmp-open","consent-modal-open","consent-modal-overflow","CookiePolicy--show","cookies","cu_k_cookie_consent_modal_open","gcdc-locked","gdpr-infobar-visible","gdprbanner_consent_gdpr_consent","gdprCookieBanner-acceptedAll","hasCookieBanner","ibeugdpr-disabled","idxrcookies-block-user-nav","modal--is-open","modal-open","no_scroll","no-scroll","noScroll","popin-gdpr-no-scroll","shopee-no-scroll","show-cookie-policy-info","sp-message-open","start-cookies","ta-cc-modal-open","touchevents-false","wt-cli-eu-country","wt-cli-geoip-on"]