@browserless.io/browserless 2.7.1 → 2.9.0

package/CHANGELOG.md

@@ -1,4 +1,22 @@
-# [Latest](https://github.com/browserless/chrome/compare/v2.7.1...main)
+# [Latest](https://github.com/browserless/chrome/compare/v2.9.0...main)
+- Dependency updates.
+
+# [v2.9.0](https://github.com/browserless/chrome/compare/v2.8.0...v2.9.0)
+- Dependency updates.
+- Debugger is now included and mounted under the `/debugger` path.
+- Browserless now uses the `Logger` utility internally. Better logs for certain classes as well.
+- Fix number of connected clients when using the Page websocket route.
+- Allows "HEAD" requests for most "GET"-based APIs.
+-
+
+# [v2.8.0](https://github.com/browserless/chrome/compare/v2.7.1...v2.8.0)
+**April 12, 2024**
+**Potentially Breaking**
+- New `Logger` class and SDK primitives in support of that.
+- Routes now get an instance of `Logger`, before the `browser` argument or as the last argument for HTTP routes.
+
+**Updates**
+- Numerous SDK fixes and updates in the CLI.
 - Dependency updates.
 
 # [v2.7.1](https://github.com/browserless/chrome/compare/v2.7.0...v2.7.1)

package/README.md

@@ -1,4 +1,4 @@
-<!-- markdownlint-disable first-line-h1 no-emphasis-as-heading -->
+<!-- markdownlint-disable commands-show-output first-line-h1 no-emphasis-as-heading -->
 
 ![browserless.io logo](/assets/logo.png)
 
@@ -28,6 +28,8 @@ If you've been struggling to deploy headless browsers without running into issue
   - [Puppeteer](#puppeteer)
   - [Playwright](#playwright)
 - [Extending (NodeJS SDK)](#extending-nodejs-sdk)
+- [Debugger](#debugger)
+  - [Install debugger](#install-debugger)
 - [Usage with other libraries](#usage-with-other-libraries)
 - [Motivations](#motivations)
 - [Licensing](#licensing)
@@ -70,7 +72,7 @@ You still execute the script itself which gives you total control over what libr
 
 ### Docker
 
-> [!TIP]  
+> [!TIP]
 > See more options on our [full documentation site](https://docs.browserless.io/Docker/docker-quickstart).
 
 1. `docker run -p 3000:3000 ghcr.io/browserless/chromium`
@@ -107,7 +109,7 @@ const browser = await puppeteer.connect({
 
 ### Playwright
 
-We support running with playwright via their their browser's remot connection protocols interface out of the box. Just make sure that your Docker image, playwright browser type _and_ endpoint match:
+We support running with playwright via their their browser's remote connection protocols interface out of the box. Just make sure that your Docker image, playwright browser type _and_ endpoint match:
 
 **Before**
 
@@ -137,6 +139,42 @@ After that, the rest of your code remains the same with no other changes require
 
 Browserless comes with built-in extension capabilities, and allows for extending nearly any aspect of the system (for Version 2+). For more details on how to write your own routes, build docker images, and more, [see our SDK README.md](/bin/scaffold/README.md) or simply run "npx @browserless.io/browserless create" in a terminal and follow the onscreen prompts.
 
+## Debugger
+
+You can install a first-party interactive debugger for Browserless, that makes writing scripts faster and interactive. You can take advantage of things like `debugger;` calls and the page's console output to see what's happening on the page while your script is running. All of the Chrome devtools are there at your disposal.
+
+![browserless.io logo](/assets/debugger.png)
+
+A small list of features includes:
+
+- Running `debugger;` and `console.log` calls
+- Errors in the script are caught and show up in the console tab
+- DOM inspection, watch network requests, and even see how the page is rendering
+- Exporting you debugging script as a Node project
+- Everything included in Chrome DevTools
+
+### Install debugger
+
+Installing the debugger is as simple as running the `install:debugger` script _after_ the project has been built. This way:
+
+```sh
+$ npm run build
+$ npm run install:debugger #or npm install:dev
+```
+
+You will then see the debugger url during the startup process.
+
+```log
+---------------------------------------------------------
+| browserless.io
+| To read documentation and more, load in your browser:
+|
+| OpenAPI: http://localhost:3000/docs
+| Full Documentation: https://docs.browserless.io/
+| Debbuger: http://localhost:3000/debugger/?token=6R0W53R135510
+---------------------------------------------------------
+```
+
 ## Usage with other libraries
 
 Most libraries allow you to specify a remote instance of Chrome to interact with. They are either looking for a websocket endpoint, a host and port, or some address. Browserless supports these by default, however if you're having issues please make an issue in this project and we'll try and work with the library authors to get them integrated with browserless. Please note that in V2 we no longer support selenium or webdriver integrations.

package/bin/browserless.js

@@ -27,7 +27,7 @@ if (typeof process.env.DEBUG === 'undefined') {
 const log = debug('browserless.io:sdk:log');
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
-const cmd = process.argv[2];
+const cmd = process.argv[2] ?? 'help';
 const subCMD = process.argv[3];
 const allowedCMDs = [
   'build',
@@ -143,7 +143,7 @@ const isConstructor = (reference) => typeof reference === 'function';
 const start = async (dev = false) => {
   const { httpRoutes, webSocketRoutes, files } = dev
     ? await build()
-    : await getSourceFiles();
+    : await getSourceFiles(projectDir);
 
   log(`Importing all class overrides if present`);
 
@@ -152,6 +152,7 @@ const start = async (dev = false) => {
     Config,
     FileSystem,
     Limiter,
+    Logger,
     Metrics,
     Monitoring,
     Router,
@@ -164,6 +165,7 @@ const start = async (dev = false) => {
     importDefault(files, 'config'),
     importDefault(files, 'file-system'),
     importDefault(files, 'limiter'),
+    importDefault(files, 'logger'),
     importDefault(files, 'metrics'),
     importDefault(files, 'monitoring'),
     importDefault(files, 'router'),
@@ -197,6 +199,7 @@ const start = async (dev = false) => {
     : Router;
 
   const browserless = new Browserless({
+    Logger,
     browserManager,
     config,
     fileSystem,
@@ -369,7 +372,7 @@ const create = async () => {
     }
   }
 
-  log(`Installing npm modules...`);
+  log(`Installing browsers and npm modules, this might take a few minutes...`);
   await installDependencies(installPath);
 
   log(
@@ -377,7 +380,8 @@ const create = async () => {
   );
 };
 
-const help = () => {
+const help = async () => {
+  console.log(`Version: ${(await browserlessPackageJSON).version}`);
   if (subCMD) {
     if (!allowedCMDs.includes(subCMD)) {
       throw new Error(`Unknown command of "${subCMD}" passed.`);

package/bin/scaffold/README.md

@@ -128,6 +128,7 @@ Internally, we use this same class-based system, so feel free to see how those w
 import {
   APITags,
   HTTPRoute,
+  Logger,
   Methods,
   contentTypes,
   writeResponse,
@@ -174,7 +175,7 @@ export default class HelloWorldRoute extends HTTPRoute {
   // Handler is a function, getting the request and response objects, and is where you'll write the
   // core logic behind this route. Use utilities like writeResponse or writeJSONResponse to help
   // return the appropriate response.
-  handler = async (_req, res): Promise<void> => {
+  handler = async (_req, res, _logger: Logger): Promise<void> => {
     const response: ResponseSchema = 'Hello World!';
     return writeResponse(res, 200, ResponseSchema, contentTypes.text);
   };
@@ -189,6 +190,7 @@ import {
   BrowserWebsocketRoute,
   ChromiumCDP,
   CDPLaunchOptions,
+  Logger,
   Request,
   SystemQueryParameters,
   WebsocketRoutes,
@@ -226,7 +228,7 @@ export default class ChromiumWebSocketRoute extends BrowserWebsocketRoute {
   // Routes with a browser type get a browser argument of the Browser instance, otherwise
   // request, socket, and head are the other 3 arguments. Here we pass them through
   // and proxy the request into Chromium to handle.
-  handler = async (req, socket, head, chromium): Promise<void> =>
+  handler = async (req, socket, head, logger, chromium): Promise<void> =>
     chromium.proxyWebSocket(req, socket, head);
 }
 ```
@@ -293,7 +295,7 @@ Then, later, in your route you can define some functionality and load the config
 
 ```ts
 // src/pdf.http.ts
-import { BrowserHTTPRoute } from '@browserless.io/browserless';
+import { BrowserHTTPRoute, Logger } from '@browserless.io/browserless';
 import MyConfig from './config';
 
 // Export the BodySchema for documentation site to parse, plus
@@ -342,7 +344,7 @@ export default class PDFToS3Route extends BrowserHTTPRoute {
   tags = [APITags.browserAPI];
 
   // Handler's are where we embed the logic that facilitates this route.
-  handler = async (req, res, browser): Promise<void> => {
+  handler = async (req, res, logger, browser): Promise<void> => {
     // Modules like Config are injected via this internal methods.
     // Use them to load core modules within the platform.
     const config = this.config() as MyConfig;

package/bin/scaffold/src/hello-world.http.ts

@@ -1,6 +1,7 @@
 import {
   APITags,
   HTTPRoute,
+  Logger,
   Methods,
   Request,
   Response,
@@ -21,7 +22,12 @@ export default class HelloWorldHTTPRoute extends HTTPRoute {
   method = Methods.get;
   path = '/hello';
   tags = [APITags.management];
-  handler = async (_req: Request, res: Response): Promise<void> => {
+  handler = async (
+    req: Request,
+    res: Response,
+    logger: Logger,
+  ): Promise<void> => {
+    logger.verbose(`${req.method} /hello was called!`);
     const response: ResponseSchema = 'Hello World!';
     return writeResponse(res, 200, response, contentTypes.text);
   };

package/build/browserless.d.ts

@@ -1,15 +1,16 @@
-/// <reference types="debug" />
 /// <reference types="node" />
 /// <reference types="node" />
-import { BrowserManager, Config, FileSystem, HTTPServer, Hooks, Limiter, Metrics, Monitoring, Router, Token, WebHooks } from '@browserless.io/browserless';
+import { Logger as BlessLogger, BrowserHTTPRoute, BrowserManager, BrowserWebsocketRoute, Config, FileSystem, HTTPRoute, HTTPServer, Hooks, Limiter, Metrics, Monitoring, Router, Token, WebHooks, WebSocketRoute } from '@browserless.io/browserless';
 import { EventEmitter } from 'events';
+type routeInstances = HTTPRoute | BrowserHTTPRoute | WebSocketRoute | BrowserWebsocketRoute;
 export declare class Browserless extends EventEmitter {
-    protected debug: debug.Debugger;
+    protected logger: BlessLogger;
     protected browserManager: BrowserManager;
     protected config: Config;
     protected fileSystem: FileSystem;
     protected hooks: Hooks;
     protected limiter: Limiter;
+    protected Logger: typeof BlessLogger;
     protected metrics: Metrics;
     protected monitoring: Monitoring;
     protected router: Router;
@@ -22,7 +23,8 @@ export declare class Browserless extends EventEmitter {
     server?: HTTPServer;
     metricsSaveInterval: number;
     metricsSaveIntervalID?: NodeJS.Timer;
-    constructor({ browserManager, config, fileSystem, hooks, limiter, metrics, monitoring, router, token, webhooks, }?: {
+    constructor({ browserManager, config, fileSystem, hooks, limiter, Logger: LoggerOverride, metrics, monitoring, router, token, webhooks, }?: {
+        Logger?: Browserless['Logger'];
         browserManager?: Browserless['browserManager'];
         config?: Browserless['config'];
         fileSystem?: Browserless['fileSystem'];
@@ -36,7 +38,7 @@ export declare class Browserless extends EventEmitter {
     });
     protected saveMetrics: () => Promise<void>;
     setMetricsSaveInterval: (interval: number) => void;
-    private routeIsDisabled;
+    protected routeIsDisabled(route: routeInstances): boolean;
     setStaticSDKDir(dir: string): void;
     disableRoutes(...routeNames: string[]): void;
     addHTTPRoute(httpRouteFilePath: string): void;
@@ -45,3 +47,4 @@ export declare class Browserless extends EventEmitter {
     stop(): Promise<[void | undefined, void, void, void, void, void, void, void, void, void, void]>;
     start(): Promise<void>;
 }
+export {};

package/build/browserless.js

@@ -1,16 +1,17 @@
 import * as path from 'path';
-import { BrowserManager, ChromeCDP, ChromiumCDP, ChromiumPlaywright, Config, FileSystem, FirefoxPlaywright, HTTPServer, Hooks, Limiter, Metrics, Monitoring, Router, Token, WebHooks, WebkitPlaywright, availableBrowsers, createLogger, getRouteFiles, makeExternalURL, printLogo, safeParse, } from '@browserless.io/browserless';
+import { Logger as BlessLogger, BrowserManager, ChromeCDP, ChromiumCDP, ChromiumPlaywright, Config, FileSystem, FirefoxPlaywright, HTTPServer, Hooks, Limiter, Metrics, Monitoring, Router, Token, WebHooks, WebkitPlaywright, availableBrowsers, getRouteFiles, makeExternalURL, printLogo, safeParse, } from '@browserless.io/browserless';
 import { EventEmitter } from 'events';
 import { readFile } from 'fs/promises';
 import { userInfo } from 'os';
 const routeSchemas = ['body', 'query'];
 export class Browserless extends EventEmitter {
-    debug = createLogger('index');
+    logger;
     browserManager;
     config;
     fileSystem;
     hooks;
     limiter;
+    Logger;
     metrics;
     monitoring;
     router;
@@ -23,8 +24,10 @@ export class Browserless extends EventEmitter {
     server;
     metricsSaveInterval = 5 * 60 * 1000;
     metricsSaveIntervalID;
-    constructor({ browserManager, config, fileSystem, hooks, limiter, metrics, monitoring, router, token, webhooks, } = {}) {
+    constructor({ browserManager, config, fileSystem, hooks, limiter, Logger: LoggerOverride, metrics, monitoring, router, token, webhooks, } = {}) {
         super();
+        this.Logger = LoggerOverride ?? BlessLogger;
+        this.logger = new this.Logger('index');
         this.config = config || new Config();
         this.metrics = metrics || new Metrics();
         this.token = token || new Token(this.config);
@@ -38,7 +41,8 @@ export class Browserless extends EventEmitter {
             limiter ||
                 new Limiter(this.config, this.metrics, this.monitoring, this.webhooks, this.hooks);
         this.router =
-            router || new Router(this.config, this.browserManager, this.limiter);
+            router ||
+                new Router(this.config, this.browserManager, this.limiter, this.Logger);
     }
     saveMetrics = async () => {
         const metricsPath = this.config.getMetricsJSONPath();
@@ -50,7 +54,7 @@ export class Browserless extends EventEmitter {
             memory,
         };
         this.metrics.reset();
-        this.debug(`Current period usage: ${JSON.stringify({
+        this.logger.info(`Current period usage: ${JSON.stringify({
             date: aggregatedStats.date,
             error: aggregatedStats.error,
             maxConcurrent: aggregatedStats.maxConcurrent,
@@ -64,7 +68,7 @@ export class Browserless extends EventEmitter {
             units: aggregatedStats.units,
         })}`);
         if (metricsPath) {
-            this.debug(`Saving metrics to "${metricsPath}"`);
+            this.logger.info(`Saving metrics to "${metricsPath}"`);
             this.fileSystem.append(metricsPath, JSON.stringify(aggregatedStats), false);
         }
     };
@@ -124,25 +128,26 @@ export class Browserless extends EventEmitter {
             WebkitPlaywright,
         ];
         const [[internalHttpRouteFiles, internalWsRouteFiles], installedBrowsers] = await Promise.all([getRouteFiles(this.config), availableBrowsers]);
+        const hasDebugger = await this.config.hasDebugger();
+        const debuggerURL = hasDebugger &&
+            makeExternalURL(this.config.getExternalAddress(), `/debugger/?token=${this.config.getToken()}`);
         const docsLink = makeExternalURL(this.config.getExternalAddress(), '/docs');
-        this.debug(printLogo(docsLink));
-        this.debug(`Running as user "${userInfo().username}"`);
-        this.debug('Starting import of HTTP Routes');
+        this.logger.info(printLogo(docsLink, debuggerURL));
+        this.logger.info(`Running as user "${userInfo().username}"`);
+        this.logger.info('Starting import of HTTP Routes');
         for (const httpRoute of [
             ...this.httpRouteFiles,
             ...internalHttpRouteFiles,
         ]) {
             if (httpRoute.endsWith('js')) {
-                const { name } = path.parse(httpRoute);
                 const [bodySchema, querySchema] = await Promise.all(routeSchemas.map(async (schemaType) => {
                     const schemaPath = path.parse(httpRoute);
                     schemaPath.base = `${schemaPath.name}.${schemaType}.json`;
                     return await readFile(path.format(schemaPath), 'utf-8').catch(() => '');
                 }));
                 const routeImport = `${this.config.getIsWin() ? 'file:///' : ''}${httpRoute}`;
-                const logger = createLogger(`http:${name}`);
                 const { default: Route, } = await import(routeImport + `?cb=${Date.now()}`);
-                const route = new Route(this.browserManager, this.config, this.fileSystem, logger, this.metrics, this.monitoring, this.staticSDKDir);
+                const route = new Route(this.browserManager, this.config, this.fileSystem, this.metrics, this.monitoring, this.staticSDKDir);
                 if (!this.routeIsDisabled(route)) {
                     route.bodySchema = safeParse(bodySchema);
                     route.querySchema = safeParse(querySchema);
@@ -150,35 +155,31 @@ export class Browserless extends EventEmitter {
                     route.metrics = () => this.metrics;
                     route.monitoring = () => this.monitoring;
                     route.fileSystem = () => this.fileSystem;
-                    route.debug = () => logger;
                     route.staticSDKDir = () => this.staticSDKDir;
                     httpRoutes.push(route);
                 }
             }
         }
-        this.debug('Starting import of WebSocket Routes');
+        this.logger.info('Starting import of WebSocket Routes');
         for (const wsRoute of [
             ...this.webSocketRouteFiles,
             ...internalWsRouteFiles,
         ]) {
             if (wsRoute.endsWith('js')) {
-                const { name } = path.parse(wsRoute);
                 const [, querySchema] = await Promise.all(routeSchemas.map(async (schemaType) => {
                     const schemaPath = path.parse(wsRoute);
                     schemaPath.base = `${schemaPath.name}.${schemaType}.json`;
                     return await readFile(path.format(schemaPath), 'utf-8').catch(() => '');
                 }));
                 const wsImport = `${this.config.getIsWin() ? 'file:///' : ''}${wsRoute}`;
-                const logger = createLogger(`ws:${name}`);
                 const { default: Route, } = await import(wsImport + `?cb=${Date.now()}`);
-                const route = new Route(this.browserManager, this.config, this.fileSystem, logger, this.metrics, this.monitoring, this.staticSDKDir);
+                const route = new Route(this.browserManager, this.config, this.fileSystem, this.metrics, this.monitoring, this.staticSDKDir);
                 if (!this.routeIsDisabled(route)) {
                     route.querySchema = safeParse(querySchema);
                     route.config = () => this.config;
                     route.metrics = () => this.metrics;
                     route.monitoring = () => this.monitoring;
                     route.fileSystem = () => this.fileSystem;
-                    route.debug = () => logger;
                     route.staticSDKDir = () => this.staticSDKDir;
                     wsRoutes.push(route);
                 }
@@ -198,14 +199,14 @@ export class Browserless extends EventEmitter {
             .filter((e, i, a) => a.findIndex((r) => r.name === e.name) !== i)
             .map((r) => r.name);
         if (duplicateNamedRoutes.length) {
-            this.debug(`Found duplicate routing names. Route names must be unique:`, duplicateNamedRoutes);
+            this.logger.warn(`Found duplicate routing names. Route names must be unique:`, duplicateNamedRoutes);
         }
         httpRoutes.forEach((r) => this.router.registerHTTPRoute(r));
         wsRoutes.forEach((r) => this.router.registerWebSocketRoute(r));
-        this.debug(`Imported and validated all route files, starting up server.`);
-        this.server = new HTTPServer(this.config, this.metrics, this.token, this.router, this.hooks);
+        this.logger.info(`Imported and validated all route files, starting up server.`);
+        this.server = new HTTPServer(this.config, this.metrics, this.token, this.router, this.hooks, this.Logger);
         await this.server.start();
-        this.debug(`Starting metrics collection.`);
+        this.logger.info(`Starting metrics collection.`);
         this.metricsSaveIntervalID = setInterval(() => this.saveMetrics(), this.metricsSaveInterval);
     }
 }

package/build/browsers/chrome.cdp.d.ts

@@ -1,6 +1,6 @@
-/// <reference types="debug" />
+import { Logger } from '@browserless.io/browserless';
 import { ChromiumCDP } from './chromium.cdp.js';
 export declare class ChromeCDP extends ChromiumCDP {
     protected executablePath: string;
-    protected debug: import("debug").Debugger;
+    protected logger: Logger;
 }

package/build/browsers/chrome.cdp.js

@@ -1,6 +1,6 @@
-import { chromeExecutablePath, createLogger, } from '@browserless.io/browserless';
+import { Logger, chromeExecutablePath } from '@browserless.io/browserless';
 import { ChromiumCDP } from './chromium.cdp.js';
 export class ChromeCDP extends ChromiumCDP {
     executablePath = chromeExecutablePath();
-    debug = createLogger('browsers:chrome:cdp');
+    logger = new Logger('browsers:chrome:cdp');
 }

package/build/browsers/chrome.playwright.d.ts

@@ -1,6 +1,6 @@
-/// <reference types="debug" />
+import { Logger } from '@browserless.io/browserless';
 import { ChromiumPlaywright } from './chromium.playwright.js';
 export declare class ChromePlaywright extends ChromiumPlaywright {
     protected executablePath: string;
-    protected debug: import("debug").Debugger;
+    protected logger: Logger;
 }

package/build/browsers/chrome.playwright.js

@@ -1,6 +1,6 @@
-import { chromeExecutablePath, createLogger, } from '@browserless.io/browserless';
+import { Logger, chromeExecutablePath } from '@browserless.io/browserless';
 import { ChromiumPlaywright } from './chromium.playwright.js';
 export class ChromePlaywright extends ChromiumPlaywright {
     executablePath = chromeExecutablePath();
-    debug = createLogger('browsers:chrome:playwright');
+    logger = new Logger('browsers:chrome:playwright');
 }

package/build/browsers/chromium.cdp.d.ts

@@ -1,10 +1,9 @@
-/// <reference types="debug" />
 /// <reference types="node" />
 /// <reference types="node" />
 /// <reference types="node" />
 /// <reference types="node" />
 /// <reference types="node" />
-import { CDPLaunchOptions, Config, Request } from '@browserless.io/browserless';
+import { CDPLaunchOptions, Config, Logger, Request } from '@browserless.io/browserless';
 import { Browser, Page, Target } from 'puppeteer-core';
 import { Duplex } from 'stream';
 import { EventEmitter } from 'events';
@@ -17,12 +16,13 @@ export declare class ChromiumCDP extends EventEmitter {
     protected browser: Browser | null;
     protected browserWSEndpoint: string | null;
     protected port?: number;
-    protected debug: import("debug").Debugger;
+    protected logger: Logger;
     protected proxy: httpProxy<import("http").IncomingMessage, import("http").ServerResponse<import("http").IncomingMessage>>;
     protected executablePath: string;
-    constructor({ blockAds, config, userDataDir, }: {
+    constructor({ blockAds, config, userDataDir, logger, }: {
         blockAds: boolean;
         config: Config;
+        logger: Logger;
         userDataDir: ChromiumCDP['userDataDir'];
     });
     protected cleanListeners(): void;