@browserless.io/browserless 2.12.0-beta-2 → 2.12.0-beta-4

package/build/browserless.d.ts

@@ -36,6 +36,7 @@ export declare class Browserless extends EventEmitter {
         token?: Browserless['token'];
         webhooks?: Browserless['webhooks'];
     });
+    protected loadPwVersions: () => Promise<void>;
     protected saveMetrics: () => Promise<void>;
     setMetricsSaveInterval: (interval: number) => void;
     protected routeIsDisabled(route: routeInstances): boolean;

package/build/browserless.js

@@ -1,3 +1,4 @@
+import * as fs from 'fs/promises';
 import * as path from 'path';
 import { Logger as BlessLogger, BrowserManager, ChromeCDP, ChromiumCDP, ChromiumPlaywright, Config, FileSystem, FirefoxPlaywright, HTTPServer, Hooks, Limiter, Metrics, Monitoring, Router, Token, WebHooks, WebkitPlaywright, availableBrowsers, getRouteFiles, makeExternalURL, printLogo, safeParse, } from '@browserless.io/browserless';
 import { EventEmitter } from 'events';
@@ -44,6 +45,10 @@ export class Browserless extends EventEmitter {
             router ||
                 new Router(this.config, this.browserManager, this.limiter, this.Logger);
     }
+    loadPwVersions = async () => {
+        const { playwrightVersions } = JSON.parse((await fs.readFile('package.json')).toString());
+        this.config.setPwVersions(playwrightVersions);
+    };
     saveMetrics = async () => {
         const metricsPath = this.config.getMetricsJSONPath();
         const { cpu, memory } = await this.monitoring.getMachineStats();
@@ -207,6 +212,7 @@ export class Browserless extends EventEmitter {
         wsRoutes.forEach((r) => this.router.registerWebSocketRoute(r));
         this.logger.info(`Imported and validated all route files, starting up server.`);
         this.server = new HTTPServer(this.config, this.metrics, this.token, this.router, this.hooks, this.Logger);
+        await this.loadPwVersions();
         await this.server.start();
         this.logger.info(`Starting metrics collection.`);
         this.metricsSaveIntervalID = setInterval(() => this.saveMetrics(), this.metricsSaveInterval);

package/build/browsers/chromium.cdp.d.ts

@@ -26,7 +26,7 @@ export declare class ChromiumCDP extends EventEmitter {
         userDataDir: ChromiumCDP['userDataDir'];
     });
     protected cleanListeners(): void;
-    keepAlive(): boolean;
+    keepUntil(): number;
     getPageId: (page: Page) => string;
     protected onTargetCreated: (target: Target) => Promise<void>;
     isRunning: () => boolean;

package/build/browsers/chromium.cdp.js

@@ -32,8 +32,8 @@ export class ChromiumCDP extends EventEmitter {
         this.browser?.removeAllListeners();
         this.removeAllListeners();
     }
-    keepAlive() {
-        return false;
+    keepUntil() {
+        return 0;
     }
     getPageId = (page) => {
         // @ts-ignore

package/build/browsers/chromium.playwright.d.ts

@@ -22,14 +22,14 @@ export declare class ChromiumPlaywright extends EventEmitter {
         userDataDir: ChromiumPlaywright['userDataDir'];
     });
     protected cleanListeners(): void;
-    keepAlive(): boolean;
+    keepUntil(): number;
     isRunning: () => boolean;
     close: () => Promise<void>;
     pages: () => Promise<[]>;
     getPageId: () => string;
     makeLiveURL: () => void;
     newPage: () => Promise<Page>;
-    launch: (options?: BrowserServerOptions) => Promise<playwright.BrowserServer>;
+    launch: (options?: BrowserServerOptions, version?: string) => Promise<playwright.BrowserServer>;
     wsEndpoint: () => string | null;
     publicWSEndpoint: (token: string | null) => string | null;
     proxyPageWebSocket: () => Promise<void>;

package/build/browsers/chromium.playwright.js

@@ -22,8 +22,8 @@ export class ChromiumPlaywright extends EventEmitter {
     cleanListeners() {
         this.removeAllListeners();
     }
-    keepAlive() {
-        return false;
+    keepUntil() {
+        return 0;
     }
     isRunning = () => this.running;
     close = async () => {
@@ -51,9 +51,9 @@ export class ChromiumPlaywright extends EventEmitter {
         const browser = await playwright.chromium.connect(this.browserWSEndpoint);
         return await browser.newPage();
     };
-    launch = async (options = {}) => {
+    launch = async (options = {}, version) => {
         this.logger.info(`Launching ${this.constructor.name} Handler`);
-        this.browser = await playwright.chromium.launchServer({
+        const opts = {
             ...options,
             args: [
                 `--no-sandbox`,
@@ -61,7 +61,9 @@ export class ChromiumPlaywright extends EventEmitter {
                 this.userDataDir ? `--user-data-dir=${this.userDataDir}` : '',
             ],
             executablePath: this.executablePath,
-        });
+        };
+        const versionedPw = await this.config.loadPwVersion(version);
+        this.browser = await versionedPw.chromium.launchServer(opts);
         const browserWSEndpoint = this.browser.wsEndpoint();
         this.logger.info(`${this.constructor.name} is running on ${browserWSEndpoint}`);
         this.running = true;

package/build/browsers/firefox.playwright.d.ts

@@ -21,14 +21,14 @@ export declare class FirefoxPlaywright extends EventEmitter {
         userDataDir: FirefoxPlaywright['userDataDir'];
     });
     protected cleanListeners(): void;
-    keepAlive(): boolean;
+    keepUntil(): number;
     isRunning: () => boolean;
     close: () => Promise<void>;
     pages: () => Promise<[]>;
     getPageId: () => string;
     makeLiveURL: () => void;
     newPage: () => Promise<Page>;
-    launch: (options?: BrowserServerOptions) => Promise<playwright.BrowserServer>;
+    launch: (options?: BrowserServerOptions, version?: string) => Promise<playwright.BrowserServer>;
     wsEndpoint: () => string | null;
     publicWSEndpoint: (token: string | null) => string | null;
     proxyPageWebSocket: () => Promise<void>;

package/build/browsers/firefox.playwright.js

@@ -21,8 +21,8 @@ export class FirefoxPlaywright extends EventEmitter {
     cleanListeners() {
         this.removeAllListeners();
     }
-    keepAlive() {
-        return false;
+    keepUntil() {
+        return 0;
     }
     isRunning = () => this.running;
     close = async () => {
@@ -46,16 +46,18 @@ export class FirefoxPlaywright extends EventEmitter {
     newPage = async () => {
         throw new ServerError(`Can't create new page with ${this.constructor.name}`);
     };
-    launch = async (options = {}) => {
+    launch = async (options = {}, version) => {
         this.logger.info(`Launching ${this.constructor.name} Handler`);
-        this.browser = await playwright.firefox.launchServer({
+        const opts = {
             ...options,
             args: [
                 ...(options.args || []),
                 this.userDataDir ? `-profile=${this.userDataDir}` : '',
             ],
             executablePath: playwright.firefox.executablePath(),
-        });
+        };
+        const versionedPw = await this.config.loadPwVersion(version);
+        this.browser = await versionedPw.firefox.launchServer(opts);
         const browserWSEndpoint = this.browser.wsEndpoint();
         this.logger.info(`${this.constructor.name} is running on ${browserWSEndpoint}`);
         this.browserWSEndpoint = browserWSEndpoint;

package/build/browsers/index.d.ts

@@ -1,10 +1,11 @@
+/// <reference types="node" />
 import { BrowserHTTPRoute, BrowserInstance, BrowserServerOptions, BrowserWebsocketRoute, BrowserlessSession, BrowserlessSessionJSON, CDPJSONPayload, CDPLaunchOptions, ChromiumCDP, Config, Hooks, Logger, Request } from '@browserless.io/browserless';
 import { Page } from 'puppeteer-core';
 export declare class BrowserManager {
     protected config: Config;
     protected hooks: Hooks;
     protected browsers: Map<BrowserInstance, BrowserlessSession>;
-    protected timers: Map<string, number>;
+    protected timers: Map<string, NodeJS.Timeout>;
     protected log: Logger;
     protected chromeBrowsers: (typeof ChromiumCDP)[];
     protected playwrightBrowserNames: string[];
@@ -38,7 +39,7 @@ export declare class BrowserManager {
         running: boolean;
         timeAliveMs: number;
         type: string;
-        id: string | null;
+        id: string;
         isTempDataDir: boolean;
         launchOptions: CDPLaunchOptions | BrowserServerOptions;
         numbConnected: number;

package/build/browsers/index.js

@@ -1,4 +1,4 @@
-import { BLESS_PAGE_IDENTIFIER, BadRequest, ChromeCDP, ChromePlaywright, ChromiumCDP, ChromiumPlaywright, FirefoxPlaywright, HTTPManagementRoutes, Logger, NotFound, ServerError, WebkitPlaywright, availableBrowsers, convertIfBase64, exists, generateDataDir, makeExternalURL, noop, parseBooleanParam, } from '@browserless.io/browserless';
+import { BLESS_PAGE_IDENTIFIER, BadRequest, ChromeCDP, ChromePlaywright, ChromiumCDP, ChromiumPlaywright, FirefoxPlaywright, HTTPManagementRoutes, Logger, NotFound, ServerError, WebkitPlaywright, availableBrowsers, convertIfBase64, exists, generateDataDir, makeExternalURL, noop, parseBooleanParam, pwVersionRegex, } from '@browserless.io/browserless';
 import { deleteAsync } from 'del';
 import path from 'path';
 export class BrowserManager {
@@ -145,7 +145,7 @@ export class BrowserManager {
             {
                 ...session,
                 browser: browser.constructor.name,
-                browserId: browser.wsEndpoint()?.split('/').pop(),
+                browserId: session.id,
                 initialConnectURL: new URL(session.initialConnectURL, serverAddress)
                     .href,
                 killURL: session.id
@@ -178,20 +178,39 @@ export class BrowserManager {
         return sessions;
     };
     close = async (browser, session) => {
+        const now = Date.now();
+        const keepUntil = browser.keepUntil();
+        const connected = session.numbConnected;
+        const hasKeepUntil = keepUntil > now;
+        const keepOpen = connected > 0 || hasKeepUntil;
         const cleanupACtions = [];
-        this.log.info(`${session.numbConnected} Client(s) are currently connected`);
-        // Don't close if there's clients still connected
-        if (session.numbConnected > 0 || browser.keepAlive()) {
-            return;
+        const priorTimer = this.timers.get(session.id);
+        if (priorTimer) {
+            this.log.info(`Deleting prior keep-until timer for "${session.id}"`);
+            global.clearTimeout(priorTimer);
         }
-        this.log.info(`Closing browser session`);
-        cleanupACtions.push(() => browser.close());
-        if (session.isTempDataDir) {
-            this.log.info(`Deleting "${session.userDataDir}" user-data-dir and session from memory`);
-            this.browsers.delete(browser);
-            cleanupACtions.push(() => this.removeUserDataDir(session.userDataDir));
+        this.log.info(`${session.numbConnected} Client(s) are currently connected, Keep-until: ${keepUntil}`);
+        if (hasKeepUntil) {
+            const timeout = keepUntil - now;
+            this.log.trace(`Setting timer ${timeout.toLocaleString()} for "${session.id}"`);
+            this.timers.set(session.id, global.setTimeout(() => {
+                const session = this.browsers.get(browser);
+                if (session) {
+                    this.log.trace(`Timer hit for "${session.id}"`),
+                        this.close(browser, session);
+                }
+            }, timeout));
+        }
+        if (!keepOpen) {
+            this.log.info(`Closing browser session`);
+            cleanupACtions.push(() => browser.close());
+            if (session.isTempDataDir) {
+                this.log.info(`Deleting "${session.userDataDir}" user-data-dir and session from memory`);
+                this.browsers.delete(browser);
+                cleanupACtions.push(() => this.removeUserDataDir(session.userDataDir));
+            }
+            await Promise.all(cleanupACtions.map((a) => a()));
         }
-        await Promise.all(cleanupACtions.map((a) => a()));
     };
     getAllSessions = async () => {
         const sessions = Array.from(this.browsers);
@@ -289,6 +308,9 @@ export class BrowserManager {
                 ? await generateDataDir(undefined, this.config)
                 : null);
         const proxyServerArg = launchOptions.args?.find((arg) => arg.includes('--proxy-server='));
+        /**
+         * If it is a playwright request
+         */
         if (launchOptions.args &&
             proxyServerArg &&
             req.parsed.pathname.startsWith('/playwright')) {
@@ -304,8 +326,12 @@ export class BrowserManager {
             logger,
             userDataDir,
         });
+        const match = (req.headers['user-agent'] || '').match(pwVersionRegex);
+        const pwVersion = match ? match[1] : 'default';
+        await browser.launch(launchOptions, pwVersion);
+        await this.hooks.browser({ browser, meta: req.parsed });
         const session = {
-            id: null,
+            id: browser.wsEndpoint()?.split('/').pop(),
             initialConnectURL: path.join(req.parsed.pathname, req.parsed.search) || '',
             isTempDataDir: !manualUserDataDir,
             launchOptions,
@@ -317,8 +343,6 @@ export class BrowserManager {
             userDataDir,
         };
         this.browsers.set(browser, session);
-        await browser.launch(launchOptions);
-        await this.hooks.browser({ browser, meta: req.parsed });
         browser.on('newPage', async (page) => {
             await this.onNewPage(req, page);
             (router.onNewPage || noop)(req.parsed || '', page);

package/build/browsers/webkit.playwright.d.ts

@@ -21,14 +21,14 @@ export declare class WebkitPlaywright extends EventEmitter {
         userDataDir: WebkitPlaywright['userDataDir'];
     });
     protected cleanListeners(): void;
-    keepAlive(): boolean;
+    keepUntil(): number;
     isRunning: () => boolean;
     close: () => Promise<void>;
     pages: () => Promise<[]>;
     getPageId: () => string;
     makeLiveURL: () => void;
     newPage: () => Promise<Page>;
-    launch: (options?: BrowserServerOptions) => Promise<playwright.BrowserServer>;
+    launch: (options?: BrowserServerOptions, version?: string) => Promise<playwright.BrowserServer>;
     wsEndpoint: () => string | null;
     publicWSEndpoint: (token: string | null) => string | null;
     proxyPageWebSocket: () => Promise<void>;

package/build/browsers/webkit.playwright.js

@@ -21,8 +21,8 @@ export class WebkitPlaywright extends EventEmitter {
     cleanListeners() {
         this.removeAllListeners();
     }
-    keepAlive() {
-        return false;
+    keepUntil() {
+        return 0;
     }
     isRunning = () => this.running;
     close = async () => {
@@ -46,16 +46,18 @@ export class WebkitPlaywright extends EventEmitter {
     newPage = async () => {
         throw new ServerError(`Can't create new page with ${this.constructor.name}`);
     };
-    launch = async (options = {}) => {
+    launch = async (options = {}, version) => {
         this.logger.info(`Launching ${this.constructor.name} Handler`);
-        this.browser = await playwright.webkit.launchServer({
+        const opts = {
             ...options,
             args: [
                 ...(options.args || []),
                 this.userDataDir ? `-profile=${this.userDataDir}` : '',
             ],
             executablePath: playwright.webkit.executablePath(),
-        });
+        };
+        const versionedPw = await this.config.loadPwVersion(version);
+        this.browser = await versionedPw.webkit.launchServer(opts);
         const browserWSEndpoint = this.browser.wsEndpoint();
         this.logger.info(`${this.constructor.name} is running on ${browserWSEndpoint}`);
         this.browserWSEndpoint = browserWSEndpoint;

package/build/config.d.ts

@@ -1,6 +1,7 @@
 /// <reference types="node" />
 /// <reference types="node" />
 import { EventEmitter } from 'events';
+import playwright from 'playwright-core';
 export declare class Config extends EventEmitter {
     protected readonly debug: string;
     protected readonly host: string;
@@ -34,12 +35,18 @@ export declare class Config extends EventEmitter {
     protected rejectAlertURL: string | null;
     protected timeoutAlertURL: string | null;
     protected errorAlertURL: string | null;
+    protected pwVersions: {
+        [key: string]: string;
+    };
     getRoutes: () => string;
     getHost: () => string;
     getPort: () => number;
     getIsWin: () => boolean;
     getToken: () => string | null;
     getDebug: () => string;
+    getPwVersions: () => {
+        [key: string]: string;
+    };
     /**
      * The maximum number of concurrent sessions allowed. Set
      * to "-1" or "Infinity" for no limit.
@@ -84,6 +91,10 @@ export declare class Config extends EventEmitter {
      */
     getAESKey: () => Buffer;
     getMetricsJSONPath: () => string;
+    setPwVersions: (versions: {
+        [key: string]: string;
+    }) => void;
+    loadPwVersion: (version: string) => Promise<typeof playwright>;
     setDataDir: (newDataDir: string) => Promise<string>;
     setRoutes: (newRoutePath: string) => string;
     setConcurrent: (newConcurrent: number) => number;

package/build/config.js

@@ -4,6 +4,7 @@ import debug from 'debug';
 import { fileURLToPath } from 'url';
 import { mkdir } from 'fs/promises';
 import path from 'path';
+import playwright from 'playwright-core';
 import { tmpdir } from 'os';
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 /**
@@ -140,12 +141,14 @@ export class Config extends EventEmitter {
     rejectAlertURL = process.env.REJECT_ALERT_URL ?? null;
     timeoutAlertURL = process.env.TIMEOUT_ALERT_URL ?? null;
     errorAlertURL = process.env.ERROR_ALERT_URL ?? null;
+    pwVersions = {};
     getRoutes = () => this.routes;
     getHost = () => this.host;
     getPort = () => this.port;
     getIsWin = () => this.isWin;
     getToken = () => this.token;
     getDebug = () => this.debug;
+    getPwVersions = () => this.pwVersions;
     /**
      * The maximum number of concurrent sessions allowed. Set
      * to "-1" or "Infinity" for no limit.
@@ -215,6 +218,19 @@ export class Config extends EventEmitter {
         return Buffer.from(aesToken.repeat(keyLength).substring(0, keyLength));
     };
     getMetricsJSONPath = () => this.metricsJSONPath;
+    setPwVersions = (versions) => {
+        this.pwVersions = versions;
+    };
+    loadPwVersion = async (version) => {
+        const versions = this.getPwVersions();
+        try {
+            return await import(versions[version] || versions['default']);
+        }
+        catch (err) {
+            debug.log('Error importing Playwright. Using default version', err);
+            return playwright;
+        }
+    };
     setDataDir = async (newDataDir) => {
         if (!(await exists(newDataDir))) {
             throw new Error(`New data-directory "${newDataDir}" doesn't exist, did you forget to mount or create it?`);

package/build/routes/chrome/http/content.post.body.json

@@ -394,14 +394,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@10952": {
+                "__@toStringTag@11009": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@10952",
+                "__@toStringTag@11009",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -436,13 +436,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@10952": {
+        "__@toStringTag@11009": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@10952",
+        "__@toStringTag@11009",
         "byteLength"
       ]
     },
@@ -452,18 +452,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@11053": {
+        "__@species@11110": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@10952": {
+        "__@toStringTag@11009": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@11053",
-        "__@toStringTag@10952",
+        "__@species@11110",
+        "__@toStringTag@11009",
         "byteLength"
       ]
     },

package/build/routes/chrome/http/pdf.post.body.json

@@ -535,14 +535,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@87099": {
+                "__@toStringTag@87555": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@87099",
+                "__@toStringTag@87555",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -577,13 +577,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@87099": {
+        "__@toStringTag@87555": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@87099",
+        "__@toStringTag@87555",
         "byteLength"
       ]
     },
@@ -593,18 +593,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@87200": {
+        "__@species@87656": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@87099": {
+        "__@toStringTag@87555": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@87200",
-        "__@toStringTag@87099",
+        "__@species@87656",
+        "__@toStringTag@87555",
         "byteLength"
       ]
     },

package/build/routes/chrome/http/scrape.post.body.json

@@ -441,14 +441,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@109051": {
+                "__@toStringTag@98721": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@109051",
+                "__@toStringTag@98721",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -483,13 +483,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@109051": {
+        "__@toStringTag@98721": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@109051",
+        "__@toStringTag@98721",
         "byteLength"
       ]
     },
@@ -499,18 +499,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@109152": {
+        "__@species@98822": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@109051": {
+        "__@toStringTag@98721": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@109152",
-        "__@toStringTag@109051",
+        "__@species@98822",
+        "__@toStringTag@98721",
         "byteLength"
       ]
     },

package/build/routes/chrome/http/screenshot.post.body.json

@@ -484,14 +484,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@131078": {
+                "__@toStringTag@109964": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@131078",
+                "__@toStringTag@109964",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -526,13 +526,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@131078": {
+        "__@toStringTag@109964": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@131078",
+        "__@toStringTag@109964",
         "byteLength"
       ]
     },
@@ -542,18 +542,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@131179": {
+        "__@species@110065": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@131078": {
+        "__@toStringTag@109964": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@131179",
-        "__@toStringTag@131078",
+        "__@species@110065",
+        "__@toStringTag@109964",
         "byteLength"
       ]
     },