@browserless.io/browserless 2.1.1 → 2.2.0-beta-3

package/build/routes/chromium/http/scrape-post.response.json

@@ -109,7 +109,7 @@
                 {
                   "type": "array",
                   "items": {
-                    "$ref": "#/definitions/Protocol.Network.Cookie"
+                    "$ref": "#/definitions/Cookie"
                   }
                 },
                 {
@@ -175,8 +175,8 @@
     "debug"
   ],
   "definitions": {
-    "Protocol.Network.Cookie": {
-      "description": "Cookie object",
+    "Cookie": {
+      "description": "Represents a cookie object.",
       "type": "object",
       "properties": {
         "name": {
@@ -196,12 +196,12 @@
           "type": "string"
         },
         "expires": {
-          "description": "Cookie expiration date as the number of seconds since the UNIX epoch.",
+          "description": "Cookie expiration date as the number of seconds since the UNIX epoch. Set to `-1` for\nsession cookies",
           "type": "number"
         },
         "size": {
           "description": "Cookie size.",
-          "type": "integer"
+          "type": "number"
         },
         "httpOnly": {
           "description": "True if cookie is http-only.",
@@ -225,27 +225,33 @@
           "type": "string"
         },
         "priority": {
-          "$ref": "#/definitions/Protocol.Network.CookiePriority",
-          "description": "Cookie Priority"
+          "description": "Cookie Priority. Supported only in Chrome.",
+          "enum": [
+            "High",
+            "Low",
+            "Medium"
+          ],
+          "type": "string"
         },
         "sameParty": {
-          "description": "True if cookie is SameParty.",
+          "description": "True if cookie is SameParty. Supported only in Chrome.",
           "type": "boolean"
         },
         "sourceScheme": {
-          "$ref": "#/definitions/Protocol.Network.CookieSourceScheme",
-          "description": "Cookie source scheme type."
-        },
-        "sourcePort": {
-          "description": "Cookie source port. Valid values are {-1, [1, 65535]}, -1 indicates an unspecified port.\nAn unspecified port value allows protocol clients to emulate legacy cookie scope for the port.\nThis is a temporary ability and it will be removed in the future.",
-          "type": "integer"
+          "description": "Cookie source scheme type. Supported only in Chrome.",
+          "enum": [
+            "NonSecure",
+            "Secure",
+            "Unset"
+          ],
+          "type": "string"
         },
         "partitionKey": {
-          "description": "Cookie partition key. The site of the top-level URL the browser was visiting at the start\nof the request to the endpoint that set the cookie.",
+          "description": "Cookie partition key. The site of the top-level URL the browser was visiting at the\nstart of the request to the endpoint that set the cookie. Supported only in Chrome.",
           "type": "string"
         },
         "partitionKeyOpaque": {
-          "description": "True if cookie partition key is opaque.",
+          "description": "True if cookie partition key is opaque. Supported only in Chrome.",
           "type": "boolean"
         }
       },
@@ -256,34 +262,12 @@
         "httpOnly",
         "name",
         "path",
-        "priority",
-        "sameParty",
         "secure",
         "session",
         "size",
-        "sourcePort",
-        "sourceScheme",
         "value"
       ]
     },
-    "Protocol.Network.CookiePriority": {
-      "description": "Represents the cookie's 'Priority' status:\nhttps://tools.ietf.org/html/draft-west-cookie-priority-00",
-      "enum": [
-        "High",
-        "Low",
-        "Medium"
-      ],
-      "type": "string"
-    },
-    "Protocol.Network.CookieSourceScheme": {
-      "description": "Represents the source scheme of the origin that originally set the cookie.\nA value of \"Unset\" allows protocol clients to emulate legacy cookie scope for the scheme.\nThis is a temporary ability and it will be removed in the future.",
-      "enum": [
-        "NonSecure",
-        "Secure",
-        "Unset"
-      ],
-      "type": "string"
-    },
     "InBoundRequest": {
       "type": "object",
       "properties": {

package/build/routes/chromium/http/screenshot-post.body.json

@@ -23,7 +23,7 @@
     "cookies": {
       "type": "array",
       "items": {
-        "$ref": "#/definitions/Protocol.Network.CookieParam"
+        "$ref": "#/definitions/CookieParam"
       }
     },
     "emulateMediaType": {
@@ -238,7 +238,7 @@
         "username"
       ]
     },
-    "Protocol.Network.CookieParam": {
+    "CookieParam": {
       "description": "Cookie parameter object",
       "type": "object",
       "properties": {
@@ -251,7 +251,7 @@
           "type": "string"
         },
         "url": {
-          "description": "The request-URI to associate with the setting of the cookie. This value can affect the\ndefault domain, path, source port, and source scheme values of the created cookie.",
+          "description": "The request-URI to associate with the setting of the cookie. This value can affect\nthe default domain, path, and source scheme values of the created cookie.",
           "type": "string"
         },
         "domain": {
@@ -284,7 +284,7 @@
           "type": "number"
         },
         "priority": {
-          "description": "Cookie Priority.",
+          "description": "Cookie Priority. Supported only in Chrome.",
           "enum": [
             "High",
             "Low",
@@ -293,11 +293,11 @@
           "type": "string"
         },
         "sameParty": {
-          "description": "True if cookie is SameParty.",
+          "description": "True if cookie is SameParty. Supported only in Chrome.",
           "type": "boolean"
         },
         "sourceScheme": {
-          "description": "Cookie source scheme type.",
+          "description": "Cookie source scheme type. Supported only in Chrome.",
           "enum": [
             "NonSecure",
             "Secure",
@@ -305,12 +305,8 @@
           ],
           "type": "string"
         },
-        "sourcePort": {
-          "description": "Cookie source port. Valid values are {-1, [1, 65535]}, -1 indicates an unspecified port.\nAn unspecified port value allows protocol clients to emulate legacy cookie scope for the port.\nThis is a temporary ability and it will be removed in the future.",
-          "type": "number"
-        },
         "partitionKey": {
-          "description": "Cookie partition key. The site of the top-level URL the browser was visiting at the start\nof the request to the endpoint that set the cookie.\nIf not set, the cookie will be set as not partitioned.",
+          "description": "Cookie partition key. The site of the top-level URL the browser was visiting at the\nstart of the request to the endpoint that set the cookie. If not set, the cookie will\nbe set as not partitioned.",
           "type": "string"
         }
       },
@@ -488,14 +484,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@118386": {
+                "__@toStringTag@96609": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@118386",
+                "__@toStringTag@96609",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -530,13 +526,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@118386": {
+        "__@toStringTag@96609": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@118386",
+        "__@toStringTag@96609",
         "byteLength"
       ]
     },
@@ -546,18 +542,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@118487": {
+        "__@species@96710": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@118386": {
+        "__@toStringTag@96609": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@118487",
-        "__@toStringTag@118386",
+        "__@species@96710",
+        "__@toStringTag@96609",
         "byteLength"
       ]
     },

package/build/routes/chromium/http/screenshot-post.d.ts

@@ -33,7 +33,7 @@ export interface BodySchema {
     waitForEvent?: WaitForEventOptions;
     waitForFunction?: WaitForFunctionOptions;
     waitForSelector?: WaitForSelectorOptions;
-    waitForTimeout?: Parameters<Page['waitForTimeout']>[0];
+    waitForTimeout?: number;
 }
 export default class ScreenshotPost extends BrowserHTTPRoute {
     accepts: contentTypes[];

package/build/routes/chromium/http/screenshot-post.js

@@ -1,4 +1,4 @@
-import { APITags, BadRequest, BrowserHTTPRoute, CDPChromium, HTTPRoutes, Methods, bestAttemptCatch, contentTypes, dedent, noop, scrollThroughPage, waitForEvent as waitForEvt, waitForFunction as waitForFn, } from '@browserless.io/browserless';
+import { APITags, BadRequest, BrowserHTTPRoute, CDPChromium, HTTPRoutes, Methods, bestAttemptCatch, contentTypes, dedent, noop, scrollThroughPage, sleep, waitForEvent as waitForEvt, waitForFunction as waitForFn, } from '@browserless.io/browserless';
 import Stream from 'stream';
 export default class ScreenshotPost extends BrowserHTTPRoute {
     accepts = [contentTypes.json];
@@ -81,9 +81,7 @@ export default class ScreenshotPost extends BrowserHTTPRoute {
         }
         const gotoResponse = await gotoCall(content, gotoOptions).catch(bestAttemptCatch(bestAttempt));
         if (waitForTimeout) {
-            await page
-                .waitForTimeout(waitForTimeout)
-                .catch(bestAttemptCatch(bestAttempt));
+            await sleep(waitForTimeout).catch(bestAttemptCatch(bestAttempt));
         }
         if (waitForFunction) {
             await waitForFn(page, waitForFunction).catch(bestAttemptCatch(bestAttempt));

package/build/routes/chromium/tests/content.spec.js

@@ -1,4 +1,4 @@
-import { Browserless, Config, Metrics } from '@browserless.io/browserless';
+import { Browserless, Config, Metrics, sleep } from '@browserless.io/browserless';
 import { expect } from 'chai';
 describe('/content API', function () {
     let browserless;
@@ -32,6 +32,32 @@ describe('/content API', function () {
             expect(res.status).to.equal(200);
         });
     });
+    it('cancels request when they are closed early', async () => {
+        const config = new Config();
+        const metrics = new Metrics();
+        await start({ config, metrics });
+        const body = {
+            url: 'https://cnn.com',
+        };
+        const controller = new AbortController();
+        const signal = controller.signal;
+        const promise = fetch('http://localhost:3000/content', {
+            body: JSON.stringify(body),
+            headers: {
+                'content-type': 'application/json',
+            },
+            method: 'POST',
+            signal,
+        }).catch(async (error) => {
+            await sleep(100);
+            expect(error).to.have.property('name', 'AbortError');
+            expect(metrics.get().error).to.equal(1);
+            expect(metrics.get().successful).to.equal(0);
+        });
+        await sleep(1000);
+        controller.abort();
+        return promise;
+    });
     it('404s GET requests', async () => {
         const config = new Config();
         config.setToken('browserless');

package/build/routes/chromium/tests/websocket.spec.js

@@ -1,4 +1,4 @@
-import { Browserless, Config, Metrics, exists, sleep, } from '@browserless.io/browserless';
+import { Browserless, Config, Metrics, exists, fetchJson, sleep, } from '@browserless.io/browserless';
 import { chromium } from 'playwright-core';
 import { deleteAsync } from 'del';
 import { expect } from 'chai';
@@ -56,6 +56,54 @@ describe('WebSocket API', function () {
         });
         await Promise.all([browser.disconnect(), browserTwo.disconnect()]);
     });
+    it('does not close browsers when multiple clients are connected', async () => {
+        const config = new Config();
+        config.setToken('browserless');
+        const metrics = new Metrics();
+        await start({ config, metrics });
+        // Single session
+        const browser = await puppeteer.connect({
+            browserWSEndpoint: `ws://localhost:3000?token=browserless`,
+        });
+        const [session] = (await fetchJson('http://localhost:3000/sessions?token=browserless'));
+        expect(session.numbConnected).to.equal(1);
+        // Two sessions
+        const browserTwo = await puppeteer.connect({
+            browserWSEndpoint: `ws://localhost:3000/devtools/browser/${session.browserId}?token=browserless`,
+        });
+        const [twoSessions] = (await fetchJson('http://localhost:3000/sessions?token=browserless'));
+        expect(twoSessions.numbConnected).to.equal(2);
+        // Back to a single session
+        await browser.disconnect();
+        await sleep(50);
+        const [oneSession] = (await fetchJson('http://localhost:3000/sessions?token=browserless'));
+        expect(oneSession.numbConnected).to.equal(1);
+        // No sessions connected
+        await browserTwo.disconnect();
+        await sleep(50);
+        const sessionsFinal = (await fetchJson('http://localhost:3000/sessions?token=browserless'));
+        expect(sessionsFinal).to.have.length(0);
+    });
+    it('disconnects all clients when the timeout is reached', async () => {
+        const config = new Config();
+        config.setToken('browserless');
+        config.setTimeout(1000);
+        config.setConcurrent(2);
+        const metrics = new Metrics();
+        await start({ config, metrics });
+        const browser = await puppeteer.connect({
+            browserWSEndpoint: `ws://localhost:3000?token=browserless`,
+        });
+        const [session] = (await fetchJson('http://localhost:3000/sessions?token=browserless'));
+        const browserTwo = await puppeteer.connect({
+            browserWSEndpoint: `ws://localhost:3000/devtools/browser/${session.browserId}?token=browserless`,
+        });
+        await sleep(3000);
+        expect(metrics.get().successful).to.equal(0);
+        expect(metrics.get().timedout).to.equal(2);
+        expect(browser.connected).to.be.false;
+        expect(browserTwo.connected).to.be.false;
+    });
     it('rejects websocket requests', async () => {
         const config = new Config();
         config.setToken('browserless');
@@ -189,12 +237,13 @@ describe('WebSocket API', function () {
         const metrics = new Metrics();
         await start({ config, metrics });
         const browser = await puppeteer.connect({
-            browserWSEndpoint: `ws://localhost:3000?timeout=500&token=browserless`,
+            browserWSEndpoint: `ws://localhost:3000?timeout=1000&token=browserless`,
         });
-        await sleep(750);
-        browser.disconnect();
+        expect(browser.connected).to.be.true;
+        await sleep(1200);
         expect(metrics.get().timedout).to.equal(1);
         expect(metrics.get().successful).to.equal(0);
+        expect(browser.connected).to.be.false;
     });
     it('allows the file-chooser', async () => new Promise(async (done) => {
         const config = new Config();

package/build/routes/chromium/ws/browser.js

@@ -2,7 +2,7 @@ import { APITags, BrowserWebsocketRoute, CDPChromium, WebsocketRoutes, } from '@
 export default class CDPExistingBrowser extends BrowserWebsocketRoute {
     auth = true;
     browser = CDPChromium;
-    concurrency = false;
+    concurrency = true;
     description = `Connect to an already-running Chromium with a library like puppeteer, or others, that work over chrome-devtools-protocol.`;
     path = WebsocketRoutes.browser;
     tags = [APITags.browserWS];

package/build/routes/management/http/sessions-get.response.json

@@ -10,6 +10,9 @@
         "browser": {
           "type": "string"
         },
+        "browserId": {
+          "type": "string"
+        },
         "id": {
           "type": [
             "null",
@@ -57,6 +60,7 @@
       "additionalProperties": false,
       "required": [
         "browser",
+        "browserId",
         "id",
         "initialConnectURL",
         "killURL",

package/build/types.d.ts

@@ -274,6 +274,7 @@ export interface BrowserlessSession {
 }
 export interface BrowserlessSessionJSON {
     browser: string;
+    browserId: string;
     id: string | null;
     initialConnectURL: string;
     killURL: string | null;

package/build/utils.d.ts

@@ -32,6 +32,7 @@ export declare const readRequestBody: (req: Request) => Promise<string>;
 export declare const safeParse: (maybeJson: string) => unknown | null;
 export declare const removeNullStringify: (json: unknown, allowNull?: boolean) => string;
 export declare const jsonOrString: (maybeJson: string) => unknown | string;
+export declare const generateDataDir: (sessionId: string | undefined, config: Config) => Promise<string>;
 export declare const readBody: (req: Request) => Promise<ReturnType<typeof safeParse>>;
 export declare const getRouteFiles: (config: Config) => Promise<string[][]>;
 export declare const make404: (...messages: string[]) => string;

package/build/utils.js

@@ -176,6 +176,19 @@ export const removeNullStringify = (json, allowNull = true) => {
     }, '  ');
 };
 export const jsonOrString = (maybeJson) => safeParse(maybeJson) ?? maybeJson;
+export const generateDataDir = async (sessionId = id(), config) => {
+    const baseDirectory = await config.getDataDir();
+    const dataDirPath = path.join(baseDirectory, `browserless-data-dir-${sessionId}`);
+    if (await exists(dataDirPath)) {
+        debug(`Data directory already exists, not creating "${dataDirPath}"`);
+        return dataDirPath;
+    }
+    debug(`Generating user-data-dir at ${dataDirPath}`);
+    await fs.mkdir(dataDirPath, { recursive: true }).catch((err) => {
+        throw new ServerError(`Error creating data-directory "${dataDirPath}": ${err}`);
+    });
+    return dataDirPath;
+};
 export const readBody = async (req) => {
     if (typeof req.body === 'string' &&
         (isBase64.test(req.body) || req.body.startsWith('{'))) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@browserless.io/browserless",
-  "version": "2.1.1",
+  "version": "2.2.0-beta-3",
   "license": "SSPL",
   "description": "The browserless platform",
   "author": "browserless.io",
@@ -56,7 +56,7 @@
     "lighthouse": "^11.1.0",
     "micromatch": "^4.0.4",
     "playwright-core": "^1.41.2",
-    "puppeteer-core": "^21.10.0",
+    "puppeteer-core": "^22.0.0",
     "puppeteer-extra": "^3.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2",
     "queue": "^7.0.0",
@@ -71,7 +71,7 @@
     "@types/mocha": "^10.0.6",
     "@types/node": "^20.11.16",
     "@types/sinon": "^17.0.3",
-    "@typescript-eslint/eslint-plugin": "^6.20.0",
+    "@typescript-eslint/eslint-plugin": "^6.21.0",
     "@typescript-eslint/parser": "^6.21.0",
     "assert": "^2.0.0",
     "chai": "^5.0.3",

package/src/browsers/index.ts

@@ -11,27 +11,33 @@ import {
   Config,
   HTTPManagementRoutes,
   NotFound,
+  PlaywrightChromium,
+  PlaywrightFirefox,
+  PlaywrightWebkit,
   Request,
-  ServerError,
   browserHook,
   convertIfBase64,
   createLogger,
   exists,
-  id,
+  generateDataDir,
   makeExternalURL,
   noop,
   pageHook,
   parseBooleanParam,
 } from '@browserless.io/browserless';
-import path, { join } from 'path';
 import { deleteAsync } from 'del';
-import { mkdir } from 'fs/promises';
+import path from 'path';
 
 export class BrowserManager {
   protected browsers: Map<BrowserInstance, BrowserlessSession> = new Map();
   protected launching: Map<string, Promise<unknown>> = new Map();
   protected timers: Map<string, number> = new Map();
   protected debug = createLogger('browser-manager');
+  protected playwrightBrowserNames = [
+    PlaywrightChromium.name,
+    PlaywrightFirefox.name,
+    PlaywrightWebkit.name,
+  ];
 
   constructor(protected config: Config) {}
 
@@ -46,42 +52,6 @@ export class BrowserManager {
     }
   };
 
-  /**
-   * Generates a directory for the user-data-dir contents to be saved in. Uses
-   * the provided sessionId, or creates one when omitted,
-   * and appends it to the name of the directory. If the
-   * directory already exists then no action is taken, verified by run `stat`
-   *
-   * @param sessionId The ID of the session
-   * @returns Promise<string> of the fully-qualified path of the directory
-   */
-  protected generateDataDir = async (
-    sessionId: string = id(),
-  ): Promise<string> => {
-    const baseDirectory = await this.config.getDataDir();
-    const dataDirPath = join(
-      baseDirectory,
-      `browserless-data-dir-${sessionId}`,
-    );
-
-    if (await exists(dataDirPath)) {
-      this.debug(
-        `Data directory already exists, not creating "${dataDirPath}"`,
-      );
-      return dataDirPath;
-    }
-
-    this.debug(`Generating user-data-dir at ${dataDirPath}`);
-
-    await mkdir(dataDirPath, { recursive: true }).catch((err) => {
-      throw new ServerError(
-        `Error creating data-directory "${dataDirPath}": ${err}`,
-      );
-    });
-
-    return dataDirPath;
-  };
-
   public getProtocolJSON = async (): Promise<object> => {
     this.debug(`Launching Chrome to generate /json/protocol results`);
     const browser = new CDPChromium({
@@ -155,7 +125,7 @@ export class BrowserManager {
       {
         ...session,
         browser: browser.constructor.name,
-        browserId: browser.wsEndpoint()?.split('/').pop(),
+        browserId: browser.wsEndpoint()?.split('/').pop() as string,
         initialConnectURL: new URL(session.initialConnectURL, serverAddress)
           .href,
         killURL: session.id
@@ -200,6 +170,11 @@ export class BrowserManager {
     const cleanupACtions: Array<() => Promise<void>> = [];
     this.debug(`${session.numbConnected} Client(s) are currently connected`);
 
+    // Don't close if there's clients still connected
+    if (session.numbConnected > 0) {
+      return;
+    }
+
     this.debug(`Closing browser session`);
     cleanupACtions.push(() => browser.close());
 
@@ -264,13 +239,15 @@ export class BrowserManager {
     if (req.parsed.pathname.includes('/devtools/browser')) {
       const sessions = Array.from(this.browsers);
       const id = req.parsed.pathname.split('/').pop() as string;
-      const browser = sessions.find(([b]) =>
+      const found = sessions.find(([b]) =>
         b.wsEndpoint()?.includes(req.parsed.pathname),
       );
 
-      if (browser) {
+      if (found) {
+        const [browser, session] = found;
+        ++session.numbConnected;
         this.debug(`Located browser with ID ${id}`);
-        return browser[0];
+        return browser;
       }
 
       throw new NotFound(
@@ -305,7 +282,9 @@ export class BrowserManager {
     // unless it's playwright which takes care of its own data-dirs
     const userDataDir =
       manualUserDataDir ||
-      (Browser.name === CDPChromium.name ? await this.generateDataDir() : null);
+      (!this.playwrightBrowserNames.includes(Browser.name)
+        ? await generateDataDir(undefined, this.config)
+        : null);
 
     const proxyServerArg = launchOptions.args?.find((arg) =>
       arg.includes('--proxy-server='),

package/src/router.ts

@@ -80,17 +80,23 @@ export class Router {
         }
 
         if (!browser) {
-          return writeResponse(res, 500, `Error loading the browser.`);
-        }
-
-        if (!isConnected(res)) {
-          this.log(`HTTP Request has closed prior to running`);
-          return Promise.resolve();
+          return writeResponse(res, 500, `Error loading the browser`);
         }
 
         try {
           this.verbose(`Running found HTTP handler.`);
-          return await handler(req, res, browser);
+          return await Promise.race([
+            handler(req, res, browser),
+            new Promise((resolve, reject) => {
+              res.once('close', () => {
+                if (!res.writableEnded) {
+                  reject(new Error(`Request closed prior to writing results`));
+                }
+                this.verbose(`Response has been written, resolving`);
+                resolve(null);
+              });
+            }),
+          ]);
         } finally {
           this.verbose(`HTTP Request handler has finished.`);
           this.browserManager.complete(browser);

package/src/routes/chromium/http/content-post.ts

@@ -21,6 +21,7 @@ import {
   rejectResourceTypes,
   requestInterceptors,
   setJavaScriptEnabled,
+  sleep,
   waitForEvent as waitForEvt,
   waitForFunction as waitForFn,
   writeResponse,
@@ -48,7 +49,7 @@ export interface BodySchema {
   waitForEvent?: WaitForEventOptions;
   waitForFunction?: WaitForFunctionOptions;
   waitForSelector?: WaitForSelectorOptions;
-  waitForTimeout?: Parameters<Page['waitForTimeout']>[0];
+  waitForTimeout?: number;
 }
 
 /**
@@ -190,9 +191,7 @@ export default class ContentPostRoute extends BrowserHTTPRoute {
     );
 
     if (waitForTimeout) {
-      await page
-        .waitForTimeout(waitForTimeout)
-        .catch(bestAttemptCatch(bestAttempt));
+      await sleep(waitForTimeout).catch(bestAttemptCatch(bestAttempt));
     }
 
     if (waitForFunction) {