@browserless.io/browserless 2.2.0-beta-2 → 2.2.0-beta-3

package/build/routes/chromium/http/screenshot-post.body.json

@@ -23,7 +23,7 @@
     "cookies": {
       "type": "array",
       "items": {
-        "$ref": "#/definitions/Protocol.Network.CookieParam"
+        "$ref": "#/definitions/CookieParam"
       }
     },
     "emulateMediaType": {
@@ -238,7 +238,7 @@
         "username"
       ]
     },
-    "Protocol.Network.CookieParam": {
+    "CookieParam": {
       "description": "Cookie parameter object",
       "type": "object",
       "properties": {
@@ -251,7 +251,7 @@
           "type": "string"
         },
         "url": {
-          "description": "The request-URI to associate with the setting of the cookie. This value can affect the\ndefault domain, path, source port, and source scheme values of the created cookie.",
+          "description": "The request-URI to associate with the setting of the cookie. This value can affect\nthe default domain, path, and source scheme values of the created cookie.",
           "type": "string"
         },
         "domain": {
@@ -284,7 +284,7 @@
           "type": "number"
         },
         "priority": {
-          "description": "Cookie Priority.",
+          "description": "Cookie Priority. Supported only in Chrome.",
           "enum": [
             "High",
             "Low",
@@ -293,11 +293,11 @@
           "type": "string"
         },
         "sameParty": {
-          "description": "True if cookie is SameParty.",
+          "description": "True if cookie is SameParty. Supported only in Chrome.",
           "type": "boolean"
         },
         "sourceScheme": {
-          "description": "Cookie source scheme type.",
+          "description": "Cookie source scheme type. Supported only in Chrome.",
           "enum": [
             "NonSecure",
             "Secure",
@@ -305,12 +305,8 @@
           ],
           "type": "string"
         },
-        "sourcePort": {
-          "description": "Cookie source port. Valid values are {-1, [1, 65535]}, -1 indicates an unspecified port.\nAn unspecified port value allows protocol clients to emulate legacy cookie scope for the port.\nThis is a temporary ability and it will be removed in the future.",
-          "type": "number"
-        },
         "partitionKey": {
-          "description": "Cookie partition key. The site of the top-level URL the browser was visiting at the start\nof the request to the endpoint that set the cookie.\nIf not set, the cookie will be set as not partitioned.",
+          "description": "Cookie partition key. The site of the top-level URL the browser was visiting at the\nstart of the request to the endpoint that set the cookie. If not set, the cookie will\nbe set as not partitioned.",
           "type": "string"
         }
       },
@@ -488,14 +484,14 @@
                 "length": {
                   "type": "number"
                 },
-                "__@toStringTag@107427": {
+                "__@toStringTag@96609": {
                   "type": "string",
                   "const": "Uint8Array"
                 }
               },
               "required": [
                 "BYTES_PER_ELEMENT",
-                "__@toStringTag@107427",
+                "__@toStringTag@96609",
                 "buffer",
                 "byteLength",
                 "byteOffset",
@@ -530,13 +526,13 @@
         "byteLength": {
           "type": "number"
         },
-        "__@toStringTag@107427": {
+        "__@toStringTag@96609": {
           "type": "string"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@toStringTag@107427",
+        "__@toStringTag@96609",
         "byteLength"
       ]
     },
@@ -546,18 +542,18 @@
         "byteLength": {
           "type": "number"
         },
-        "__@species@107528": {
+        "__@species@96710": {
           "$ref": "#/definitions/SharedArrayBuffer"
         },
-        "__@toStringTag@107427": {
+        "__@toStringTag@96609": {
           "type": "string",
           "const": "SharedArrayBuffer"
         }
       },
       "additionalProperties": false,
       "required": [
-        "__@species@107528",
-        "__@toStringTag@107427",
+        "__@species@96710",
+        "__@toStringTag@96609",
         "byteLength"
       ]
     },

package/build/routes/chromium/http/screenshot-post.d.ts

@@ -33,7 +33,7 @@ export interface BodySchema {
     waitForEvent?: WaitForEventOptions;
     waitForFunction?: WaitForFunctionOptions;
     waitForSelector?: WaitForSelectorOptions;
-    waitForTimeout?: Parameters<Page['waitForTimeout']>[0];
+    waitForTimeout?: number;
 }
 export default class ScreenshotPost extends BrowserHTTPRoute {
     accepts: contentTypes[];

package/build/routes/chromium/http/screenshot-post.js

@@ -1,4 +1,4 @@
-import { APITags, BadRequest, BrowserHTTPRoute, CDPChromium, HTTPRoutes, Methods, bestAttemptCatch, contentTypes, dedent, noop, scrollThroughPage, waitForEvent as waitForEvt, waitForFunction as waitForFn, } from '@browserless.io/browserless';
+import { APITags, BadRequest, BrowserHTTPRoute, CDPChromium, HTTPRoutes, Methods, bestAttemptCatch, contentTypes, dedent, noop, scrollThroughPage, sleep, waitForEvent as waitForEvt, waitForFunction as waitForFn, } from '@browserless.io/browserless';
 import Stream from 'stream';
 export default class ScreenshotPost extends BrowserHTTPRoute {
     accepts = [contentTypes.json];
@@ -81,9 +81,7 @@ export default class ScreenshotPost extends BrowserHTTPRoute {
         }
         const gotoResponse = await gotoCall(content, gotoOptions).catch(bestAttemptCatch(bestAttempt));
         if (waitForTimeout) {
-            await page
-                .waitForTimeout(waitForTimeout)
-                .catch(bestAttemptCatch(bestAttempt));
+            await sleep(waitForTimeout).catch(bestAttemptCatch(bestAttempt));
         }
         if (waitForFunction) {
             await waitForFn(page, waitForFunction).catch(bestAttemptCatch(bestAttempt));

package/build/routes/chromium/tests/content.spec.js

@@ -1,4 +1,4 @@
-import { Browserless, Config, Metrics } from '@browserless.io/browserless';
+import { Browserless, Config, Metrics, sleep } from '@browserless.io/browserless';
 import { expect } from 'chai';
 describe('/content API', function () {
     let browserless;
@@ -32,6 +32,32 @@ describe('/content API', function () {
             expect(res.status).to.equal(200);
         });
     });
+    it('cancels request when they are closed early', async () => {
+        const config = new Config();
+        const metrics = new Metrics();
+        await start({ config, metrics });
+        const body = {
+            url: 'https://cnn.com',
+        };
+        const controller = new AbortController();
+        const signal = controller.signal;
+        const promise = fetch('http://localhost:3000/content', {
+            body: JSON.stringify(body),
+            headers: {
+                'content-type': 'application/json',
+            },
+            method: 'POST',
+            signal,
+        }).catch(async (error) => {
+            await sleep(100);
+            expect(error).to.have.property('name', 'AbortError');
+            expect(metrics.get().error).to.equal(1);
+            expect(metrics.get().successful).to.equal(0);
+        });
+        await sleep(1000);
+        controller.abort();
+        return promise;
+    });
     it('404s GET requests', async () => {
         const config = new Config();
         config.setToken('browserless');

package/build/routes/chromium/tests/websocket.spec.js

@@ -1,4 +1,4 @@
-import { Browserless, Config, Metrics, exists, sleep, } from '@browserless.io/browserless';
+import { Browserless, Config, Metrics, exists, fetchJson, sleep, } from '@browserless.io/browserless';
 import { chromium } from 'playwright-core';
 import { deleteAsync } from 'del';
 import { expect } from 'chai';
@@ -56,6 +56,54 @@ describe('WebSocket API', function () {
         });
         await Promise.all([browser.disconnect(), browserTwo.disconnect()]);
     });
+    it('does not close browsers when multiple clients are connected', async () => {
+        const config = new Config();
+        config.setToken('browserless');
+        const metrics = new Metrics();
+        await start({ config, metrics });
+        // Single session
+        const browser = await puppeteer.connect({
+            browserWSEndpoint: `ws://localhost:3000?token=browserless`,
+        });
+        const [session] = (await fetchJson('http://localhost:3000/sessions?token=browserless'));
+        expect(session.numbConnected).to.equal(1);
+        // Two sessions
+        const browserTwo = await puppeteer.connect({
+            browserWSEndpoint: `ws://localhost:3000/devtools/browser/${session.browserId}?token=browserless`,
+        });
+        const [twoSessions] = (await fetchJson('http://localhost:3000/sessions?token=browserless'));
+        expect(twoSessions.numbConnected).to.equal(2);
+        // Back to a single session
+        await browser.disconnect();
+        await sleep(50);
+        const [oneSession] = (await fetchJson('http://localhost:3000/sessions?token=browserless'));
+        expect(oneSession.numbConnected).to.equal(1);
+        // No sessions connected
+        await browserTwo.disconnect();
+        await sleep(50);
+        const sessionsFinal = (await fetchJson('http://localhost:3000/sessions?token=browserless'));
+        expect(sessionsFinal).to.have.length(0);
+    });
+    it('disconnects all clients when the timeout is reached', async () => {
+        const config = new Config();
+        config.setToken('browserless');
+        config.setTimeout(1000);
+        config.setConcurrent(2);
+        const metrics = new Metrics();
+        await start({ config, metrics });
+        const browser = await puppeteer.connect({
+            browserWSEndpoint: `ws://localhost:3000?token=browserless`,
+        });
+        const [session] = (await fetchJson('http://localhost:3000/sessions?token=browserless'));
+        const browserTwo = await puppeteer.connect({
+            browserWSEndpoint: `ws://localhost:3000/devtools/browser/${session.browserId}?token=browserless`,
+        });
+        await sleep(3000);
+        expect(metrics.get().successful).to.equal(0);
+        expect(metrics.get().timedout).to.equal(2);
+        expect(browser.connected).to.be.false;
+        expect(browserTwo.connected).to.be.false;
+    });
     it('rejects websocket requests', async () => {
         const config = new Config();
         config.setToken('browserless');
@@ -189,12 +237,13 @@ describe('WebSocket API', function () {
         const metrics = new Metrics();
         await start({ config, metrics });
         const browser = await puppeteer.connect({
-            browserWSEndpoint: `ws://localhost:3000?timeout=500&token=browserless`,
+            browserWSEndpoint: `ws://localhost:3000?timeout=1000&token=browserless`,
         });
-        await sleep(750);
-        browser.disconnect();
+        expect(browser.connected).to.be.true;
+        await sleep(1200);
         expect(metrics.get().timedout).to.equal(1);
         expect(metrics.get().successful).to.equal(0);
+        expect(browser.connected).to.be.false;
     });
     it('allows the file-chooser', async () => new Promise(async (done) => {
         const config = new Config();

package/build/routes/chromium/ws/browser.js

@@ -2,7 +2,7 @@ import { APITags, BrowserWebsocketRoute, CDPChromium, WebsocketRoutes, } from '@
 export default class CDPExistingBrowser extends BrowserWebsocketRoute {
     auth = true;
     browser = CDPChromium;
-    concurrency = false;
+    concurrency = true;
     description = `Connect to an already-running Chromium with a library like puppeteer, or others, that work over chrome-devtools-protocol.`;
     path = WebsocketRoutes.browser;
     tags = [APITags.browserWS];

package/build/routes/management/http/sessions-get.response.json

@@ -10,6 +10,9 @@
         "browser": {
           "type": "string"
         },
+        "browserId": {
+          "type": "string"
+        },
         "id": {
           "type": [
             "null",
@@ -57,6 +60,7 @@
       "additionalProperties": false,
       "required": [
         "browser",
+        "browserId",
         "id",
         "initialConnectURL",
         "killURL",

package/build/types.d.ts

@@ -274,6 +274,7 @@ export interface BrowserlessSession {
 }
 export interface BrowserlessSessionJSON {
     browser: string;
+    browserId: string;
     id: string | null;
     initialConnectURL: string;
     killURL: string | null;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@browserless.io/browserless",
-  "version": "2.2.0-beta-2",
+  "version": "2.2.0-beta-3",
   "license": "SSPL",
   "description": "The browserless platform",
   "author": "browserless.io",
@@ -56,7 +56,7 @@
     "lighthouse": "^11.1.0",
     "micromatch": "^4.0.4",
     "playwright-core": "^1.41.2",
-    "puppeteer-core": "^21.10.0",
+    "puppeteer-core": "^22.0.0",
     "puppeteer-extra": "^3.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2",
     "queue": "^7.0.0",
@@ -71,7 +71,7 @@
     "@types/mocha": "^10.0.6",
     "@types/node": "^20.11.16",
     "@types/sinon": "^17.0.3",
-    "@typescript-eslint/eslint-plugin": "^6.20.0",
+    "@typescript-eslint/eslint-plugin": "^6.21.0",
     "@typescript-eslint/parser": "^6.21.0",
     "assert": "^2.0.0",
     "chai": "^5.0.3",

package/src/browsers/index.ts

@@ -125,7 +125,7 @@ export class BrowserManager {
       {
         ...session,
         browser: browser.constructor.name,
-        browserId: browser.wsEndpoint()?.split('/').pop(),
+        browserId: browser.wsEndpoint()?.split('/').pop() as string,
         initialConnectURL: new URL(session.initialConnectURL, serverAddress)
           .href,
         killURL: session.id
@@ -170,6 +170,11 @@ export class BrowserManager {
     const cleanupACtions: Array<() => Promise<void>> = [];
     this.debug(`${session.numbConnected} Client(s) are currently connected`);
 
+    // Don't close if there's clients still connected
+    if (session.numbConnected > 0) {
+      return;
+    }
+
     this.debug(`Closing browser session`);
     cleanupACtions.push(() => browser.close());
 
@@ -234,13 +239,15 @@ export class BrowserManager {
     if (req.parsed.pathname.includes('/devtools/browser')) {
       const sessions = Array.from(this.browsers);
       const id = req.parsed.pathname.split('/').pop() as string;
-      const browser = sessions.find(([b]) =>
+      const found = sessions.find(([b]) =>
         b.wsEndpoint()?.includes(req.parsed.pathname),
       );
 
-      if (browser) {
+      if (found) {
+        const [browser, session] = found;
+        ++session.numbConnected;
         this.debug(`Located browser with ID ${id}`);
-        return browser[0];
+        return browser;
       }
 
       throw new NotFound(

package/src/router.ts

@@ -80,17 +80,23 @@ export class Router {
         }
 
         if (!browser) {
-          return writeResponse(res, 500, `Error loading the browser.`);
-        }
-
-        if (!isConnected(res)) {
-          this.log(`HTTP Request has closed prior to running`);
-          return Promise.resolve();
+          return writeResponse(res, 500, `Error loading the browser`);
         }
 
         try {
           this.verbose(`Running found HTTP handler.`);
-          return await handler(req, res, browser);
+          return await Promise.race([
+            handler(req, res, browser),
+            new Promise((resolve, reject) => {
+              res.once('close', () => {
+                if (!res.writableEnded) {
+                  reject(new Error(`Request closed prior to writing results`));
+                }
+                this.verbose(`Response has been written, resolving`);
+                resolve(null);
+              });
+            }),
+          ]);
         } finally {
           this.verbose(`HTTP Request handler has finished.`);
           this.browserManager.complete(browser);

package/src/routes/chromium/http/content-post.ts

@@ -21,6 +21,7 @@ import {
   rejectResourceTypes,
   requestInterceptors,
   setJavaScriptEnabled,
+  sleep,
   waitForEvent as waitForEvt,
   waitForFunction as waitForFn,
   writeResponse,
@@ -48,7 +49,7 @@ export interface BodySchema {
   waitForEvent?: WaitForEventOptions;
   waitForFunction?: WaitForFunctionOptions;
   waitForSelector?: WaitForSelectorOptions;
-  waitForTimeout?: Parameters<Page['waitForTimeout']>[0];
+  waitForTimeout?: number;
 }
 
 /**
@@ -190,9 +191,7 @@ export default class ContentPostRoute extends BrowserHTTPRoute {
     );
 
     if (waitForTimeout) {
-      await page
-        .waitForTimeout(waitForTimeout)
-        .catch(bestAttemptCatch(bestAttempt));
+      await sleep(waitForTimeout).catch(bestAttemptCatch(bestAttempt));
     }
 
     if (waitForFunction) {

package/src/routes/chromium/http/pdf-post.ts

@@ -21,12 +21,14 @@ import {
   rejectRequestPattern,
   rejectResourceTypes,
   requestInterceptors,
+  sleep,
   waitForEvent as waitForEvt,
   waitForFunction as waitForFn,
   writeResponse,
 } from '@browserless.io/browserless';
 import { Page } from 'puppeteer-core';
 import { ServerResponse } from 'http';
+import { Stream } from 'stream';
 
 export interface BodySchema {
   addScriptTag?: Array<Parameters<Page['addScriptTag']>[0]>;
@@ -50,7 +52,7 @@ export interface BodySchema {
   waitForEvent?: WaitForEventOptions;
   waitForFunction?: WaitForFunctionOptions;
   waitForSelector?: WaitForSelectorOptions;
-  waitForTimeout?: Parameters<Page['waitForTimeout']>[0];
+  waitForTimeout?: number;
 }
 
 export interface QuerySchema extends SystemQueryParameters {
@@ -111,9 +113,10 @@ export default class PDFPost extends BrowserHTTPRoute {
       setJavaScriptEnabled,
       userAgent,
       viewport,
+      waitForEvent,
       waitForFunction,
       waitForSelector,
-      waitForEvent,
+      waitForTimeout,
       bestAttempt = false,
     } = req.body as BodySchema;
 
@@ -196,6 +199,10 @@ export default class PDFPost extends BrowserHTTPRoute {
       bestAttemptCatch(bestAttempt),
     );
 
+    if (waitForTimeout) {
+      await sleep(waitForTimeout).catch(bestAttemptCatch(bestAttempt));
+    }
+
     if (waitForFunction) {
       await waitForFn(page, waitForFunction).catch(
         bestAttemptCatch(bestAttempt),
@@ -227,11 +234,11 @@ export default class PDFPost extends BrowserHTTPRoute {
       }
     }
 
-    const pdfStream = await page.createPDFStream(options);
+    const pdfBuffer = await page.pdf(options);
+    const readStream = new Stream.PassThrough();
+    readStream.end(pdfBuffer);
 
-    await new Promise((resolve, reject) => {
-      return pdfStream.pipe(res).once('finish', resolve).once('error', reject);
-    });
+    await new Promise((r) => readStream.pipe(res).once('close', r));
 
     page.close().catch(noop);
   };

package/src/routes/chromium/http/scrape-post.ts

@@ -28,10 +28,11 @@ import {
   rejectRequestPattern,
   rejectResourceTypes,
   requestInterceptors,
+  sleep,
   waitForEvent as waitForEvt,
   waitForFunction as waitForFn,
 } from '@browserless.io/browserless';
-import { Page, Protocol } from 'puppeteer-core';
+import { Cookie, Page } from 'puppeteer-core';
 import { ServerResponse } from 'http';
 
 export interface BodySchema {
@@ -56,7 +57,7 @@ export interface BodySchema {
   waitForEvent?: WaitForEventOptions;
   waitForFunction?: WaitForFunctionOptions;
   waitForSelector?: WaitForSelectorOptions;
-  waitForTimeout?: Parameters<Page['waitForTimeout']>[0];
+  waitForTimeout?: number;
 }
 
 export type QuerySchema = SystemQueryParameters & {
@@ -135,7 +136,7 @@ export interface ResponseSchema {
     /**
      * List of cookies for the site or null
      */
-    cookies: Protocol.Network.Cookie[] | null;
+    cookies: Cookie[] | null;
 
     /**
      * The HTML string of the website or null
@@ -361,9 +362,7 @@ export default class ScrapePost extends BrowserHTTPRoute {
     }
 
     if (waitForTimeout) {
-      await page
-        .waitForTimeout(waitForTimeout)
-        .catch(bestAttemptCatch(bestAttempt));
+      await sleep(waitForTimeout).catch(bestAttemptCatch(bestAttempt));
     }
 
     if (waitForFunction) {

package/src/routes/chromium/http/screenshot-post.ts

@@ -22,6 +22,7 @@ import {
   rejectResourceTypes,
   requestInterceptors,
   scrollThroughPage,
+  sleep,
   waitForEvent as waitForEvt,
   waitForFunction as waitForFn,
 } from '@browserless.io/browserless';
@@ -62,7 +63,7 @@ export interface BodySchema {
   waitForEvent?: WaitForEventOptions;
   waitForFunction?: WaitForFunctionOptions;
   waitForSelector?: WaitForSelectorOptions;
-  waitForTimeout?: Parameters<Page['waitForTimeout']>[0];
+  waitForTimeout?: number;
 }
 
 export default class ScreenshotPost extends BrowserHTTPRoute {
@@ -205,9 +206,7 @@ export default class ScreenshotPost extends BrowserHTTPRoute {
     );
 
     if (waitForTimeout) {
-      await page
-        .waitForTimeout(waitForTimeout)
-        .catch(bestAttemptCatch(bestAttempt));
+      await sleep(waitForTimeout).catch(bestAttemptCatch(bestAttempt));
     }
 
     if (waitForFunction) {

package/src/routes/chromium/tests/content.spec.ts

@@ -1,4 +1,4 @@
-import { Browserless, Config, Metrics } from '@browserless.io/browserless';
+import { Browserless, Config, Metrics, sleep } from '@browserless.io/browserless';
 import { expect } from 'chai';
 
 describe('/content API', function () {
@@ -43,6 +43,33 @@ describe('/content API', function () {
     });
   });
 
+  it('cancels request when they are closed early', async () => {
+    const config = new Config();
+    const metrics = new Metrics();
+    await start({ config, metrics });
+    const body = {
+      url: 'https://cnn.com',
+    };
+    const controller = new AbortController();
+    const signal = controller.signal;
+    const promise = fetch('http://localhost:3000/content', {
+      body: JSON.stringify(body),
+      headers: {
+        'content-type': 'application/json',
+      },
+      method: 'POST',
+      signal,
+    }).catch(async (error) => {
+      await sleep(100);
+      expect(error).to.have.property('name', 'AbortError');
+      expect(metrics.get().error).to.equal(1);
+      expect(metrics.get().successful).to.equal(0);
+    });
+    await sleep(1000);
+    controller.abort();
+    return promise;
+  });
+
   it('404s GET requests', async () => {
     const config = new Config();
     config.setToken('browserless');