@browserless.io/browserless 2.1.1 → 2.2.0-beta-3

package/src/routes/chromium/http/pdf-post.ts

@@ -21,12 +21,14 @@ import {
   rejectRequestPattern,
   rejectResourceTypes,
   requestInterceptors,
+  sleep,
   waitForEvent as waitForEvt,
   waitForFunction as waitForFn,
   writeResponse,
 } from '@browserless.io/browserless';
 import { Page } from 'puppeteer-core';
 import { ServerResponse } from 'http';
+import { Stream } from 'stream';
 
 export interface BodySchema {
   addScriptTag?: Array<Parameters<Page['addScriptTag']>[0]>;
@@ -50,7 +52,7 @@ export interface BodySchema {
   waitForEvent?: WaitForEventOptions;
   waitForFunction?: WaitForFunctionOptions;
   waitForSelector?: WaitForSelectorOptions;
-  waitForTimeout?: Parameters<Page['waitForTimeout']>[0];
+  waitForTimeout?: number;
 }
 
 export interface QuerySchema extends SystemQueryParameters {
@@ -111,9 +113,10 @@ export default class PDFPost extends BrowserHTTPRoute {
       setJavaScriptEnabled,
       userAgent,
       viewport,
+      waitForEvent,
       waitForFunction,
       waitForSelector,
-      waitForEvent,
+      waitForTimeout,
       bestAttempt = false,
     } = req.body as BodySchema;
 
@@ -196,6 +199,10 @@ export default class PDFPost extends BrowserHTTPRoute {
       bestAttemptCatch(bestAttempt),
     );
 
+    if (waitForTimeout) {
+      await sleep(waitForTimeout).catch(bestAttemptCatch(bestAttempt));
+    }
+
     if (waitForFunction) {
       await waitForFn(page, waitForFunction).catch(
         bestAttemptCatch(bestAttempt),
@@ -227,11 +234,11 @@ export default class PDFPost extends BrowserHTTPRoute {
       }
     }
 
-    const pdfStream = await page.createPDFStream(options);
+    const pdfBuffer = await page.pdf(options);
+    const readStream = new Stream.PassThrough();
+    readStream.end(pdfBuffer);
 
-    await new Promise((resolve, reject) => {
-      return pdfStream.pipe(res).once('finish', resolve).once('error', reject);
-    });
+    await new Promise((r) => readStream.pipe(res).once('close', r));
 
     page.close().catch(noop);
   };

package/src/routes/chromium/http/scrape-post.ts

@@ -28,10 +28,11 @@ import {
   rejectRequestPattern,
   rejectResourceTypes,
   requestInterceptors,
+  sleep,
   waitForEvent as waitForEvt,
   waitForFunction as waitForFn,
 } from '@browserless.io/browserless';
-import { Page, Protocol } from 'puppeteer-core';
+import { Cookie, Page } from 'puppeteer-core';
 import { ServerResponse } from 'http';
 
 export interface BodySchema {
@@ -56,7 +57,7 @@ export interface BodySchema {
   waitForEvent?: WaitForEventOptions;
   waitForFunction?: WaitForFunctionOptions;
   waitForSelector?: WaitForSelectorOptions;
-  waitForTimeout?: Parameters<Page['waitForTimeout']>[0];
+  waitForTimeout?: number;
 }
 
 export type QuerySchema = SystemQueryParameters & {
@@ -135,7 +136,7 @@ export interface ResponseSchema {
     /**
      * List of cookies for the site or null
      */
-    cookies: Protocol.Network.Cookie[] | null;
+    cookies: Cookie[] | null;
 
     /**
      * The HTML string of the website or null
@@ -361,9 +362,7 @@ export default class ScrapePost extends BrowserHTTPRoute {
     }
 
     if (waitForTimeout) {
-      await page
-        .waitForTimeout(waitForTimeout)
-        .catch(bestAttemptCatch(bestAttempt));
+      await sleep(waitForTimeout).catch(bestAttemptCatch(bestAttempt));
     }
 
     if (waitForFunction) {

package/src/routes/chromium/http/screenshot-post.ts

@@ -22,6 +22,7 @@ import {
   rejectResourceTypes,
   requestInterceptors,
   scrollThroughPage,
+  sleep,
   waitForEvent as waitForEvt,
   waitForFunction as waitForFn,
 } from '@browserless.io/browserless';
@@ -62,7 +63,7 @@ export interface BodySchema {
   waitForEvent?: WaitForEventOptions;
   waitForFunction?: WaitForFunctionOptions;
   waitForSelector?: WaitForSelectorOptions;
-  waitForTimeout?: Parameters<Page['waitForTimeout']>[0];
+  waitForTimeout?: number;
 }
 
 export default class ScreenshotPost extends BrowserHTTPRoute {
@@ -205,9 +206,7 @@ export default class ScreenshotPost extends BrowserHTTPRoute {
     );
 
     if (waitForTimeout) {
-      await page
-        .waitForTimeout(waitForTimeout)
-        .catch(bestAttemptCatch(bestAttempt));
+      await sleep(waitForTimeout).catch(bestAttemptCatch(bestAttempt));
     }
 
     if (waitForFunction) {

package/src/routes/chromium/tests/content.spec.ts

@@ -1,4 +1,4 @@
-import { Browserless, Config, Metrics } from '@browserless.io/browserless';
+import { Browserless, Config, Metrics, sleep } from '@browserless.io/browserless';
 import { expect } from 'chai';
 
 describe('/content API', function () {
@@ -43,6 +43,33 @@ describe('/content API', function () {
     });
   });
 
+  it('cancels request when they are closed early', async () => {
+    const config = new Config();
+    const metrics = new Metrics();
+    await start({ config, metrics });
+    const body = {
+      url: 'https://cnn.com',
+    };
+    const controller = new AbortController();
+    const signal = controller.signal;
+    const promise = fetch('http://localhost:3000/content', {
+      body: JSON.stringify(body),
+      headers: {
+        'content-type': 'application/json',
+      },
+      method: 'POST',
+      signal,
+    }).catch(async (error) => {
+      await sleep(100);
+      expect(error).to.have.property('name', 'AbortError');
+      expect(metrics.get().error).to.equal(1);
+      expect(metrics.get().successful).to.equal(0);
+    });
+    await sleep(1000);
+    controller.abort();
+    return promise;
+  });
+
   it('404s GET requests', async () => {
     const config = new Config();
     config.setToken('browserless');

package/src/routes/chromium/tests/websocket.spec.ts

@@ -1,8 +1,10 @@
 import {
   Browserless,
+  BrowserlessSessionJSON,
   Config,
   Metrics,
   exists,
+  fetchJson,
   sleep,
 } from '@browserless.io/browserless';
 import { chromium } from 'playwright-core';
@@ -85,6 +87,70 @@ describe('WebSocket API', function () {
     await Promise.all([browser.disconnect(), browserTwo.disconnect()]);
   });
 
+  it('does not close browsers when multiple clients are connected', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    // Single session
+    const browser = await puppeteer.connect({
+      browserWSEndpoint: `ws://localhost:3000?token=browserless`,
+    });
+    const [session] = (await fetchJson(
+      'http://localhost:3000/sessions?token=browserless',
+    )) as BrowserlessSessionJSON[];
+    expect(session.numbConnected).to.equal(1);
+
+    // Two sessions
+    const browserTwo = await puppeteer.connect({
+      browserWSEndpoint: `ws://localhost:3000/devtools/browser/${session.browserId}?token=browserless`,
+    });
+    const [twoSessions] = (await fetchJson(
+      'http://localhost:3000/sessions?token=browserless',
+    )) as BrowserlessSessionJSON[];
+    expect(twoSessions.numbConnected).to.equal(2);
+
+    // Back to a single session
+    await browser.disconnect();
+    await sleep(50);
+    const [oneSession] = (await fetchJson(
+      'http://localhost:3000/sessions?token=browserless',
+    )) as BrowserlessSessionJSON[];
+    expect(oneSession.numbConnected).to.equal(1);
+
+    // No sessions connected
+    await browserTwo.disconnect();
+    await sleep(50);
+    const sessionsFinal = (await fetchJson(
+      'http://localhost:3000/sessions?token=browserless',
+    )) as BrowserlessSessionJSON[];
+    expect(sessionsFinal).to.have.length(0);
+  });
+
+  it('disconnects all clients when the timeout is reached', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    config.setTimeout(1000);
+    config.setConcurrent(2);
+    const metrics = new Metrics();
+    await start({ config, metrics });
+    const browser = await puppeteer.connect({
+      browserWSEndpoint: `ws://localhost:3000?token=browserless`,
+    });
+    const [session] = (await fetchJson(
+      'http://localhost:3000/sessions?token=browserless',
+    )) as BrowserlessSessionJSON[];
+    const browserTwo = await puppeteer.connect({
+      browserWSEndpoint: `ws://localhost:3000/devtools/browser/${session.browserId}?token=browserless`,
+    });
+    await sleep(3000);
+    expect(metrics.get().successful).to.equal(0);
+    expect(metrics.get().timedout).to.equal(2);
+    expect(browser.connected).to.be.false;
+    expect(browserTwo.connected).to.be.false;
+  });
+
   it('rejects websocket requests', async () => {
     const config = new Config();
     config.setToken('browserless');
@@ -261,13 +327,13 @@ describe('WebSocket API', function () {
     await start({ config, metrics });
 
     const browser = await puppeteer.connect({
-      browserWSEndpoint: `ws://localhost:3000?timeout=500&token=browserless`,
+      browserWSEndpoint: `ws://localhost:3000?timeout=1000&token=browserless`,
     });
-
-    await sleep(750);
-    browser.disconnect();
+    expect(browser.connected).to.be.true;
+    await sleep(1200);
     expect(metrics.get().timedout).to.equal(1);
     expect(metrics.get().successful).to.equal(0);
+    expect(browser.connected).to.be.false;
   });
 
   it('allows the file-chooser', async () =>

package/src/routes/chromium/ws/browser.ts

@@ -16,7 +16,7 @@ export interface QuerySchema extends SystemQueryParameters {
 export default class CDPExistingBrowser extends BrowserWebsocketRoute {
   auth = true;
   browser = CDPChromium;
-  concurrency = false;
+  concurrency = true;
   description = `Connect to an already-running Chromium with a library like puppeteer, or others, that work over chrome-devtools-protocol.`;
   path = WebsocketRoutes.browser;
   tags = [APITags.browserWS];

package/src/types.ts

@@ -365,6 +365,7 @@ export interface BrowserlessSession {
 
 export interface BrowserlessSessionJSON {
   browser: string;
+  browserId: string;
   id: string | null;
   initialConnectURL: string;
   killURL: string | null;

package/src/utils.ts

@@ -255,6 +255,32 @@ export const removeNullStringify = (
 export const jsonOrString = (maybeJson: string): unknown | string =>
   safeParse(maybeJson) ?? maybeJson;
 
+export const generateDataDir = async (
+  sessionId: string = id(),
+  config: Config,
+): Promise<string> => {
+  const baseDirectory = await config.getDataDir();
+  const dataDirPath = path.join(
+    baseDirectory,
+    `browserless-data-dir-${sessionId}`,
+  );
+
+  if (await exists(dataDirPath)) {
+    debug(`Data directory already exists, not creating "${dataDirPath}"`);
+    return dataDirPath;
+  }
+
+  debug(`Generating user-data-dir at ${dataDirPath}`);
+
+  await fs.mkdir(dataDirPath, { recursive: true }).catch((err) => {
+    throw new ServerError(
+      `Error creating data-directory "${dataDirPath}": ${err}`,
+    );
+  });
+
+  return dataDirPath;
+};
+
 export const readBody = async (
   req: Request,
 ): Promise<ReturnType<typeof safeParse>> => {