@browserless.io/browserless 2.25.0-beta-2 → 2.25.0

package/src/browsers/browsers.playwright.ts

@@ -6,6 +6,7 @@ import {
   Request,
   ServerError,
   chromeExecutablePath,
+  edgeExecutablePath,
 } from '@browserless.io/browserless';
 import playwright, { Page } from 'playwright-core';
 import { Duplex } from 'stream';
@@ -56,10 +57,17 @@ class BasePlaywright extends EventEmitter {
   }
 
   protected makeLaunchOptions(opts: BrowserServerOptions) {
+    // Strip headless=old as it'll cause issues with newer Chromium
+    const args = (opts.args ?? []).filter((a) => !a.includes('--headless=old'));
+
+    if (!args.some((a) => a.startsWith('--headless'))) {
+      args.push('--headless=new');
+    }
+
     return {
       ...opts,
       args: [
-        ...(opts.args || []),
+        ...args,
         this.userDataDir ? `--user-data-dir=${this.userDataDir}` : '',
       ],
       executablePath: this.executablePath(),
@@ -216,6 +224,11 @@ export class ChromePlaywright extends ChromiumPlaywright {
   protected playwrightBrowserType = PlaywrightBrowserTypes.chromium;
 }
 
+export class EdgePlaywright extends ChromiumPlaywright {
+  protected executablePath = () => edgeExecutablePath();
+  protected playwrightBrowserType = PlaywrightBrowserTypes.chromium;
+}
+
 export class FirefoxPlaywright extends BasePlaywright {
   protected playwrightBrowserType = PlaywrightBrowserTypes.firefox;
 

package/src/browsers/index.ts

@@ -14,6 +14,8 @@ import {
   ChromiumCDP,
   ChromiumPlaywright,
   Config,
+  EdgeCDP,
+  EdgePlaywright,
   FileSystem,
   FirefoxPlaywright,
   Hooks,
@@ -41,10 +43,11 @@ export class BrowserManager {
   protected browsers: Map<BrowserInstance, BrowserlessSession> = new Map();
   protected timers: Map<string, NodeJS.Timeout> = new Map();
   protected log = new Logger('browser-manager');
-  protected chromeBrowsers = [ChromiumCDP, ChromeCDP];
+  protected chromeBrowsers = [ChromiumCDP, ChromeCDP, EdgeCDP];
   protected playwrightBrowserNames = [
     ChromiumPlaywright.name,
     ChromePlaywright.name,
+    EdgePlaywright.name,
     FirefoxPlaywright.name,
     WebKitPlaywright.name,
   ];
@@ -538,7 +541,7 @@ export class BrowserManager {
     if (
       launchOptions.args &&
       proxyServerArg &&
-      req.parsed.pathname.startsWith('/playwright')
+      req.parsed.pathname.includes('/playwright')
     ) {
       (launchOptions as BrowserServerOptions).proxy = {
         server: proxyServerArg.split('=')[1],

package/src/http.ts

@@ -83,7 +83,9 @@ export enum WebsocketRoutes {
   '/' = '?(/)',
   browser = '/devtools/browser/*',
   chrome = '/chrome?(/)',
+  edge = '/edge?(/)',
   chromePlaywright = '/chrome/playwright?(/)',
+  edgePlaywright = '/edge/playwright?(/)',
   chromium = '/chromium?(/)',
   chromiumPlaywright = '/chromium/playwright?(/)',
   firefoxPlaywright = '/firefox/playwright?(/)',
@@ -103,6 +105,13 @@ export enum HTTPRoutes {
   chromePerformance = '/chrome/performance?(/)',
   chromeScrape = '/chrome/scrape?(/)',
   chromeScreenshot = '/chrome/screenshot?(/)',
+  edgeContent = '/edge/content?(/)',
+  edgeDownload = '/edge/download?(/)',
+  edgeFunction = '/edge/function?(/)',
+  edgePdf = '/edge/pdf?(/)',
+  edgePerformance = '/edge/performance?(/)',
+  edgeScrape = '/edge/scrape?(/)',
+  edgeScreenshot = '/edge/screenshot?(/)',
   chromiumContent = '/chromium/content?(/)',
   chromiumDownload = '/chromium/download?(/)',
   chromiumFunction = '/chromium/function?(/)',

package/src/routes/chrome/tests/kill-sessions.spec.ts

@@ -47,7 +47,7 @@ describe('/kill API', function () {
     }
     expect((errorThrown1 as Error).message).contains('closed');
     expect((errorThrown2 as Error).message).contains('closed');
-  });
+  }).timeout(45000);
 
   it('Kill session by browserId', async () => {
     await start();

package/src/routes/chromium/tests/kill-sessions.spec.ts

@@ -47,7 +47,7 @@ describe('/kill API', function () {
     }
     expect((errorThrown1 as Error).message).contains('closed');
     expect((errorThrown2 as Error).message).contains('closed');
-  });
+  }).timeout(45000);
 
   it('Kill session by browserId', async () => {
     await start();

package/src/routes/edge/http/content.post.ts

@@ -0,0 +1,20 @@
+import {
+  BrowserlessRoutes,
+  EdgeCDP,
+  HTTPRoutes,
+} from '@browserless.io/browserless';
+
+import {
+  BodySchema,
+  default as Content,
+  QuerySchema,
+  ResponseSchema,
+} from '../../../shared/content.http.js';
+
+export default class EdgeContentPostRoute extends Content {
+  name = BrowserlessRoutes.EdgeContentPostRoute;
+  browser = EdgeCDP;
+  path = [HTTPRoutes.edgeContent];
+}
+
+export { BodySchema, QuerySchema, ResponseSchema };

package/src/routes/edge/http/download.post.ts

@@ -0,0 +1,20 @@
+import {
+  BrowserlessRoutes,
+  EdgeCDP,
+  HTTPRoutes,
+} from '@browserless.io/browserless';
+
+import {
+  BodySchema,
+  default as Download,
+  QuerySchema,
+  ResponseSchema,
+} from '../../../shared/download.http.js';
+
+export default class EdgeDownloadPostRoute extends Download {
+  name = BrowserlessRoutes.EdgeDownloadPostRoute;
+  browser = EdgeCDP;
+  path = [HTTPRoutes.edgeDownload];
+}
+
+export { BodySchema, QuerySchema, ResponseSchema };

package/src/routes/edge/http/function.post.ts

@@ -0,0 +1,20 @@
+import {
+  BrowserlessRoutes,
+  EdgeCDP,
+  HTTPRoutes,
+} from '@browserless.io/browserless';
+
+import {
+  BodySchema,
+  default as Function,
+  QuerySchema,
+  ResponseSchema,
+} from '../../../shared/function.http.js';
+
+export default class EdgeFunctionPostRoute extends Function {
+  name = BrowserlessRoutes.EdgeFunctionPostRoute;
+  browser = EdgeCDP;
+  path = [HTTPRoutes.edgeFunction];
+}
+
+export { BodySchema, QuerySchema, ResponseSchema };

package/src/routes/edge/http/json-list.get.ts

@@ -0,0 +1,7 @@
+import { BrowserlessRoutes } from '@browserless.io/browserless';
+import { default as ChromiumJSONListGetRoute } from '../../../shared/json-list.http.js';
+export { ResponseSchema } from '../../../shared/json-list.http.js';
+
+export default class EdgeJSONListGetRoute extends ChromiumJSONListGetRoute {
+  name = BrowserlessRoutes.EdgeJSONListGetRoute;
+}

package/src/routes/edge/http/json-new.put.ts

@@ -0,0 +1,7 @@
+import { BrowserlessRoutes } from '@browserless.io/browserless';
+import { default as ChromiumJSONNewPutRoute } from '../../../shared/json-new.http.js';
+export { ResponseSchema } from '../../../shared/json-new.http.js';
+
+export default class EdgeJSONNewPutRoute extends ChromiumJSONNewPutRoute {
+  name = BrowserlessRoutes.EdgeJSONNewPutRoute;
+}

package/src/routes/edge/http/json-protocol.get.ts

@@ -0,0 +1,7 @@
+import { BrowserlessRoutes } from '@browserless.io/browserless';
+import { default as ChromiumJSONProtocolGetRoute } from '../../../shared/json-protocol.http.js';
+export { ResponseSchema } from '../../../shared/json-protocol.http.js';
+
+export default class EdgeJSONProtocolGetRoute extends ChromiumJSONProtocolGetRoute {
+  name = BrowserlessRoutes.EdgeJSONProtocolGetRoute;
+}

package/src/routes/edge/http/json-version.get.ts

@@ -0,0 +1,7 @@
+import { BrowserlessRoutes } from '@browserless.io/browserless';
+import { default as ChromiumJSONVersionGetRoute } from '../../../shared/json-version.http.js';
+export { ResponseSchema } from '../../../shared/json-version.http.js';
+
+export default class EdgeJSONVersionGetRoute extends ChromiumJSONVersionGetRoute {
+  name = BrowserlessRoutes.EdgeJSONVersionGetRoute;
+}

package/src/routes/edge/http/pdf.post.ts

@@ -0,0 +1,20 @@
+import {
+  BrowserlessRoutes,
+  EdgeCDP,
+  HTTPRoutes,
+} from '@browserless.io/browserless';
+
+import {
+  BodySchema,
+  default as PDF,
+  QuerySchema,
+  ResponseSchema,
+} from '../../../shared/pdf.http.js';
+
+export default class EdgePDFPostRoute extends PDF {
+  name = BrowserlessRoutes.EdgePDFPostRoute;
+  browser = EdgeCDP;
+  path = [HTTPRoutes.edgePdf];
+}
+
+export { BodySchema, QuerySchema, ResponseSchema };

package/src/routes/edge/http/performance.post.ts

@@ -0,0 +1,20 @@
+import {
+  BrowserlessRoutes,
+  EdgeCDP,
+  HTTPRoutes,
+} from '@browserless.io/browserless';
+
+import {
+  BodySchema,
+  default as Performance,
+  QuerySchema,
+  ResponseSchema,
+} from '../../../shared/performance.http.js';
+
+export default class EdgePerformancePostRoute extends Performance {
+  name = BrowserlessRoutes.EdgePerformancePostRoute;
+  browser = EdgeCDP;
+  path = [HTTPRoutes.edgePerformance];
+}
+
+export { BodySchema, QuerySchema, ResponseSchema };

package/src/routes/edge/http/scrape.post.ts

@@ -0,0 +1,20 @@
+import {
+  BrowserlessRoutes,
+  EdgeCDP,
+  HTTPRoutes,
+} from '@browserless.io/browserless';
+
+import {
+  BodySchema,
+  QuerySchema,
+  ResponseSchema,
+  default as Scrape,
+} from '../../../shared/scrape.http.js';
+
+export default class EdgeScrapePostRoute extends Scrape {
+  name = BrowserlessRoutes.EdgeScrapePostRoute;
+  browser = EdgeCDP;
+  path = [HTTPRoutes.edgeScrape];
+}
+
+export { BodySchema, QuerySchema, ResponseSchema };

package/src/routes/edge/http/screenshot.post.ts

@@ -0,0 +1,20 @@
+import {
+  BrowserlessRoutes,
+  EdgeCDP,
+  HTTPRoutes,
+} from '@browserless.io/browserless';
+
+import {
+  BodySchema,
+  QuerySchema,
+  ResponseSchema,
+  default as Screenshot,
+} from '../../../shared/screenshot.http.js';
+
+export default class EdgeScreenshotPostRoute extends Screenshot {
+  name = BrowserlessRoutes.EdgeScreenshotPostRoute;
+  browser = EdgeCDP;
+  path = [HTTPRoutes.edgeScreenshot];
+}
+
+export { BodySchema, QuerySchema, ResponseSchema };

package/src/routes/edge/tests/content.spec.ts

@@ -0,0 +1,376 @@
+/* eslint-disable no-unused-expressions */
+import {
+  Browserless,
+  Config,
+  Metrics,
+  sleep,
+} from '@browserless.io/browserless';
+import { expect } from 'chai';
+
+describe('/edge/content API', function () {
+  let browserless: Browserless;
+
+  const start = ({
+    config = new Config(),
+    metrics = new Metrics(),
+  }: { config?: Config; metrics?: Metrics } = {}) => {
+    browserless = new Browserless({ config, metrics });
+    return browserless.start();
+  };
+
+  afterEach(async () => {
+    await browserless.stop();
+  });
+
+  it('allows requests', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+    const body = {
+      url: 'https://example.com',
+    };
+
+    await fetch('http://localhost:3000/edge/content?token=browserless', {
+      body: JSON.stringify(body),
+      headers: {
+        'content-type': 'application/json',
+      },
+      method: 'POST',
+    }).then((res) => {
+      expect(res.headers.get('x-response-code')).to.not.be.undefined;
+      expect(res.headers.get('x-response-url')).to.not.be.undefined;
+      expect(res.headers.get('x-response-ip')).to.not.be.undefined;
+      expect(res.headers.get('x-response-por')).to.not.be.undefined;
+      expect(res.headers.get('content-type')).to.equal(
+        'text/html; charset=UTF-8',
+      );
+      expect(res.status).to.equal(200);
+    });
+  });
+
+  it('cancels request when they are closed early', async () => {
+    const config = new Config();
+    const metrics = new Metrics();
+    await start({ config, metrics });
+    const body = {
+      url: 'https://cnn.com',
+    };
+    const controller = new AbortController();
+    const signal = controller.signal;
+    const promise = fetch('http://localhost:3000/edge/content', {
+      body: JSON.stringify(body),
+      headers: {
+        'content-type': 'application/json',
+      },
+      method: 'POST',
+      signal,
+    }).catch(async (error) => {
+      await sleep(100);
+      expect(error).to.have.property('name', 'AbortError');
+      expect(metrics.get().error).to.equal(1);
+      expect(metrics.get().successful).to.equal(0);
+    });
+    await sleep(1000);
+    controller.abort();
+    return promise;
+  });
+
+  it('404s GET requests', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    await fetch('http://localhost:3000/edge/content?token=browserless').then(
+      (res) => {
+        expect(res.headers.get('content-type')).to.equal(
+          'text/plain; charset=UTF-8',
+        );
+        expect(res.status).not.to.equal(200);
+      },
+    );
+  });
+
+  it('handles `waitForFunction` properties', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+    const body = {
+      url: 'https://example.com',
+      waitForFunction: {
+        fn: '() => 5 + 5',
+      },
+    };
+
+    await fetch('http://localhost:3000/edge/content?token=browserless', {
+      body: JSON.stringify(body),
+      headers: {
+        'content-type': 'application/json',
+      },
+      method: 'POST',
+    }).then((res) => {
+      expect(res.headers.get('content-type')).to.equal(
+        'text/html; charset=UTF-8',
+      );
+      expect(res.status).to.equal(200);
+    });
+  });
+
+  it('handles async `waitForFunction` properties', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+    const body = {
+      url: 'https://example.com',
+      waitForFunction: {
+        fn: 'async () => new Promise(resolve => resolve(5))',
+      },
+    };
+
+    await fetch('http://localhost:3000/edge/content?token=browserless', {
+      body: JSON.stringify(body),
+      headers: {
+        'content-type': 'application/json',
+      },
+      method: 'POST',
+    }).then((res) => {
+      expect(res.headers.get('content-type')).to.equal(
+        'text/html; charset=UTF-8',
+      );
+      expect(res.status).to.equal(200);
+    });
+  });
+
+  it('handles `waitForSelector` properties', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+    const body = {
+      url: 'https://example.com',
+      waitForSelector: {
+        selector: 'h1',
+      },
+    };
+
+    await fetch('http://localhost:3000/edge/content?token=browserless', {
+      body: JSON.stringify(body),
+      headers: {
+        'content-type': 'application/json',
+      },
+      method: 'POST',
+    }).then((res) => {
+      expect(res.headers.get('content-type')).to.equal(
+        'text/html; charset=UTF-8',
+      );
+      expect(res.status).to.equal(200);
+    });
+  });
+
+  it('handles `waitForTimeout` properties', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+    const body = {
+      url: 'https://example.com',
+      waitForTimeout: 500,
+    };
+
+    await fetch('http://localhost:3000/edge/content?token=browserless', {
+      body: JSON.stringify(body),
+      headers: {
+        'content-type': 'application/json',
+      },
+      method: 'POST',
+    }).then((res) => {
+      expect(res.headers.get('content-type')).to.equal(
+        'text/html; charset=UTF-8',
+      );
+      expect(res.status).to.equal(200);
+    });
+  });
+
+  it('handles `waitForEvent` properties', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+    const body = {
+      html: `<script type="text/javascript">
+      const event = new Event("customEvent");
+      setTimeout(() => document.dispatchEvent(event), 1500);
+      </script>`,
+      waitForEvent: {
+        event: 'customEvent',
+      },
+    };
+
+    await fetch('http://localhost:3000/edge/content?token=browserless', {
+      body: JSON.stringify(body),
+      headers: {
+        'content-type': 'application/json',
+      },
+      method: 'POST',
+    }).then((res) => {
+      expect(res.headers.get('content-type')).to.equal(
+        'text/html; charset=UTF-8',
+      );
+      expect(res.status).to.equal(200);
+    });
+  });
+
+  it('allows cookies', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+    const body = {
+      cookies: [{ domain: 'example.com', name: 'foo', value: 'bar' }],
+      url: 'https://example.com',
+    };
+
+    await fetch('http://localhost:3000/edge/content?token=browserless', {
+      body: JSON.stringify(body),
+      headers: {
+        'content-type': 'application/json',
+      },
+      method: 'POST',
+    }).then((res) => {
+      expect(res.headers.get('content-type')).to.equal(
+        'text/html; charset=UTF-8',
+      );
+      expect(res.status).to.equal(200);
+    });
+  });
+
+  it('times out requests', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+    const body = {
+      url: 'https://example.com',
+    };
+
+    await fetch(
+      'http://localhost:3000/edge/content?token=browserless&timeout=10',
+      {
+        body: JSON.stringify(body),
+        headers: {
+          'content-type': 'application/json',
+        },
+        method: 'POST',
+      },
+    ).then((res) => {
+      expect(res.status).to.equal(408);
+    });
+  });
+
+  it('rejects requests', async () => {
+    const config = new Config();
+    config.setConcurrent(0);
+    config.setQueued(0);
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const body = {
+      url: 'https://example.com',
+    };
+
+    await fetch('http://localhost:3000/edge/content?token=browserless', {
+      body: JSON.stringify(body),
+      headers: {
+        'content-type': 'application/json',
+      },
+      method: 'POST',
+    }).then(async (res) => {
+      expect(await res.text()).to.equal('Too many requests\n');
+      expect(res.status).to.equal(429);
+    });
+  });
+
+  it('allows for providing http response payloads', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    config.setTimeout(30000);
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const body = {
+      requestInterceptors: [
+        {
+          pattern: '.*data.json',
+          response: {
+            body: '{"data": 123}',
+            contentType: 'application/json',
+            status: 200,
+          },
+        },
+      ],
+      url: 'https://example.com',
+    };
+
+    await fetch('http://localhost:3000/edge/content?token=browserless', {
+      body: JSON.stringify(body),
+      headers: {
+        'content-type': 'application/json',
+      },
+      method: 'POST',
+    }).then(async (res) => {
+      expect(res.status).to.equal(200);
+    });
+  });
+
+  it('allows goto options', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const body = {
+      gotoOptions: {
+        waitUntil: `networkidle2`,
+      },
+      url: 'https://example.com',
+    };
+
+    await fetch('http://localhost:3000/edge/content?token=browserless', {
+      body: JSON.stringify(body),
+      headers: {
+        'content-type': 'application/json',
+      },
+      method: 'POST',
+    }).then(async (res) => {
+      expect(res.status).to.equal(200);
+    });
+  });
+
+  it('allows requests without token when auth token is not set', async () => {
+    await start();
+    const body = {
+      url: 'https://example.com',
+    };
+
+    await fetch('http://localhost:3000/edge/content', {
+      body: JSON.stringify(body),
+      headers: {
+        'content-type': 'application/json',
+      },
+      method: 'POST',
+    }).then((res) => {
+      expect(res.headers.get('x-response-code')).to.not.be.undefined;
+      expect(res.headers.get('x-response-url')).to.not.be.undefined;
+      expect(res.headers.get('x-response-ip')).to.not.be.undefined;
+      expect(res.headers.get('x-response-por')).to.not.be.undefined;
+      expect(res.headers.get('content-type')).to.equal(
+        'text/html; charset=UTF-8',
+      );
+      expect(res.status).to.equal(200);
+    });
+  });
+});