@browserless.io/browserless 2.36.0 → 2.37.1

package/src/routes/chromium/tests/json.spec.ts

@@ -0,0 +1,134 @@
+import { Browserless, Config, Metrics } from '@browserless.io/browserless';
+import { expect } from 'chai';
+
+describe('/json/ API', function () {
+  let browserless: Browserless;
+
+  const start = ({
+    config = new Config(),
+    metrics = new Metrics(),
+  }: { config?: Config; metrics?: Metrics } = {}) => {
+    browserless = new Browserless({ config, metrics });
+    return browserless.start();
+  };
+
+  afterEach(async () => {
+    await browserless.stop();
+  });
+
+  it('allows requests to /json/version', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const res = await fetch(
+      'http://localhost:3000/json/version?token=browserless',
+    );
+    const resJSON = await res.json();
+
+    [
+      'Browser',
+      'Protocol-Version',
+      'User-Agent',
+      'V8-Version',
+      'WebKit-Version',
+      'webSocketDebuggerUrl',
+      'Debugger-Version',
+    ].forEach((k) => expect(resJSON).to.haveOwnProperty(k));
+  });
+
+  it('rejects unauthorized requests to /json/version', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const res = await fetch(
+      'http://localhost:3000/json/version?token=imabadboi',
+    );
+    expect(res.status).to.equal(401);
+  });
+
+  it('rejects unauthorized requests to /json/list', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const res = await fetch('http://localhost:3000/json/list?token=imabadboi');
+    expect(res.status).to.equal(401);
+  });
+
+  it('allows requests to /json/protocol', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const res = await fetch(
+      'http://localhost:3000/json/protocol?token=browserless',
+    );
+    expect(res.status).to.equal(200);
+    const resJSON = await res.json();
+    expect(resJSON).to.be.an('object');
+    expect(resJSON).to.haveOwnProperty('version');
+    expect(resJSON).to.haveOwnProperty('domains');
+  });
+
+  it('rejects unauthorized requests to /json/protocol', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const res = await fetch(
+      'http://localhost:3000/json/protocol?token=imabadboi',
+    );
+    expect(res.status).to.equal(401);
+  });
+
+  it('allows requests to PUT /json/new', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const res = await fetch(
+      'http://localhost:3000/json/new?token=browserless',
+      {
+        method: 'PUT',
+      },
+    );
+    expect(res.status).to.equal(200);
+    const resJSON = await res.json();
+
+    const keys = [
+      'description',
+      'devtoolsFrontendUrl',
+      'id',
+      'title',
+      'type',
+      'url',
+      'webSocketDebuggerUrl',
+    ];
+    keys.forEach((k) => expect(resJSON).to.haveOwnProperty(k));
+
+    expect(resJSON.type).to.equal('page');
+    expect(resJSON.title).to.equal('New Tab');
+    expect(resJSON.url).to.equal('about:blank');
+    expect(resJSON.description).to.equal('');
+  });
+
+  it('rejects unauthorized requests to PUT /json/new', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const res = await fetch('http://localhost:3000/json/new?token=imabadboi', {
+      method: 'PUT',
+    });
+    expect(res.status).to.equal(401);
+  });
+});

package/src/routes/chromium/tests/scrape.spec.ts

@@ -454,7 +454,9 @@ describe('/chromium/scrape API', function () {
       expect(json.data).to.have.length(3);
 
       const aResults = json.data.find((item: any) => item.selector === 'a');
-      const nonexistentResults = json.data.find((item: any) => item.selector === 'nonexistent-element');
+      const nonexistentResults = json.data.find(
+        (item: any) => item.selector === 'nonexistent-element',
+      );
       const h1Results = json.data.find((item: any) => item.selector === 'h1');
 
       expect(aResults.results).to.be.an('array');

package/src/routes/chromium/ws/function-connect.ts

@@ -0,0 +1 @@
+export { default, QuerySchema } from '../../../shared/function-connect.ws.js';

package/src/routes/edge/tests/content.spec.ts

@@ -373,4 +373,28 @@ describe('/edge/content API', function () {
       expect(res.status).to.equal(200);
     });
   });
+
+  it('can accept insecure certs', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const body = {
+      gotoOptions: {
+        waitUntil: `networkidle2`,
+      },
+      url: 'https://self-signed.badssl.com',
+    };
+
+    await fetch('http://localhost:3000/edge/content?token=browserless&launch={"acceptInsecureCerts":true}', {
+      body: JSON.stringify(body),
+      headers: {
+        'content-type': 'application/json',
+      },
+      method: 'POST',
+    }).then(async (res) => {
+      expect(res.status).to.equal(200);
+    });
+  });
 });

package/src/routes/edge/tests/json.spec.ts

@@ -0,0 +1,134 @@
+import { Browserless, Config, Metrics } from '@browserless.io/browserless';
+import { expect } from 'chai';
+
+describe('/json/ API', function () {
+  let browserless: Browserless;
+
+  const start = ({
+    config = new Config(),
+    metrics = new Metrics(),
+  }: { config?: Config; metrics?: Metrics } = {}) => {
+    browserless = new Browserless({ config, metrics });
+    return browserless.start();
+  };
+
+  afterEach(async () => {
+    await browserless.stop();
+  });
+
+  it('allows requests to /json/version', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const res = await fetch(
+      'http://localhost:3000/json/version?token=browserless',
+    );
+    const resJSON = await res.json();
+
+    [
+      'Browser',
+      'Protocol-Version',
+      'User-Agent',
+      'V8-Version',
+      'WebKit-Version',
+      'webSocketDebuggerUrl',
+      'Debugger-Version',
+    ].forEach((k) => expect(resJSON).to.haveOwnProperty(k));
+  });
+
+  it('rejects unauthorized requests to /json/version', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const res = await fetch(
+      'http://localhost:3000/json/version?token=imabadboi',
+    );
+    expect(res.status).to.equal(401);
+  });
+
+  it('rejects unauthorized requests to /json/list', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const res = await fetch('http://localhost:3000/json/list?token=imabadboi');
+    expect(res.status).to.equal(401);
+  });
+
+  it('allows requests to /json/protocol', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const res = await fetch(
+      'http://localhost:3000/json/protocol?token=browserless',
+    );
+    expect(res.status).to.equal(200);
+    const resJSON = await res.json();
+    expect(resJSON).to.be.an('object');
+    expect(resJSON).to.haveOwnProperty('version');
+    expect(resJSON).to.haveOwnProperty('domains');
+  });
+
+  it('rejects unauthorized requests to /json/protocol', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const res = await fetch(
+      'http://localhost:3000/json/protocol?token=imabadboi',
+    );
+    expect(res.status).to.equal(401);
+  });
+
+  it('allows requests to PUT /json/new', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const res = await fetch(
+      'http://localhost:3000/json/new?token=browserless',
+      {
+        method: 'PUT',
+      },
+    );
+    expect(res.status).to.equal(200);
+    const resJSON = await res.json();
+
+    const keys = [
+      'description',
+      'devtoolsFrontendUrl',
+      'id',
+      'title',
+      'type',
+      'url',
+      'webSocketDebuggerUrl',
+    ];
+    keys.forEach((k) => expect(resJSON).to.haveOwnProperty(k));
+
+    expect(resJSON.type).to.equal('page');
+    expect(resJSON.title).to.equal('New Tab');
+    expect(resJSON.url).to.equal('about:blank');
+    expect(resJSON.description).to.equal('');
+  });
+
+  it('rejects unauthorized requests to PUT /json/new', async () => {
+    const config = new Config();
+    config.setToken('browserless');
+    const metrics = new Metrics();
+    await start({ config, metrics });
+
+    const res = await fetch('http://localhost:3000/json/new?token=imabadboi', {
+      method: 'PUT',
+    });
+    expect(res.status).to.equal(401);
+  });
+});

package/src/routes/edge/ws/function-connect.ts

@@ -0,0 +1,12 @@
+import { BrowserlessRoutes, EdgeCDP } from '@browserless.io/browserless';
+import {
+  default as FunctionConnect,
+  QuerySchema,
+} from '../../../shared/function-connect.ws.js';
+
+export default class EdgeFunctionConnectWebSocketRoute extends FunctionConnect {
+  name = BrowserlessRoutes.EdgeFunctionConnectWebSocketRoute;
+  browser = EdgeCDP;
+}
+
+export { QuerySchema };

package/src/routes/management/tests/management.spec.ts

@@ -151,6 +151,72 @@ describe('Management APIs', function () {
     });
   });
 
+  describe('Static Files Serving', () => {
+    it('serves docs pages', async () => {
+      await start();
+
+      await fetch(
+        'http://localhost:3000/docs/index.html?token=6R0W53R135510',
+      ).then(async (res) => {
+        expect(res.status).to.equal(200);
+        expect(res.headers.get('content-type')).to.equal('text/html');
+        const content = await res.text();
+        expect(content).to.include('<title>Browserless Docs</title>');
+      });
+
+      await fetch(
+        'http://localhost:3000/docs/docs.js?token=6R0W53R135510',
+      ).then(async (res) => {
+        expect(res.status).to.equal(200);
+        expect(res.headers.get('content-type')).to.equal(
+          'application/javascript',
+        );
+        const content = await res.text();
+        expect(content).to.be.an('string');
+      });
+    });
+
+    it('serves swagger.json', async () => {
+      await start();
+
+      await fetch(
+        'http://localhost:3000/docs/swagger.json?token=6R0W53R135510',
+      ).then(async (res) => {
+        expect(res.status).to.equal(200);
+        expect(res.headers.get('content-type')).to.equal('application/json');
+        const content = await res.json();
+        expect(content).to.be.an('object');
+      });
+    });
+
+    it('returns 404 if debugger is disabled', async () => {
+      process.env.ENABLE_DEBUGGER = 'false';
+      const config = new Config();
+      config.setToken('6R0W53R135510');
+      // Ensure debugger is not available
+
+      await start({ config });
+
+      await fetch(
+        'http://localhost:3000/debugger/some-file.html?token=6R0W53R135510',
+      ).then(async (res) => {
+        expect(res.status).to.equal(404);
+        expect(res.headers.get('content-type')).to.equal(
+          'text/plain; charset=UTF-8',
+        );
+      });
+    });
+
+    it('handles requests without authentication token for static files', async () => {
+      await start();
+
+      await fetch('http://localhost:3000/docs/index.html').then(async (res) => {
+        expect(res.status).to.equal(200);
+        expect(res.headers.get('content-type')).to.equal('text/html');
+      });
+    });
+  });
+
   it('allows requests to /config', async () => {
     await start();
 

package/src/sdk-utils.ts

@@ -118,7 +118,7 @@ export const prompt = async (question: string) => {
 export const installDependencies = async (
   workingDirectory: string,
 ): Promise<void> => {
-  await waitForCommand('npm clean-install', workingDirectory);
+  await waitForCommand('npm install', workingDirectory);
   await installBrowsers(workingDirectory);
 };
 

package/src/server.ts

@@ -424,7 +424,7 @@ export class HTTPServer extends EventEmitter {
 
   public async shutdown(): Promise<void> {
     this.logger.info(`HTTP Server is shutting down`);
-    await new Promise((r) => this.server.close(r));
+    await new Promise((r) => this.server?.close(r));
 
     if (this.server) {
       this.server.removeAllListeners();

package/src/shared/function-connect.ws.ts

@@ -0,0 +1,38 @@
+import {
+  APITags,
+  BrowserWebsocketRoute,
+  BrowserlessRoutes,
+  CDPLaunchOptions,
+  ChromiumCDP,
+  Logger,
+  Request,
+  SystemQueryParameters,
+  WebsocketRoutes,
+  dedent,
+} from '@browserless.io/browserless';
+import { Duplex } from 'stream';
+
+export interface QuerySchema extends SystemQueryParameters {
+  launch?: CDPLaunchOptions | string;
+}
+
+export default class ChromiumFunctionConnectWebSocketRoute extends BrowserWebsocketRoute {
+  name = BrowserlessRoutes.ChromiumFunctionConnectWebSocketRoute;
+  auth = true;
+  concurrency = false;
+  browser = ChromiumCDP;
+  description = dedent(
+    `Internally used by the POST /function API to connect the underlying client-side code to. Not intended for direct use but documented for completeness and to distinguish between other reconnect style calls.`,
+  );
+  path = WebsocketRoutes.functionClientConnect;
+  tags = [APITags.browserWS];
+  async handler(
+    req: Request,
+    socket: Duplex,
+    head: Buffer,
+    _logger: Logger,
+    browser: ChromiumCDP,
+  ): Promise<void> {
+    return browser.proxyWebSocket(req, socket, head);
+  }
+}

package/src/shared/screenshot.http.ts

@@ -63,7 +63,7 @@ export interface BodySchema {
   url?: Parameters<Page['goto']>[0];
   userAgent?: Parameters<Page['setUserAgent']>[0];
   viewport?: Parameters<Page['setViewport']>[0];
-  waitForEvent?: WaitForEventOptions;
+  waitForEvent?: WaitForEventOptions;                                                       
   waitForFunction?: WaitForFunctionOptions;
   waitForSelector?: WaitForSelectorOptions;
   waitForTimeout?: number;

package/src/shared/utils/function/handler.ts

@@ -12,7 +12,6 @@ import {
   convertIfBase64,
   exists,
   getTokenFromRequest,
-  id,
   makeExternalURL,
   mimeTypes,
 } from '@browserless.io/browserless';
@@ -63,17 +62,23 @@ export default (config: Config, logger: Logger, options: HandlerOptions = {}) =>
 
     const context = JSON.stringify(rawContext);
     const code = convertIfBase64(rawCode);
-    const browserWSEndpoint = browser.publicWSEndpoint(
-      getTokenFromRequest(req) ?? '',
-    );
+    const privateWSEndpoint = browser.wsEndpoint();
 
-    if (!browserWSEndpoint) {
+    if (!privateWSEndpoint) {
       throw new ServerError(
         `No browser endpoint was found, is the browser running?`,
       );
     }
 
-    const functionCodeJS = `browserless-function-${id()}.js`;
+    const browserID = privateWSEndpoint.split('/').pop() as string;
+    const browserWSEndpoint = makeExternalURL(
+      config.getExternalWebSocketAddress(),
+      'function',
+      'connect',
+      browserID,
+      '?token=' + getTokenFromRequest(req),
+    );
+    const functionCodeJS = `browserless-function-${browserID}.js`;
     const page = (await browser.newPage()) as UnwrapPromise<
       ReturnType<ChromiumCDP['newPage']>
     >;

package/src/shim.spec.ts

@@ -174,6 +174,24 @@ describe('Request Shimming', () => {
     });
   });
 
+  describe('insecure certs', () => {
+    it('converts ignoreHTTPSErrors to acceptInsecureCerts', () => {
+      const url = 'wss://localhost?ignoreHTTPSErrors';
+      const final = 'wss://localhost/?launch={"acceptInsecureCerts":true}';
+      const shimmed = shimLegacyRequests(new URL(url));
+
+      expect(decodeURIComponent(shimmed.href)).to.equal(final);
+    });
+
+    it('acceptInsecureCerts takes precedence over ignoreHTTPSErrors', () => {
+      const url = 'wss://localhost?ignoreHTTPSErrors&launch={"acceptInsecureCerts":false}';
+      const final = 'wss://localhost/?launch={"acceptInsecureCerts":false}';
+      const shimmed = shimLegacyRequests(new URL(url));
+
+      expect(decodeURIComponent(shimmed.href)).to.equal(final);
+    });
+  });
+
   describe('token shimming', () => {
     it('converts token query parameters to an authorization header', () => {
       const url = 'wss://localhost?token=12345';

package/src/shim.ts

@@ -5,7 +5,7 @@ import {
   safeParse,
 } from '@browserless.io/browserless';
 
-const shimParam = ['headless', 'stealth', 'ignoreDefaultArgs', 'slowMo'];
+const shimParam = ['headless', 'stealth', 'ignoreDefaultArgs', 'slowMo', 'ignoreHTTPSErrors'];
 
 /**
  * Obfuscates the ?token parameter by shifting it to a header instead of a query-parameter.
@@ -74,6 +74,20 @@ export function shimLegacyRequests(url: URL): URL {
       launchParams.ignoreHTTPSErrors = ignoreHTTPSErrors !== 'false';
     }
 
+    // When acceptInsecureCerts is set, ignoreHTTPSErrors is ignored
+    if (launchParams.acceptInsecureCerts !== undefined) {
+      launchParams.ignoreHTTPSErrors = undefined;
+    }
+
+    // When ignoreHTTPSErrors sent, convert it to acceptInsecureCerts
+    if (
+      typeof ignoreHTTPSErrors !== 'undefined' &&
+      launchParams.acceptInsecureCerts === undefined
+    ) {
+      launchParams.acceptInsecureCerts = ignoreHTTPSErrors !== 'false';
+      launchParams.ignoreHTTPSErrors = undefined;
+    }
+
     if (
       typeof ignoreDefaultArgs !== 'undefined' &&
       launchParams.ignoreDefaultArgs === undefined

package/src/types.ts

@@ -373,7 +373,9 @@ export interface CDPLaunchOptions extends BrowserlessLaunch {
   dumpio?: boolean;
   headless?: boolean | 'shell';
   ignoreDefaultArgs?: boolean | string[];
+  /** @deprecated use acceptInsecureCerts field instead */
   ignoreHTTPSErrors?: boolean;
+  acceptInsecureCerts?: boolean;
   slowMo?: number;
   stealth?: boolean;
   timeout?: number;
@@ -651,6 +653,7 @@ export const BrowserlessChromeRoutes = {
   ChromeCDPWebSocketRoute: 'ChromeCDPWebSocketRoute',
   ChromeContentPostRoute: 'ChromeContentPostRoute',
   ChromeDownloadPostRoute: 'ChromeDownloadPostRoute',
+  ChromeFunctionConnectWebSocketRoute: 'ChromeFunctionConnectWebSocketRoute',
   ChromeFunctionPostRoute: 'ChromeFunctionPostRoute',
   ChromeJSONListGetRoute: 'ChromeJSONListGetRoute',
   ChromeJSONNewPutRoute: 'ChromeJSONNewPutRoute',
@@ -669,6 +672,7 @@ export const BrowserlessEdgeRoutes = {
   EdgeCDPWebSocketRoute: 'EdgeCDPWebSocketRoute',
   EdgeContentPostRoute: 'EdgeContentPostRoute',
   EdgeDownloadPostRoute: 'EdgeDownloadPostRoute',
+  EdgeFunctionConnectWebSocketRoute: 'EdgeFunctionConnectWebSocketRoute',
   EdgeFunctionPostRoute: 'EdgeFunctionPostRoute',
   EdgeJSONListGetRoute: 'EdgeJSONListGetRoute',
   EdgeJSONNewPutRoute: 'EdgeJSONNewPutRoute',
@@ -687,6 +691,8 @@ export const BrowserlessChromiumRoutes = {
   ChromiumCDPWebSocketRoute: 'ChromiumCDPWebSocketRoute',
   ChromiumContentPostRoute: 'ChromiumContentPostRoute',
   ChromiumDownloadPostRoute: 'ChromiumDownloadPostRoute',
+  ChromiumFunctionConnectWebSocketRoute:
+    'ChromiumFunctionConnectWebSocketRoute',
   ChromiumFunctionPostRoute: 'ChromiumFunctionPostRoute',
   ChromiumJSONListGetRoute: 'ChromiumJSONListGetRoute',
   ChromiumJSONNewPutRoute: 'ChromiumJSONNewPutRoute',