@opentermsarchive/engine 16.0.0 → 16.0.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "16.0.0",
+  "version": "16.0.1",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {

package/src/archivist/fetcher/fullDomFetcher.js

@@ -1,4 +1,5 @@
-import puppeteer from 'puppeteer-extra';
+import puppeteer from 'puppeteer';
+import { addExtra } from 'puppeteer-extra';
 import stealthPlugin from 'puppeteer-extra-plugin-stealth';
 import navigatorLanguages from 'puppeteer-extra-plugin-stealth/evasions/navigator.languages/index.js';
 import userAgentOverride from 'puppeteer-extra-plugin-stealth/evasions/user-agent-override/index.js';
@@ -39,7 +40,7 @@ export default async function fetch(url, cssSelectors, config) {
     page = await context.newPage();
     client = await page.createCDPSession();
 
-    await configurePage(page, config);
+    await configurePage(page, client, config);
 
     const selectors = [].concat(cssSelectors).filter(Boolean);
 
@@ -47,7 +48,7 @@ export default async function fetch(url, cssSelectors, config) {
     let handled = null;
 
     if (!selectors.length) { // CSS selectors are specified only for HTML content and omitted when fetching a PDF
-      ({ pdf, handled } = setupPdfInterception(client));
+      ({ pdf, handled } = await setupPdfInterception(client));
     }
 
     let response;
@@ -125,13 +126,13 @@ export async function launchHeadlessBrowser(language) {
   }
 
   const { locale, languages } = parseLanguage(language);
+  const puppeteerExtra = addExtra(puppeteer); // Fresh instance for each launch, as plugins registered on the shared instance accumulate across launches and their hooks would run once per copy on every page
   const stealth = stealthPlugin();
 
   stealth.enabledEvasions.delete('user-agent-override');
   stealth.enabledEvasions.delete('navigator.languages');
-  puppeteer.use(stealth);
-  puppeteer.use(userAgentOverride({ locale }));
-  puppeteer.use(navigatorLanguages({ languages }));
+  puppeteerExtra.use(stealth);
+  puppeteerExtra.use(navigatorLanguages({ languages }));
 
   const options = {
     args: [],
@@ -156,12 +157,21 @@ export async function launchHeadlessBrowser(language) {
     options.args.push('--disable-setuid-sandbox');
   }
 
-  browser = await puppeteer.launch(options);
+  const launchedBrowser = await puppeteerExtra.launch(options);
+
+  try {
+    launchedBrowser.userAgentOverride = await captureUserAgentOverride(launchedBrowser, locale); // Computed once since the user agent is the same for every page of the browser
+  } catch (error) {
+    await launchedBrowser.close().catch(() => {});
+    throw error;
+  }
 
   if (proxyCredentials) {
-    browser.proxyCredentials = proxyCredentials;
+    launchedBrowser.proxyCredentials = proxyCredentials;
   }
 
+  browser = launchedBrowser; // Set only once fully configured, so that no caller gets a browser without its user agent override
+
   return browser;
 }
 
@@ -184,22 +194,44 @@ function isValidHttpStatus(status) {
   return (status >= 200 && status < 300) || status === 304;
 }
 
-async function configurePage(page, config) {
+async function configurePage(page, client, config) {
   await page.setViewport({ width: 1920, height: 1080 }); // Realistic viewport to avoid detection based on default Puppeteer dimensions (800x600)
   await page.setDefaultNavigationTimeout(config.navigationTimeout);
+  await client.send('Network.setUserAgentOverride', browser.userAgentOverride);
 
   if (browser.proxyCredentials?.username && browser.proxyCredentials?.password) {
     await page.authenticate(browser.proxyCredentials);
   }
 }
 
-function setupPdfInterception(client) {
+async function captureUserAgentOverride(browser, locale) { // The stealth evasion sends the override without awaiting it, so a failure would crash the process on an unhandled rejection; running it once on a stand-in page records the override for the fetcher to send itself
+  const evasion = userAgentOverride({ locale });
+  let override;
+
+  await evasion.beforeLaunch({ headless: true }); // Otherwise the evasion leaves Accept-Language to browser preferences
+  await evasion.onPageCreated({ // The stand-in page exposes only what the evasion reads
+    browser: () => browser,
+    _client: () => ({
+      send: (method, params) => {
+        if (method === 'Network.setUserAgentOverride') {
+          override = params;
+        }
+      },
+    }),
+  });
+
+  if (!override?.userAgent) {
+    throw new Error('Could not capture the user agent override from the stealth user-agent-override evasion');
+  }
+
+  return override;
+}
+
+async function setupPdfInterception(client) {
   const pdf = { content: null, status: null };
   let onHandled;
   const handled = new Promise(resolve => { onHandled = resolve; });
 
-  client.send('Fetch.enable', { patterns: [{ urlPattern: '*', requestStage: 'Response' }] }); // Intercept all responses before Chrome processes them, allowing to capture PDF content before it's handled by the PDF viewer
-
   client.on('Fetch.requestPaused', async ({ requestId, resourceType, responseHeaders, responseStatusCode }) => {
     try {
       const contentType = responseHeaders?.find(header => header.name.toLowerCase() === 'content-type')?.value;
@@ -234,6 +266,8 @@ function setupPdfInterception(client) {
     }
   });
 
+  await client.send('Fetch.enable', { patterns: [{ urlPattern: '*', requestStage: 'Response' }] }); // Intercept all responses before Chrome processes them, allowing to capture PDF content before it is handled by the PDF viewer; enabled once the listener is registered so that no paused request is missed
+
   return { pdf, handled };
 }
 

package/src/archivist/fetcher/fullDomFetcher.test.js

@@ -17,7 +17,7 @@ use(chaiAsPromised);
 const dynamicHTML = '<!DOCTYPE html><html><head><title>Dynamic Page</title><script>setTimeout(() => { document.body.innerHTML += "<div class=\'dynamic\'>Loaded</div>"; }, 100);</script></head><body></body></html>';
 const delayedContentHTML = '<!DOCTYPE html><html><head><title>Delayed Content</title><script>setTimeout(() => { document.querySelector(".content").textContent = "Final content"; }, 100);</script></head><body><div class="content"></div></body></html>';
 const langEchoHTML = '<!DOCTYPE html><html><body><script>document.body.setAttribute("data-language", navigator.language); document.body.setAttribute("data-languages", navigator.languages.join(","));</script></body></html>';
-const stealthProbeHTML = '<!DOCTYPE html><html><body><script>document.body.setAttribute("data-webdriver", String(navigator.webdriver)); document.body.setAttribute("data-user-agent", navigator.userAgent); document.body.setAttribute("data-plugin-count", String(navigator.plugins.length)); document.body.setAttribute("data-viewport-width", String(window.innerWidth)); document.body.setAttribute("data-viewport-height", String(window.innerHeight)); (() => { const canvas = document.createElement("canvas"); const gl = canvas.getContext("webgl") || canvas.getContext("experimental-webgl"); if (!gl) { document.body.setAttribute("data-webgl-vendor", "none"); return; } const ext = gl.getExtension("WEBGL_debug_renderer_info"); document.body.setAttribute("data-webgl-vendor", ext ? gl.getParameter(ext.UNMASKED_VENDOR_WEBGL) : ""); document.body.setAttribute("data-webgl-renderer", ext ? gl.getParameter(ext.UNMASKED_RENDERER_WEBGL) : ""); })();</script></body></html>';
+const stealthProbeHTML = '<!DOCTYPE html><html><body><script>document.body.setAttribute("data-webdriver", String(navigator.webdriver)); document.body.setAttribute("data-user-agent", navigator.userAgent); document.body.setAttribute("data-platform", navigator.platform); document.body.setAttribute("data-ua-brands", navigator.userAgentData ? navigator.userAgentData.brands.map(entry => entry.brand).join(",") : ""); document.body.setAttribute("data-plugin-count", String(navigator.plugins.length)); document.body.setAttribute("data-viewport-width", String(window.innerWidth)); document.body.setAttribute("data-viewport-height", String(window.innerHeight)); (() => { const canvas = document.createElement("canvas"); const gl = canvas.getContext("webgl") || canvas.getContext("experimental-webgl"); if (!gl) { document.body.setAttribute("data-webgl-vendor", "none"); return; } const ext = gl.getExtension("WEBGL_debug_renderer_info"); document.body.setAttribute("data-webgl-vendor", ext ? gl.getParameter(ext.UNMASKED_VENDOR_WEBGL) : ""); document.body.setAttribute("data-webgl-renderer", ext ? gl.getParameter(ext.UNMASKED_RENDERER_WEBGL) : ""); })();</script></body></html>';
 
 describe('Full DOM Fetcher', function () {
   this.timeout(60000);
@@ -40,6 +40,11 @@ describe('Full DOM Fetcher', function () {
 
         response.writeHead(200, { 'Content-Type': 'text/html' }).write(`<!DOCTYPE html><html><body data-accept-language="${acceptLanguage}"></body></html>`);
       }
+      if (request.url === '/client-hints-header') {
+        const clientHintsBrands = request.headers['sec-ch-ua'] || '';
+
+        response.writeHead(200, { 'Content-Type': 'text/html' }).write(`<!DOCTYPE html><html><body>${clientHintsBrands}</body></html>`);
+      }
       if (request.url === '/lang-echo') {
         response.writeHead(200, { 'Content-Type': 'text/html' }).write(langEchoHTML);
       }
@@ -76,6 +81,16 @@ describe('Full DOM Fetcher', function () {
 
       expect(browser1).to.equal(browser2);
     });
+
+    it('registers plugin hooks once per launch', async () => {
+      const initialListenerCount = (await launchHeadlessBrowser('en')).listenerCount('targetcreated');
+
+      await stopHeadlessBrowser();
+
+      const relaunchedBrowser = await launchHeadlessBrowser('en');
+
+      expect(relaunchedBrowser.listenerCount('targetcreated')).to.equal(initialListenerCount);
+    });
   });
 
   describe('#fetch', () => {
@@ -136,6 +151,25 @@ describe('Full DOM Fetcher', function () {
         expect(content.equals(expectedPDFContent)).to.be.true;
       });
     });
+
+    context('when the user agent override cannot be applied', () => {
+      let browser;
+      let userAgentOverride;
+
+      before(async () => {
+        browser = await launchHeadlessBrowser('en');
+        ({ userAgentOverride } = browser);
+        browser.userAgentOverride = {}; // Rejected by Chrome since the user agent is mandatory
+      });
+
+      after(() => {
+        browser.userAgentOverride = userAgentOverride;
+      });
+
+      it('rejects with the protocol error', async () => {
+        await expect(fetch(`http://127.0.0.1:${SERVER_PORT}/dynamic`, [], config)).to.be.rejectedWith('Network.setUserAgentOverride');
+      });
+    });
   });
 
   describe('Language configuration', () => {
@@ -201,6 +235,23 @@ describe('Full DOM Fetcher', function () {
       expect(content).not.to.match(/HeadlessChrome/);
     });
 
+    it('removes HeadlessChrome from navigator.userAgentData brands', () => {
+      expect(content).to.match(/data-ua-brands="[^"]*Google Chrome[^"]*"/);
+      expect(content).not.to.match(/data-ua-brands="[^"]*Headless[^"]*"/);
+    });
+
+    it('sends the Sec-CH-UA header without HeadlessChrome', async () => {
+      const result = await fetch(`http://127.0.0.1:${SERVER_PORT}/client-hints-header`, [], config);
+
+      expect(result.content).to.match(/"Google Chrome";v="\d+"/);
+      expect(result.content).not.to.match(/Headless/);
+    });
+
+    it('does not expose a Linux host through navigator.platform', () => {
+      expect(content).to.match(/data-platform="[^"]+"/);
+      expect(content).not.to.match(/data-platform="Linux/);
+    });
+
     it('uses a realistic viewport instead of Puppeteer default', () => {
       expect(content).to.match(/data-viewport-width="1920"/);
       expect(content).to.match(/data-viewport-height="1080"/);