@opentermsarchive/engine 12.0.2 → 13.0.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "12.0.2",
+  "version": "13.0.0",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {
@@ -92,7 +92,7 @@
     "morgan": "^1.10.1",
     "node-fetch": "^3.3.2",
     "octokit": "^5.0.3",
-    "puppeteer": "^24.22.0",
+    "puppeteer": "^25.1.0",
     "puppeteer-extra": "^3.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2",
     "simple-git": "^3.28.0",

package/scripts/declarations/validate/index.mocha.js

@@ -51,9 +51,11 @@ export default async options => {
       const filePath = path.join(declarationsPath, `${serviceId}.json`);
       const historyFilePath = path.join(declarationsPath, `${serviceId}.history.json`);
 
-      before(launchHeadlessBrowser);
+      if (!schemaOnly) {
+        before(() => launchHeadlessBrowser(config.get('@opentermsarchive/engine.fetcher.language')));
 
-      after(stopHeadlessBrowser);
+        after(stopHeadlessBrowser);
+      }
 
       context(serviceId, () => {
         before(function () {

package/src/archivist/fetcher/fullDomFetcher.js

@@ -1,13 +1,31 @@
 import puppeteer from 'puppeteer-extra';
 import stealthPlugin from 'puppeteer-extra-plugin-stealth';
+import navigatorLanguages from 'puppeteer-extra-plugin-stealth/evasions/navigator.languages/index.js';
+import userAgentOverride from 'puppeteer-extra-plugin-stealth/evasions/user-agent-override/index.js';
 
 import { resolveProxyConfiguration, extractProxyCredentials } from './proxyUtils.js';
 
 let browser;
 
-export default async function fetch(url, cssSelectors, config) {
-  puppeteer.use(stealthPlugin({ locale: config.language }));
+export function parseLanguage(value) {
+  if (typeof value !== 'string') {
+    throw new Error(`Fetcher language must be a string; received ${JSON.stringify(value)}. Pass the value of "@opentermsarchive/engine.fetcher.language" explicitly.`);
+  }
+
+  if (value.toLowerCase().includes(';q=')) {
+    throw new Error(`Quality factors are not supported in fetcher language configuration; received "${value}". Provide a comma-separated list of BCP 47 tags in priority order, for example "en-IE,en-GB,en".`);
+  }
+
+  const languages = value.split(',').map(tag => tag.trim()).filter(Boolean);
+
+  if (!languages.length) {
+    throw new Error(`Fetcher language must contain at least one tag; received "${value}".`);
+  }
 
+  return { locale: languages.join(','), languages };
+}
+
+export default async function fetch(url, cssSelectors, config) {
   if (!browser) {
     throw new Error('The headless browser should be controlled manually with "launchHeadlessBrowser" and "stopHeadlessBrowser".');
   }
@@ -21,7 +39,7 @@ export default async function fetch(url, cssSelectors, config) {
     page = await context.newPage();
     client = await page.createCDPSession();
 
-    await configurePage(page, client, config);
+    await configurePage(page, config);
 
     const selectors = [].concat(cssSelectors).filter(Boolean);
 
@@ -97,14 +115,24 @@ export default async function fetch(url, cssSelectors, config) {
 /**
  * Launches a headless browser instance using Puppeteer if one is not already running. Returns the existing browser instance if one is already running, otherwise creates and returns a new instance.
  * @function launchHeadlessBrowser
- * @returns {Promise<puppeteer.Browser>} The Puppeteer browser instance.
+ * @param   {string}                     language Accept-Language header value applied to the browser context
+ * @returns {Promise<puppeteer.Browser>}          The Puppeteer browser instance.
  * @async
  */
-export async function launchHeadlessBrowser() {
+export async function launchHeadlessBrowser(language) {
   if (browser) {
     return browser;
   }
 
+  const { locale, languages } = parseLanguage(language);
+  const stealth = stealthPlugin();
+
+  stealth.enabledEvasions.delete('user-agent-override');
+  stealth.enabledEvasions.delete('navigator.languages');
+  puppeteer.use(stealth);
+  puppeteer.use(userAgentOverride({ locale }));
+  puppeteer.use(navigatorLanguages({ languages }));
+
   const options = {
     args: [],
     headless: !process.env.OTA_ENGINE_FETCHER_NO_HEADLESS,
@@ -156,16 +184,9 @@ function isValidHttpStatus(status) {
   return (status >= 200 && status < 300) || status === 304;
 }
 
-async function configurePage(page, client, config) {
+async function configurePage(page, config) {
   await page.setViewport({ width: 1920, height: 1080 }); // Realistic viewport to avoid detection based on default Puppeteer dimensions (800x600)
   await page.setDefaultNavigationTimeout(config.navigationTimeout);
-  await page.setExtraHTTPHeaders({ 'Accept-Language': config.language });
-
-  // Use CDP to ensure browser language is set correctly (see https://zirkelc.dev/posts/puppeteer-language-experiment)
-  await client.send('Network.setUserAgentOverride', {
-    userAgent: await browser.userAgent(),
-    acceptLanguage: config.language,
-  });
 
   if (browser.proxyCredentials?.username && browser.proxyCredentials?.password) {
     await page.authenticate(browser.proxyCredentials);

package/src/archivist/fetcher/fullDomFetcher.test.js

@@ -6,7 +6,7 @@ import { fileURLToPath } from 'url';
 import { expect, use } from 'chai';
 import chaiAsPromised from 'chai-as-promised';
 
-import fetch, { launchHeadlessBrowser, stopHeadlessBrowser } from './fullDomFetcher.js';
+import fetch, { launchHeadlessBrowser, stopHeadlessBrowser, parseLanguage } from './fullDomFetcher.js';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
@@ -16,6 +16,8 @@ use(chaiAsPromised);
 
 const dynamicHTML = '<!DOCTYPE html><html><head><title>Dynamic Page</title><script>setTimeout(() => { document.body.innerHTML += "<div class=\'dynamic\'>Loaded</div>"; }, 100);</script></head><body></body></html>';
 const delayedContentHTML = '<!DOCTYPE html><html><head><title>Delayed Content</title><script>setTimeout(() => { document.querySelector(".content").textContent = "Final content"; }, 100);</script></head><body><div class="content"></div></body></html>';
+const langEchoHTML = '<!DOCTYPE html><html><body><script>document.body.setAttribute("data-language", navigator.language); document.body.setAttribute("data-languages", navigator.languages.join(","));</script></body></html>';
+const stealthProbeHTML = '<!DOCTYPE html><html><body><script>document.body.setAttribute("data-webdriver", String(navigator.webdriver)); document.body.setAttribute("data-user-agent", navigator.userAgent); document.body.setAttribute("data-plugin-count", String(navigator.plugins.length)); document.body.setAttribute("data-viewport-width", String(window.innerWidth)); document.body.setAttribute("data-viewport-height", String(window.innerHeight)); (() => { const canvas = document.createElement("canvas"); const gl = canvas.getContext("webgl") || canvas.getContext("experimental-webgl"); if (!gl) { document.body.setAttribute("data-webgl-vendor", "none"); return; } const ext = gl.getExtension("WEBGL_debug_renderer_info"); document.body.setAttribute("data-webgl-vendor", ext ? gl.getParameter(ext.UNMASKED_VENDOR_WEBGL) : ""); document.body.setAttribute("data-webgl-renderer", ext ? gl.getParameter(ext.UNMASKED_RENDERER_WEBGL) : ""); })();</script></body></html>';
 
 describe('Full DOM Fetcher', function () {
   this.timeout(60000);
@@ -24,7 +26,7 @@ describe('Full DOM Fetcher', function () {
   let expectedPDFContent;
 
   before(async () => {
-    await launchHeadlessBrowser();
+    await launchHeadlessBrowser('en');
 
     temporaryServer = http.createServer((request, response) => {
       if (request.url === '/dynamic') {
@@ -33,6 +35,17 @@ describe('Full DOM Fetcher', function () {
       if (request.url === '/delayed-content') {
         response.writeHead(200, { 'Content-Type': 'text/html' }).write(delayedContentHTML);
       }
+      if (request.url === '/lang-header') {
+        const acceptLanguage = request.headers['accept-language'] || '';
+
+        response.writeHead(200, { 'Content-Type': 'text/html' }).write(`<!DOCTYPE html><html><body data-accept-language="${acceptLanguage}"></body></html>`);
+      }
+      if (request.url === '/lang-echo') {
+        response.writeHead(200, { 'Content-Type': 'text/html' }).write(langEchoHTML);
+      }
+      if (request.url === '/stealth-probe') {
+        response.writeHead(200, { 'Content-Type': 'text/html' }).write(stealthProbeHTML);
+      }
       if (request.url === '/terms.pdf') {
         expectedPDFContent = fs.readFileSync(path.resolve(__dirname, '../../../test/fixtures/terms.pdf'));
         response.writeHead(200, { 'Content-Type': 'application/pdf' }).write(expectedPDFContent);
@@ -54,12 +67,12 @@ describe('Full DOM Fetcher', function () {
       await stopHeadlessBrowser();
       await expect(fetch('http://example.com', [], { navigationTimeout: 5000, waitForElementsTimeout: 5000, language: 'en' }))
         .to.be.rejectedWith('The headless browser should be controlled manually');
-      await launchHeadlessBrowser();
+      await launchHeadlessBrowser('en');
     });
 
     it('reuses existing browser instance', async () => {
-      const browser1 = await launchHeadlessBrowser();
-      const browser2 = await launchHeadlessBrowser();
+      const browser1 = await launchHeadlessBrowser('en');
+      const browser2 = await launchHeadlessBrowser('en');
 
       expect(browser1).to.equal(browser2);
     });
@@ -81,6 +94,12 @@ describe('Full DOM Fetcher', function () {
       await expect(fetch(url, ['.non-existent'], { ...config, navigationTimeout: timeout })).to.be.rejectedWith(`Timed out after ${timeout / 1000} seconds when trying to fetch '${url}'`);
     });
 
+    it('sends the configured language as Accept-Language header', async () => {
+      const result = await fetch(`http://127.0.0.1:${SERVER_PORT}/lang-header`, [], config);
+
+      expect(result.content).to.match(/data-accept-language="en"/);
+    });
+
     context('when a DOM element exists but its content is loaded asynchronously', () => {
       it('waits for the element content to be fully loaded', async () => {
         const result = await fetch(`http://127.0.0.1:${SERVER_PORT}/delayed-content`, ['.content'], config);
@@ -118,4 +137,145 @@ describe('Full DOM Fetcher', function () {
       });
     });
   });
+
+  describe('Language configuration', () => {
+    context('with a regional locale such as fr-FR,fr', () => {
+      const language = 'fr-FR,fr';
+      const config = { navigationTimeout: 5000, waitForElementsTimeout: 5000, language };
+
+      before(async () => {
+        await stopHeadlessBrowser();
+        await launchHeadlessBrowser(language);
+      });
+
+      it('exposes the primary tag through navigator.language', async () => {
+        const result = await fetch(`http://127.0.0.1:${SERVER_PORT}/lang-echo`, [], config);
+
+        expect(result.content).to.match(/data-language="fr-FR"/);
+      });
+
+      it('exposes every tag through navigator.languages', async () => {
+        const result = await fetch(`http://127.0.0.1:${SERVER_PORT}/lang-echo`, [], config);
+
+        expect(result.content).to.match(/data-languages="fr-FR,fr"/);
+      });
+
+      it('sends every configured tag as Accept-Language header', async () => {
+        const result = await fetch(`http://127.0.0.1:${SERVER_PORT}/lang-header`, [], config);
+
+        expect(result.content).to.match(/data-accept-language="fr-FR,fr[^"]*"/);
+      });
+    });
+
+    context('with multiple comma-separated tags such as en-IE,en-GB,en', () => {
+      const language = 'en-IE,en-GB,en';
+      const config = { navigationTimeout: 5000, waitForElementsTimeout: 5000, language };
+
+      before(async () => {
+        await stopHeadlessBrowser();
+        await launchHeadlessBrowser(language);
+      });
+
+      it('derives Accept-Language quality factors from tag order', async () => {
+        const result = await fetch(`http://127.0.0.1:${SERVER_PORT}/lang-header`, [], config);
+
+        expect(result.content).to.match(/data-accept-language="en-IE,en-GB;q=0\.9,en;q=0\.8"/);
+      });
+    });
+  });
+
+  describe('Stealth evasions', () => {
+    // These assertions guard against a class of regressions where the puppeteer-extra-plugin-stealth fails to be registered before puppeteer.launch(): if it is registered later, puppeteer-extra never binds its onPageCreated hooks and all evasions stay inactive, leaving navigator.webdriver === true and HeadlessChrome in the UA.
+    const config = { navigationTimeout: 5000, waitForElementsTimeout: 5000, language: 'en' };
+    let content;
+
+    before(async () => {
+      ({ content } = await fetch(`http://127.0.0.1:${SERVER_PORT}/stealth-probe`, [], config));
+    });
+
+    it('hides navigator.webdriver', () => {
+      expect(content).to.match(/data-webdriver="false"/);
+    });
+
+    it('removes HeadlessChrome from the user agent', () => {
+      expect(content).not.to.match(/HeadlessChrome/);
+    });
+
+    it('uses a realistic viewport instead of Puppeteer default', () => {
+      expect(content).to.match(/data-viewport-width="1920"/);
+      expect(content).to.match(/data-viewport-height="1080"/);
+    });
+
+    it('exposes a non-empty navigator.plugins list', () => {
+      const match = content.match(/data-plugin-count="(\d+)"/);
+
+      expect(match).to.not.be.null;
+      expect(Number(match[1])).to.be.greaterThan(0);
+    });
+
+    it('hides headless WebGL vendor and renderer signature', () => {
+      expect(content).to.not.match(/data-webgl-vendor="[^"]*Google[^"]*"/);
+      expect(content).to.not.match(/data-webgl-renderer="[^"]*(?:SwiftShader|ANGLE)[^"]*"/);
+    });
+  });
+});
+
+describe('parseLanguage', () => {
+  context('with a single tag', () => {
+    it('returns the tag as both locale and a singleton languages array', () => {
+      expect(parseLanguage('en')).to.deep.equal({ locale: 'en', languages: ['en'] });
+    });
+
+    it('preserves the region of a regional tag', () => {
+      expect(parseLanguage('fr-FR')).to.deep.equal({ locale: 'fr-FR', languages: ['fr-FR'] });
+    });
+  });
+
+  context('with a comma-separated priority list', () => {
+    it('splits every tag into the languages array and rejoins them as locale', () => {
+      expect(parseLanguage('en-IE,en-GB,en')).to.deep.equal({ locale: 'en-IE,en-GB,en', languages: [ 'en-IE', 'en-GB', 'en' ] });
+    });
+
+    it('trims whitespace surrounding each tag', () => {
+      expect(parseLanguage('fr-FR, fr')).to.deep.equal({ locale: 'fr-FR,fr', languages: [ 'fr-FR', 'fr' ] });
+    });
+
+    it('drops empty entries produced by consecutive commas', () => {
+      expect(parseLanguage('en,,fr')).to.deep.equal({ locale: 'en,fr', languages: [ 'en', 'fr' ] });
+    });
+
+    it('drops a trailing comma', () => {
+      expect(parseLanguage('en,fr,')).to.deep.equal({ locale: 'en,fr', languages: [ 'en', 'fr' ] });
+    });
+  });
+
+  context('with quality factors', () => {
+    it('throws for a priority list carrying quality factors', () => {
+      expect(() => parseLanguage('en-IE,en-GB;q=0.9,en;q=0.8')).to.throw('Quality factors are not supported');
+    });
+
+    it('throws for a single tag carrying a quality factor', () => {
+      expect(() => parseLanguage('en;q=1.0')).to.throw('Quality factors are not supported');
+    });
+
+    it('throws regardless of quality-factor letter case', () => {
+      expect(() => parseLanguage('en-GB;Q=0.9,en')).to.throw('Quality factors are not supported');
+    });
+  });
+
+  context('with invalid input', () => {
+    it('throws when the value is not a string', () => {
+      expect(() => parseLanguage(undefined)).to.throw('must be a string');
+      expect(() => parseLanguage(42)).to.throw('must be a string');
+    });
+
+    it('throws when the value is empty or blank', () => {
+      expect(() => parseLanguage('')).to.throw('must contain at least one tag');
+      expect(() => parseLanguage('   ')).to.throw('must contain at least one tag');
+    });
+
+    it('throws when the value contains only separators', () => {
+      expect(() => parseLanguage(',,')).to.throw('must contain at least one tag');
+    });
+  });
 });

package/src/archivist/fetcher/index.js

@@ -21,7 +21,7 @@ export const FETCHER_TYPES = {
  * @param   {string|Array}                                                             [params.cssSelectors]                  List of CSS selectors to await when loading the resource in a headless browser. Can be a CSS selector or an array of CSS selectors. Only relevant when `executeClientScripts` is enabled
  * @param   {object}                                                                   [params.config]                        Fetcher configuration
  * @param   {number}                                                                   [params.config.navigationTimeout]      Maximum time (in milliseconds) to wait before considering the fetch failed
- * @param   {string}                                                                   [params.config.language]               Language (in [ISO 639-1 format](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes)) to be passed in request headers
+ * @param   {string}                                                                   [params.config.language]               Accept-Language value applied to the browser context. Accepts a comma-separated list of [BCP 47](https://www.rfc-editor.org/rfc/rfc5646) language tags in priority order, without quality factors (for example `fr`, `en-US`, or `en-IE,en-GB,en`). The tag list drives `navigator.language` and `navigator.languages` in the headless browser, and the browser derives the `Accept-Language` HTTP header from the tag order
  * @param   {number}                                                                   [params.config.waitForElementsTimeout] Maximum time (in milliseconds) to wait for selectors to exist on page before considering the fetch failed. Only relevant when `executeClientScripts` is enabled
  * @returns {Promise<{ mimeType: string, content: string | Buffer, fetcher: string }>}                                        Promise containing the fetched resource's MIME type, content, and fetcher type
  * @throws {FetchDocumentError} When the fetch operation fails

package/src/archivist/fetcher/index.test.js

@@ -21,7 +21,7 @@ const termsWithOtherCharsetHTML = '<!DOCTYPE html><html><head><meta http-equiv="
 describe('Fetcher', function () {
   this.timeout(60000);
 
-  before(launchHeadlessBrowser);
+  before(() => launchHeadlessBrowser('en'));
 
   after(stopHeadlessBrowser);
 

package/src/archivist/index.js

@@ -45,6 +45,7 @@ export default class Archivist extends events.EventEmitter {
 
   constructor({ recorderConfig, fetcherConfig }) {
     super();
+    this.fetcherConfig = fetcherConfig;
     this.recorder = new Recorder(recorderConfig);
     this.fetch = params => fetch({ ...params, config: fetcherConfig });
     this.extract = extract;
@@ -151,7 +152,7 @@ export default class Archivist extends events.EventEmitter {
 
     this.emit('trackingStarted', servicesIds.length, numberOfTerms, technicalUpgradeOnly);
 
-    await Promise.all([ launchHeadlessBrowser(), this.recorder.initialize() ]);
+    await Promise.all([ launchHeadlessBrowser(this.fetcherConfig.language), this.recorder.initialize() ]);
 
     this.trackingQueue.concurrency = concurrency;