@opentermsarchive/engine 10.3.0 → 10.3.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "10.3.0",
+  "version": "10.3.2",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {

package/src/archivist/extract/index.js

@@ -35,15 +35,16 @@ export async function extractFromHTML(sourceDocument) {
   const filteredDOM = await filter(webPageDOM, sourceDocument);
   const cleanedDOM = filteredDOM.remove(insignificantContentSelectors);
   const selectedDOM = cleanedDOM.select(contentSelectors);
+  const contentSelectorsDisplay = typeof contentSelectors === 'object' ? JSON.stringify(contentSelectors) : contentSelectors;
 
   if (!selectedDOM?.children.length) {
-    throw new Error(`The provided selector "${contentSelectors}" has no match in the web page at '${location}'. This could be due to elements being removed before content selection if "remove" and "select" selectors match the same content.`);
+    throw new Error(`The provided selector "${contentSelectorsDisplay}" has no match in the web page at '${location}'. This could be due to elements being removed before content selection if "remove" and "select" selectors match the same content.`);
   }
 
   const markdownContent = transformFromHTML(selectedDOM);
 
   if (!markdownContent) {
-    throw new Error(`The provided selector "${contentSelectors}" matches an empty content in the web page at '${location}'`);
+    throw new Error(`The provided selector "${contentSelectorsDisplay}" matches an empty content in the web page at '${location}'`);
   }
 
   return markdownContent;

package/src/archivist/fetcher/fullDomFetcher.js

@@ -3,13 +3,14 @@ import stealthPlugin from 'puppeteer-extra-plugin-stealth';
 
 import { resolveProxyConfiguration, extractProxyCredentials } from './proxyUtils.js';
 
-puppeteer.use(stealthPlugin());
-
 let browser;
 
 export default async function fetch(url, cssSelectors, config) {
+  puppeteer.use(stealthPlugin({ locale: config.language }));
+
   let context;
   let page;
+  let client;
   let response;
   const selectors = [].concat(cssSelectors);
 
@@ -25,6 +26,14 @@ export default async function fetch(url, cssSelectors, config) {
     await page.setDefaultNavigationTimeout(config.navigationTimeout);
     await page.setExtraHTTPHeaders({ 'Accept-Language': config.language });
 
+    // Use CDP to ensure the browser language is set correctly (most reliable method, see https://zirkelc.dev/posts/puppeteer-language-experiment)
+    client = await page.createCDPSession();
+
+    await client.send('Network.setUserAgentOverride', {
+      userAgent: await browser.userAgent(),
+      acceptLanguage: config.language,
+    });
+
     if (browser.proxyCredentials?.username && browser.proxyCredentials?.password) {
       await page.authenticate(browser.proxyCredentials);
     }
@@ -73,6 +82,9 @@ export default async function fetch(url, cssSelectors, config) {
     }
     throw new Error(error.message);
   } finally {
+    if (client) {
+      await client.detach();
+    }
     if (page) {
       await page.close();
     }