@opentermsarchive/engine 5.6.0 → 5.7.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "5.6.0",
+  "version": "5.7.0",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {

package/src/archivist/fetcher/fullDomFetcher.js

@@ -20,6 +20,11 @@ export default async function fetch(url, cssSelectors, config) {
     await page.setDefaultNavigationTimeout(config.navigationTimeout);
     await page.setExtraHTTPHeaders({ 'Accept-Language': config.language });
 
+    await page.setCacheEnabled(false); // Disable cache to ensure fresh content on each fetch and prevent stale data from previous requests
+    const client = await page.target().createCDPSession();
+
+    await client.send('Network.clearBrowserCookies'); // Clear cookies to ensure clean state between fetches and prevent session persistence across different URLs
+
     response = await page.goto(url, { waitUntil: 'load' }); // Using `load` instead of `networkidle0` as it's more reliable and faster. The 'load' event fires when the page and all its resources (stylesheets, scripts, images) have finished loading. `networkidle0` can be problematic as it waits for 500ms of network inactivity, which may never occur on dynamic pages and then triggers a navigation timeout.
 
     if (!response) {

package/src/archivist/fetcher/index.js

@@ -48,15 +48,11 @@ export default async function fetch({
     executeClientScripts,
   };
 
-  try {
-    if (executeClientScripts) {
-      return await fetchWithFullDom(url, cssSelectors, fetcherConfig);
-    }
-
-    return await fetchWithFallback(url, cssSelectors, fetcherConfig);
-  } catch (error) {
-    throw new FetchDocumentError(error.message);
+  if (executeClientScripts) {
+    return fetchWithFullDom(url, cssSelectors, fetcherConfig);
   }
+
+  return fetchWithFallback(url, cssSelectors, fetcherConfig);
 }
 
 async function fetchWithFallback(url, cssSelectors, fetcherConfig) {

package/src/reporter/index.js

@@ -17,8 +17,8 @@ const ERROR_MESSAGE_TO_ISSUE_LABEL_MAP = {
   'HTTP code 502': '502',
   'HTTP code 503': '503',
   'Timed out after': 'timeout',
-  'getaddrinfo EAI_AGAIN': 'EAI_AGAIN',
-  'getaddrinfo ENOTFOUND': 'ENOTFOUND',
+  EAI_AGAIN: 'EAI_AGAIN',
+  ENOTFOUND: 'ENOTFOUND',
   'Response is empty': 'empty response',
   'unable to verify the first certificate': 'first certificate',
   'certificate has expired': 'certificate expired',