@opentermsarchive/engine 5.5.0 → 5.6.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "5.5.0",
+  "version": "5.6.0",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {

package/src/archivist/fetcher/errors.js

@@ -1,18 +1,25 @@
 export class FetchDocumentError extends Error {
+  static LIKELY_BOT_BLOCKING_ERRORS = [
+    'HTTP code 403',
+    'HTTP code 406',
+    'HTTP code 502',
+    'ECONNRESET',
+  ];
+
   static LIKELY_TRANSIENT_ERRORS = [
     'EAI_AGAIN', // DNS lookup temporary failure - DNS server is temporarily unavailable or overloaded
     'ETIMEDOUT', // Connection timeout - network latency or server load issues
-    'ECONNRESET', // Connection reset - connection was forcibly closed, often due to network issues
     'ERR_NAME_NOT_RESOLVED', // DNS lookup temporary failure - DNS server is temporarily unavailable or overloaded
     'HTTP code 500', // Internal Server Error - server encountered an error while processing the request
-    'HTTP code 502', // Bad Gateway - upstream server returned invalid response, often temporary
     'HTTP code 503', // Service Unavailable - server is temporarily overloaded or down for maintenance
     'HTTP code 504', // Gateway Timeout - upstream server took too long to respond, might be temporary
+    ...FetchDocumentError.LIKELY_BOT_BLOCKING_ERRORS,
   ];
 
   constructor(message) {
     super(`Fetch failed: ${message}`);
     this.name = 'FetchDocumentError';
     this.mayBeTransient = FetchDocumentError.LIKELY_TRANSIENT_ERRORS.some(err => message.includes(err));
+    this.mayBeBotBlocking = FetchDocumentError.LIKELY_BOT_BLOCKING_ERRORS.some(err => message.includes(err));
   }
 }

package/src/archivist/fetcher/errors.test.js

@@ -30,7 +30,6 @@ describe('FetchDocumentError', () => {
 
     describe('non-transient errors', () => {
       [
-        'HTTP code 403',
         'HTTP code 404',
         'HTTP code 429',
       ].forEach(errorMessage => {

package/src/archivist/fetcher/index.js

@@ -12,13 +12,6 @@ export const FETCHER_TYPES = {
   HTML_ONLY: 'htmlOnly',
 };
 
-const LIKELY_BOT_BLOCKING_ERRORS = [
-  'HTTP code 403',
-  'HTTP code 406',
-  'HTTP code 502',
-  'ECONNRESET',
-];
-
 /**
  * Fetch a resource from the network, returning a promise which is fulfilled once the response is available
  * @function fetch
@@ -70,9 +63,7 @@ async function fetchWithFallback(url, cssSelectors, fetcherConfig) {
   try {
     return await fetchWithHtmlOnly(url, fetcherConfig);
   } catch (error) {
-    const isBotBlockingError = LIKELY_BOT_BLOCKING_ERRORS.some(code => error.message.includes(code));
-
-    if (!isBotBlockingError || fetcherConfig.executeClientScripts === false) {
+    if (!error.mayBeBotBlocking || fetcherConfig.executeClientScripts === false) {
       throw error;
     }
 
@@ -81,15 +72,23 @@ async function fetchWithFallback(url, cssSelectors, fetcherConfig) {
 }
 
 async function fetchWithFullDom(url, cssSelectors, fetcherConfig) {
-  return {
-    ...await fetchFullDom(url, cssSelectors, fetcherConfig),
-    fetcher: FETCHER_TYPES.FULL_DOM,
-  };
+  try {
+    return {
+      ...await fetchFullDom(url, cssSelectors, fetcherConfig),
+      fetcher: FETCHER_TYPES.FULL_DOM,
+    };
+  } catch (error) {
+    throw new FetchDocumentError(error.message);
+  }
 }
 
 async function fetchWithHtmlOnly(url, fetcherConfig) {
-  return {
-    ...await fetchHtmlOnly(url, fetcherConfig),
-    fetcher: FETCHER_TYPES.HTML_ONLY,
-  };
+  try {
+    return {
+      ...await fetchHtmlOnly(url, fetcherConfig),
+      fetcher: FETCHER_TYPES.HTML_ONLY,
+    };
+  } catch (error) {
+    throw new FetchDocumentError(error.message);
+  }
 }