@opentermsarchive/engine 5.0.6 → 5.1.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "5.0.6",
+  "version": "5.1.1",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {

package/src/archivist/errors.js

@@ -1,9 +1,11 @@
 export class InaccessibleContentError extends Error {
-  constructor(reasonOrReasons) {
-    const reasons = [].concat(reasonOrReasons);
+  constructor(errors) {
+    const errorsArray = Array.isArray(errors) ? errors : [errors];
+    const reasons = errorsArray.map(error => (error instanceof Error ? error.message : String(error)));
 
     super(`The documents cannot be accessed or their contents can not be selected:${`\n - ${reasons.join('\n - ')}`}`);
     this.name = 'InaccessibleContentError';
     this.reasons = reasons;
+    this.errors = errorsArray;
   }
 }

package/src/archivist/fetcher/errors.js

@@ -1,6 +1,18 @@
 export class FetchDocumentError extends Error {
+  static LIKELY_TRANSIENT_ERRORS = [
+    'EAI_AGAIN', // DNS lookup temporary failure - DNS server is temporarily unavailable or overloaded
+    'ETIMEDOUT', // Connection timeout - network latency or server load issues
+    'ECONNRESET', // Connection reset - connection was forcibly closed, often due to network issues
+    'ERR_NAME_NOT_RESOLVED', // DNS lookup temporary failure - DNS server is temporarily unavailable or overloaded
+    'HTTP code 500', // Internal Server Error - server encountered an error while processing the request
+    'HTTP code 502', // Bad Gateway - upstream server returned invalid response, often temporary
+    'HTTP code 503', // Service Unavailable - server is temporarily overloaded or down for maintenance
+    'HTTP code 504', // Gateway Timeout - upstream server took too long to respond, might be temporary
+  ];
+
   constructor(message) {
     super(`Fetch failed: ${message}`);
     this.name = 'FetchDocumentError';
+    this.mayBeTransient = FetchDocumentError.LIKELY_TRANSIENT_ERRORS.some(err => message.includes(err));
   }
 }

package/src/archivist/fetcher/errors.test.js

@@ -0,0 +1,45 @@
+import { expect } from 'chai';
+
+import { FetchDocumentError } from './errors.js';
+
+describe('FetchDocumentError', () => {
+  describe('constructor', () => {
+    it('formats the error message with "Fetch failed:" prefix', () => {
+      const error = new FetchDocumentError('test error');
+
+      expect(error.message).to.equal('Fetch failed: test error');
+    });
+
+    it('sets the error name correctly', () => {
+      const error = new FetchDocumentError('test error');
+
+      expect(error.name).to.equal('FetchDocumentError');
+    });
+  });
+
+  describe('#mayBeTransient', () => {
+    describe('transient errors', () => {
+      FetchDocumentError.LIKELY_TRANSIENT_ERRORS.forEach(errorCode => {
+        it(`returns true for ${errorCode}`, () => {
+          const error = new FetchDocumentError(errorCode);
+
+          expect(error.mayBeTransient).to.be.true;
+        });
+      });
+    });
+
+    describe('non-transient errors', () => {
+      [
+        'HTTP code 403',
+        'HTTP code 404',
+        'HTTP code 429',
+      ].forEach(errorMessage => {
+        it(`returns false for "${errorMessage}"`, () => {
+          const error = new FetchDocumentError(errorMessage);
+
+          expect(error.mayBeTransient).to.be.false;
+        });
+      });
+    });
+  });
+});

package/src/archivist/fetcher/fullDomFetcher.js

@@ -21,7 +21,7 @@ export default async function fetch(url, cssSelectors, config) {
     await page.setDefaultNavigationTimeout(config.navigationTimeout);
     await page.setExtraHTTPHeaders({ 'Accept-Language': config.language });
 
-    response = await page.goto(url, { waitUntil: 'networkidle0' });
+    response = await page.goto(url, { waitUntil: 'load' }); // Using `load` instead of `networkidle0` as it's more reliable and faster. The 'load' event fires when the page and all its resources (stylesheets, scripts, images) have finished loading. `networkidle0` can be problematic as it waits for 500ms of network inactivity, which may never occur on dynamic pages and then triggers a navigation timeout.
 
     if (!response) {
       throw new Error(`Response is empty when trying to fetch '${url}'`);

package/src/archivist/index.js

@@ -29,6 +29,7 @@ export const EVENTS = [
   'trackingCompleted',
   'inaccessibleContent',
   'info',
+  'warn',
   'error',
   'pluginError',
 ];
@@ -76,15 +77,35 @@ export default class Archivist extends events.EventEmitter {
 
   initQueue() {
     this.trackingQueue = async.queue(this.trackTermsChanges.bind(this), MAX_PARALLEL_TRACKING);
-    this.trackingQueue.error((error, { terms }) => {
-      if (error instanceof InaccessibleContentError) {
-        this.emit('inaccessibleContent', error, terms);
+    this.trackingQueue.error(this.handleTrackingError.bind(this));
+  }
 
-        return;
-      }
+  handleTrackingError(error, { terms, isRetry }) {
+    if (!(error instanceof InaccessibleContentError)) {
+      this.emit('error', {
+        message: error.stack,
+        serviceId: terms.service.id,
+        termsType: terms.type,
+      });
 
-      this.emit('error', error, terms);
-    });
+      return;
+    }
+
+    const isErrorLikelyTransient = error.errors.some(err => err instanceof FetchDocumentError && err.mayBeTransient);
+
+    if (isErrorLikelyTransient && !isRetry) {
+      this.emit('warn', {
+        message: `The documents cannot be accessed due to the following likely transient errors:\n- ${error.errors.map(err => err.message).join('\n- ')}\nA new attempt will be made once the current tracking is complete`,
+        serviceId: terms.service.id,
+        termsType: terms.type,
+      });
+
+      this.trackingQueue.push({ terms, isRetry: true });
+
+      return;
+    }
+
+    this.emit('inaccessibleContent', error, terms);
   }
 
   attach(listener) {
@@ -171,7 +192,7 @@ export default class Archivist extends events.EventEmitter {
           throw error;
         }
 
-        fetchDocumentErrors.push(error.message);
+        fetchDocumentErrors.push(error);
       }
     }));
 
@@ -206,7 +227,7 @@ export default class Archivist extends events.EventEmitter {
           throw error;
         }
 
-        extractDocumentErrors.push(error.message);
+        extractDocumentErrors.push(error);
       }
     }));
 

package/src/archivist/index.test.js

@@ -8,6 +8,8 @@ import nock from 'nock';
 import sinon from 'sinon';
 import sinonChai from 'sinon-chai';
 
+import { InaccessibleContentError } from './errors.js';
+import { FetchDocumentError } from './fetcher/index.js';
 import Git from './recorder/repositories/git/git.js';
 
 import Archivist, { EVENTS } from './index.js';
@@ -245,6 +247,100 @@ describe('Archivist', function () {
     });
   });
 
+  describe('#handleTrackingError', () => {
+    let errorSpy;
+    let warnSpy;
+    let inaccessibleContentSpy;
+    let pushSpy;
+    let terms;
+    let app;
+    const retryableError = new FetchDocumentError(FetchDocumentError.LIKELY_TRANSIENT_ERRORS[0]);
+
+    before(async () => {
+      app = new Archivist({
+        recorderConfig: config.get('@opentermsarchive/engine.recorder'),
+        fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
+      });
+      await app.initialize();
+    });
+
+    beforeEach(() => {
+      errorSpy = sinon.spy();
+      warnSpy = sinon.spy();
+      inaccessibleContentSpy = sinon.spy();
+      pushSpy = sinon.spy(app.trackingQueue, 'push');
+      app.on('error', errorSpy);
+      app.on('warn', warnSpy);
+      app.on('inaccessibleContent', inaccessibleContentSpy);
+
+      terms = {
+        service: { id: 'test-service' },
+        type: 'test-type',
+        sourceDocuments: [
+          { location: 'https://example.com/doc1' },
+          { location: 'https://example.com/doc2' },
+        ],
+      };
+    });
+
+    afterEach(() => {
+      errorSpy.resetHistory();
+      warnSpy.resetHistory();
+      inaccessibleContentSpy.resetHistory();
+      pushSpy.restore();
+    });
+
+    context('with an InaccessibleContentError', () => {
+      context('when error may be transient', () => {
+        beforeEach(() => {
+          const error = new InaccessibleContentError([retryableError]);
+
+          app.handleTrackingError(error, { terms });
+        });
+
+        it('does not emit an error event', () => {
+          expect(errorSpy).to.not.have.been.called;
+        });
+
+        it('does not emit an inaccessibleContent event', () => {
+          expect(inaccessibleContentSpy).to.not.have.been.called;
+        });
+
+        it('emits a warning', () => {
+          expect(warnSpy).to.have.been.called;
+        });
+
+        it('pushes terms to tracking queue for retry', () => {
+          expect(pushSpy).to.have.been.calledWith({ terms, isRetry: true });
+        });
+      });
+
+      context('when error comes from a retry', () => {
+        beforeEach(() => {
+          const error = new InaccessibleContentError([retryableError]);
+
+          app.handleTrackingError(error, { terms, isRetry: true });
+        });
+
+        it('does not emit an error event', () => {
+          expect(errorSpy).to.not.have.been.called;
+        });
+
+        it('does not emit a warning', () => {
+          expect(warnSpy).to.not.have.been.called;
+        });
+
+        it('emits an inaccessibleContent event with error and terms', () => {
+          expect(inaccessibleContentSpy).to.have.been.called;
+        });
+
+        it('does not push terms to tracking queue for retry', () => {
+          expect(pushSpy).to.not.have.been.called;
+        });
+      });
+    });
+  });
+
   describe('Plugin system', () => {
     const plugin = {};
 

package/src/logger/index.js

@@ -141,14 +141,20 @@ logger.onInaccessibleContent = ({ message }, terms) => {
   logger.warn({ message, serviceId: terms.service.id, termsType: terms.type });
 };
 
-logger.onError = (error, terms) => {
-  logger.error({ message: error.stack, serviceId: terms.service.id, termsType: terms.type });
-};
+const createLogHandler = level => params => {
+  if (typeof params === 'string') {
+    logger[level]({ message: params });
+  } else {
+    const { serviceId, termsType, documentId, id, message } = params;
 
-logger.onInfo = message => {
-  logger.info({ message });
+    logger[level]({ message, serviceId, termsType, documentId, id });
+  }
 };
 
+logger.onError = createLogHandler('error');
+logger.onInfo = createLogHandler('info');
+logger.onWarn = createLogHandler('warn');
+
 logger.onPluginError = (error, pluginName) => {
   logger.error({ message: `Error in "${pluginName}" plugin: ${error.stack}` });
 };