@opentermsarchive/engine 1.1.3 → 1.2.0

package/src/archivist/fetcher/errors.js

@@ -1,6 +1,6 @@
 export class FetchDocumentError extends Error {
   constructor(message) {
-    super(message);
+    super(`Fetch failed: ${message}`);
     this.name = 'FetchDocumentError';
   }
 }

package/src/archivist/fetcher/fullDomFetcher.js

@@ -2,8 +2,6 @@ import puppeteer from 'puppeteer';
 import puppeteerExtra from 'puppeteer-extra';
 import stealthPlugin from 'puppeteer-extra-plugin-stealth';
 
-import { FetchDocumentError } from './errors.js';
-
 puppeteerExtra.use(stealthPlugin());
 
 let browser;
@@ -26,13 +24,13 @@ export default async function fetch(url, cssSelectors, config) {
     response = await page.goto(url, { waitUntil: 'networkidle0' });
 
     if (!response) {
-      throw new FetchDocumentError(`Response is empty when trying to fetch '${url}'`);
+      throw new Error(`Response is empty when trying to fetch '${url}'`);
     }
 
     const statusCode = response.status();
 
     if (statusCode < 200 || (statusCode >= 300 && statusCode !== 304)) {
-      throw new FetchDocumentError(`Received HTTP code ${statusCode} when trying to fetch '${url}'`);
+      throw new Error(`Received HTTP code ${statusCode} when trying to fetch '${url}'`);
     }
 
     const waitForSelectorsPromises = selectors.map(selector => page.waitForSelector(selector, { timeout: config.waitForElementsTimeout }));
@@ -54,9 +52,9 @@ export default async function fetch(url, cssSelectors, config) {
     };
   } catch (error) {
     if (error instanceof puppeteer.errors.TimeoutError) {
-      throw new FetchDocumentError(`Timed out after ${config.navigationTimeout / 1000} seconds when trying to fetch '${url}'`);
+      throw new Error(`Timed out after ${config.navigationTimeout / 1000} seconds when trying to fetch '${url}'`);
     }
-    throw new FetchDocumentError(error.message);
+    throw new Error(error.message);
   } finally {
     if (page) {
       await page.close();

package/src/archivist/fetcher/htmlOnlyFetcher.js

@@ -5,8 +5,6 @@ import HttpProxyAgent from 'http-proxy-agent';
 import HttpsProxyAgent from 'https-proxy-agent';
 import nodeFetch, { AbortError } from 'node-fetch';
 
-import { FetchDocumentError } from './errors.js';
-
 export default async function fetch(url, configuration) {
   const controller = new AbortController();
   const timeout = setTimeout(() => controller.abort(), configuration.navigationTimeout);
@@ -29,10 +27,11 @@ export default async function fetch(url, configuration) {
     response = await nodeFetch(url, nodeFetchOptions);
 
     if (!response.ok) {
-      throw new FetchDocumentError(`Received HTTP code ${response.status} when trying to fetch '${url}'`);
+      throw new Error(`Received HTTP code ${response.status} when trying to fetch '${url}'`);
     }
 
     const mimeType = response.headers.get('content-type');
+    const contentLength = response.headers.get('content-length');
     const responseBuffer = await response.arrayBuffer();
     let content;
 
@@ -42,8 +41,8 @@ export default async function fetch(url, configuration) {
       content = Buffer.from(responseBuffer);
     }
 
-    if (!content) {
-      throw new FetchDocumentError(`Received an empty content when fetching '${url}'`);
+    if (contentLength == 0 || !content) {
+      throw new Error(`Received an empty content when fetching '${url}'`);
     }
 
     return {
@@ -52,10 +51,10 @@ export default async function fetch(url, configuration) {
     };
   } catch (error) {
     if (error instanceof AbortError) {
-      throw new FetchDocumentError(`Timed out after ${configuration.navigationTimeout / 1000} seconds when trying to fetch '${url}'`);
+      throw new Error(`Timed out after ${configuration.navigationTimeout / 1000} seconds when trying to fetch '${url}'`);
     }
 
-    throw new FetchDocumentError(error.message);
+    throw new Error(error.message);
   } finally {
     clearTimeout(timeout);
   }

package/src/archivist/fetcher/index.js

@@ -1,5 +1,6 @@
 import config from 'config';
 
+import { FetchDocumentError } from './errors.js';
 import fetchFullDom from './fullDomFetcher.js';
 import fetchHtmlOnly from './htmlOnlyFetcher.js';
 
@@ -28,9 +29,13 @@ export default async function fetch({
     waitForElementsTimeout = config.get('fetcher.waitForElementsTimeout'),
   } = {},
 }) {
-  if (executeClientScripts) {
-    return fetchFullDom(url, cssSelectors, { navigationTimeout, language, waitForElementsTimeout });
-  }
+  try {
+    if (executeClientScripts) {
+      return await fetchFullDom(url, cssSelectors, { navigationTimeout, language, waitForElementsTimeout });
+    }
 
-  return fetchHtmlOnly(url, { navigationTimeout, language });
+    return await fetchHtmlOnly(url, { navigationTimeout, language });
+  } catch (error) {
+    throw new FetchDocumentError(error.message);
+  }
 }

package/src/archivist/fetcher/index.test.js

@@ -41,6 +41,9 @@ describe('Fetcher', function () {
         if (request.url == '/404') {
           response.writeHead(404, { 'Content-Type': 'text/html' }).write('<!DOCTYPE html><html><body>404</body></html>');
         }
+        if (request.url === '/zero-content') {
+          response.writeHead(200, { 'Content-Type': 'text/html', 'Content-Length': '0' }).write('');
+        }
         if (request.url == '/terms.pdf') {
           expectedPDFContent = fs.readFileSync(path.resolve(__dirname, '../../../test/fixtures/terms.pdf'));
 
@@ -70,11 +73,11 @@ describe('Fetcher', function () {
             ({ content, mimeType } = await fetch({ url, selectors: 'body' }));
           });
 
-          it('returns the web page content of the given URL', async () => {
+          it('returns the web page content of the given URL', () => {
             expect(content).to.equal(termsHTML);
           });
 
-          it('returns the MIME type of the given URL', async () => {
+          it('returns the MIME type of the given URL', () => {
             expect(mimeType).to.equal('text/html');
           });
 
@@ -83,11 +86,11 @@ describe('Fetcher', function () {
               ({ content, mimeType } = await fetch({ url, selectors: 'body', executeClientScripts: true }));
             });
 
-            it('returns the web page content of the given URL', async () => {
+            it('returns the web page content of the given URL', () => {
               expect(content).to.equal(termsHTML);
             });
 
-            it('returns the MIME type of the given URL', async () => {
+            it('returns the MIME type of the given URL', () => {
               expect(mimeType).to.equal('text/html');
             });
           });
@@ -100,11 +103,11 @@ describe('Fetcher', function () {
             ({ content, mimeType } = await fetch({ url, selectors: NOT_PRESENT_SELECTOR }));
           });
 
-          it('returns the web page content of the given URL', async () => {
+          it('returns the web page content of the given URL', () => {
             expect(content).to.equal(termsHTML);
           });
 
-          it('returns the MIME type of the given URL', async () => {
+          it('returns the MIME type of the given URL', () => {
             expect(mimeType).to.equal('text/html');
           });
 
@@ -113,11 +116,11 @@ describe('Fetcher', function () {
               ({ content, mimeType } = await fetch({ url, selectors: NOT_PRESENT_SELECTOR, executeClientScripts: true }));
             });
 
-            it('returns the web page content of the given URL', async () => {
+            it('returns the web page content of the given URL', () => {
               expect(content).to.equal(termsHTML);
             });
 
-            it('returns the MIME type of the given URL', async () => {
+            it('returns the MIME type of the given URL', () => {
               expect(mimeType).to.equal('text/html');
             });
           });
@@ -133,7 +136,7 @@ describe('Fetcher', function () {
             ({ content } = await fetch({ url, selectors: 'body' }));
           });
 
-          it('returns the web page content of the given URL', async () => {
+          it('returns the web page content of the given URL', () => {
             expect(content).to.equal(termsWithOtherCharsetHTML);
           });
         });
@@ -148,18 +151,26 @@ describe('Fetcher', function () {
           ({ content, mimeType } = await fetch({ url: pdfUrl }));
         });
 
-        it('returns a buffer for PDF content', async () => {
+        it('returns a buffer for PDF content', () => {
           expect(content).to.be.an.instanceOf(Buffer);
         });
 
-        it('returns a blob with the file type', async () => {
+        it('returns a blob with the file type', () => {
           expect(mimeType).to.equal('application/pdf');
         });
 
-        it('returns a blob with the file content', async () => {
+        it('returns a blob with the file content', () => {
           expect(content.equals(expectedPDFContent)).to.be.true;
         });
       });
+
+      context('when server responds with empty content', () => {
+        const zeroContentUrl = `http://127.0.0.1:${SERVER_PORT}/zero-content`;
+
+        it('throws a FetchDocumentError error', async () => {
+          await expect(fetch({ url: zeroContentUrl })).to.be.rejectedWith(FetchDocumentError, /empty content/);
+        });
+      });
     });
 
     describe('Error handling', () => {

package/src/archivist/index.js

@@ -3,7 +3,7 @@ import events from 'events';
 import async from 'async';
 
 import { InaccessibleContentError } from './errors.js';
-import extract from './extract/index.js';
+import extract, { ExtractDocumentError } from './extract/index.js';
 import fetch, { launchHeadlessBrowser, stopHeadlessBrowser, FetchDocumentError } from './fetcher/index.js';
 import Recorder from './recorder/index.js';
 import Snapshot from './recorder/snapshot.js';
@@ -72,7 +72,7 @@ export default class Archivist extends events.EventEmitter {
 
   initQueue() {
     this.trackingQueue = async.queue(this.trackTermsChanges.bind(this), MAX_PARALLEL_TRACKING);
-    this.trackingQueue.error(async (error, { terms }) => {
+    this.trackingQueue.error((error, { terms }) => {
       if (error instanceof InaccessibleContentError) {
         this.emit('inaccessibleContent', error, terms);
 
@@ -99,8 +99,11 @@ export default class Archivist extends events.EventEmitter {
     });
   }
 
-  async track({ services: servicesIds = this.servicesIds, terms: termsTypes = [], extractOnly = false }) {
-    this.emit('trackingStarted', servicesIds.length, Service.getNumberOfTerms(this.services, servicesIds), extractOnly);
+  async track({ services: servicesIds = this.servicesIds, types: termsTypes = [], extractOnly = false } = {}) {
+    const numberOfTerms = Service.getNumberOfTerms(this.services, servicesIds, termsTypes);
+
+    this.emit('trackingStarted', servicesIds.length, numberOfTerms, extractOnly);
+
     await Promise.all([ launchHeadlessBrowser(), this.recorder.initialize() ]);
 
     this.trackingQueue.concurrency = extractOnly ? MAX_PARALLEL_EXTRACTING : MAX_PARALLEL_TRACKING;
@@ -120,7 +123,8 @@ export default class Archivist extends events.EventEmitter {
     }
 
     await Promise.all([ stopHeadlessBrowser(), this.recorder.finalize() ]);
-    this.emit('trackingCompleted', servicesIds.length, Service.getNumberOfTerms(this.services, servicesIds), extractOnly);
+
+    this.emit('trackingCompleted', servicesIds.length, numberOfTerms, extractOnly);
   }
 
   async trackTermsChanges({ terms, extractOnly = false }) {
@@ -142,7 +146,7 @@ export default class Archivist extends events.EventEmitter {
   async fetchSourceDocuments(terms) {
     terms.fetchDate = new Date();
 
-    const inaccessibleContentErrors = [];
+    const fetchDocumentErrors = [];
 
     await Promise.all(terms.sourceDocuments.map(async sourceDocument => {
       const { location: url, executeClientScripts, cssSelectors } = sourceDocument;
@@ -157,16 +161,16 @@ export default class Archivist extends events.EventEmitter {
           throw error;
         }
 
-        inaccessibleContentErrors.push(error.message);
+        fetchDocumentErrors.push(error.message);
       }
     }));
 
-    if (inaccessibleContentErrors.length) {
-      throw new InaccessibleContentError(inaccessibleContentErrors);
+    if (fetchDocumentErrors.length) {
+      throw new InaccessibleContentError(fetchDocumentErrors);
     }
   }
 
-  async loadSourceDocumentsFromSnapshots(terms) {
+  loadSourceDocumentsFromSnapshots(terms) {
     return Promise.all(terms.sourceDocuments.map(async sourceDocument => {
       const snapshot = await this.recorder.getLatestSnapshot(terms, sourceDocument.id);
 
@@ -182,12 +186,32 @@ export default class Archivist extends events.EventEmitter {
   }
 
   async extractVersionContent(sourceDocuments) {
-    return (await Promise.all(sourceDocuments.map(async sourceDocument => this.extract(sourceDocument)))).join(Version.SOURCE_DOCUMENTS_SEPARATOR);
+    const extractDocumentErrors = [];
+
+    const result = await Promise.all(sourceDocuments.map(async sourceDocument => {
+      try {
+        return await this.extract(sourceDocument);
+      } catch (error) {
+        if (!(error instanceof ExtractDocumentError)) {
+          throw error;
+        }
+
+        extractDocumentErrors.push(error.message);
+      }
+    }));
+
+    if (extractDocumentErrors.length) {
+      throw new InaccessibleContentError(extractDocumentErrors);
+    }
+
+    return result.join(Version.SOURCE_DOCUMENTS_SEPARATOR);
   }
 
   async recordVersion(terms, extractOnly) {
+    const content = await this.extractVersionContent(terms.sourceDocuments);
+
     const record = new Version({
-      content: await this.extractVersionContent(terms.sourceDocuments),
+      content,
       snapshotIds: terms.sourceDocuments.map(sourceDocuments => sourceDocuments.snapshotId),
       serviceId: terms.service.id,
       termsType: terms.type,
@@ -208,7 +232,7 @@ export default class Archivist extends events.EventEmitter {
     return record;
   }
 
-  async recordSnapshots(terms) {
+  recordSnapshots(terms) {
     return Promise.all(terms.sourceDocuments.map(async sourceDocument => {
       const record = new Snapshot({
         serviceId: terms.service.id,

package/src/archivist/index.test.js

@@ -390,7 +390,7 @@ describe('Archivist', function () {
           before(async () => {
             await app.recordSnapshots(terms);
             resetSpiesHistory();
-            terms.sourceDocuments.forEach(async sourceDocument => {
+            terms.sourceDocuments.forEach(sourceDocument => {
               sourceDocument.content = serviceBSnapshotExpectedContent;
             });
             [changedSnapshot] = await app.recordSnapshots(terms);
@@ -402,7 +402,7 @@ describe('Archivist', function () {
             return resetGitRepositories();
           });
 
-          it('emits "snapshotRecorded" event', async () => {
+          it('emits "snapshotRecorded" event', () => {
             expect(spies.onSnapshotRecorded).to.have.been.calledWith(changedSnapshot);
           });
 
@@ -437,10 +437,10 @@ describe('Archivist', function () {
       let terms;
       let version;
 
-      before(async () => {
+      before(() => {
         terms = app.services.service·A.getTerms({ type: SERVICE_A_TYPE });
         terms.fetchDate = FETCH_DATE;
-        terms.sourceDocuments.forEach(async sourceDocument => {
+        terms.sourceDocuments.forEach(sourceDocument => {
           sourceDocument.content = serviceASnapshotExpectedContent;
           sourceDocument.mimeType = MIME_TYPE;
         });
@@ -458,7 +458,7 @@ describe('Archivist', function () {
           return resetGitRepositories();
         });
 
-        it('emits "firstVersionRecorded" event', async () => {
+        it('emits "firstVersionRecorded" event', () => {
           expect(spies.onFirstVersionRecorded).to.have.been.calledWith(version);
         });
 
@@ -529,23 +529,23 @@ describe('Archivist', function () {
         return resetGitRepositories();
       });
 
-      it('emits "trackingStarted" event', async () => {
+      it('emits "trackingStarted" event', () => {
         expect(spies.onTrackingStarted).to.have.been.calledOnce;
       });
 
-      it('emits "firstSnapshotRecorded" events', async () => {
+      it('emits "firstSnapshotRecorded" events', () => {
         expect(spies.onFirstSnapshotRecorded).to.have.been.calledTwice;
       });
 
-      it('emits "firstVersionRecorded" events', async () => {
+      it('emits "firstVersionRecorded" events', () => {
         expect(spies.onFirstVersionRecorded).to.have.been.calledTwice;
       });
 
-      it('emits "firstVersionRecorded" events after "firstSnapshotRecorded" events', async () => {
+      it('emits "firstVersionRecorded" events after "firstSnapshotRecorded" events', () => {
         expect(spies.onFirstVersionRecorded).to.have.been.calledAfter(spies.onFirstSnapshotRecorded);
       });
 
-      it('emits "trackingCompleted" event', async () => {
+      it('emits "trackingCompleted" event', () => {
         expect(spies.onTrackingCompleted).to.have.been.calledAfter(spies.onTrackingStarted);
       });
 
@@ -579,23 +579,23 @@ describe('Archivist', function () {
           return resetGitRepositories();
         });
 
-        it('emits "trackingStarted" event', async () => {
+        it('emits "trackingStarted" event', () => {
           expect(spies.onTrackingStarted).to.have.been.calledOnce;
         });
 
-        it('emits "snapshotNotChanged" events', async () => {
+        it('emits "snapshotNotChanged" events', () => {
           expect(spies.onSnapshotNotChanged).to.have.been.calledTwice;
         });
 
-        it('emits "versionNotChanged" events', async () => {
+        it('emits "versionNotChanged" events', () => {
           expect(spies.onVersionNotChanged).to.have.been.calledTwice;
         });
 
-        it('emits "versionNotChanged" events after "snapshotRecorded" events', async () => {
+        it('emits "versionNotChanged" events after "snapshotRecorded" events', () => {
           expect(spies.onVersionNotChanged).to.have.been.calledAfter(spies.onSnapshotNotChanged);
         });
 
-        it('emits "trackingCompleted" event', async () => {
+        it('emits "trackingCompleted" event', () => {
           expect(spies.onTrackingCompleted).to.have.been.calledAfter(spies.onTrackingStarted);
         });
 
@@ -644,31 +644,31 @@ describe('Archivist', function () {
           return resetGitRepositories();
         });
 
-        it('emits "trackingStarted" event', async () => {
+        it('emits "trackingStarted" event', () => {
           expect(spies.onTrackingStarted).to.have.been.calledOnce;
         });
 
-        it('emits "snapshotNotChanged" event for the service that was not changed', async () => {
+        it('emits "snapshotNotChanged" event for the service that was not changed', () => {
           expect(spies.onSnapshotNotChanged).to.have.been.calledOnceWith(snapshotB);
         });
 
-        it('emits "snapshotRecorded" event for the service that was changed', async () => {
+        it('emits "snapshotRecorded" event for the service that was changed', () => {
           expect(spies.onSnapshotRecorded).to.have.been.calledOnceWith(snapshotA);
         });
 
-        it('emits "versionNotChanged" events for the service that was not changed', async () => {
+        it('emits "versionNotChanged" events for the service that was not changed', () => {
           expect(spies.onVersionNotChanged).to.have.been.calledOnceWith(versionB);
         });
 
-        it('emits "versionRecorded" event for the service that was changed', async () => {
+        it('emits "versionRecorded" event for the service that was changed', () => {
           expect(spies.onVersionRecorded).to.have.been.calledOnceWith(versionA);
         });
 
-        it('emits "snapshotRecorded" events after "versionRecorded" events', async () => {
+        it('emits "snapshotRecorded" events after "versionRecorded" events', () => {
           expect(spies.onVersionRecorded).to.have.been.calledAfter(spies.onSnapshotRecorded);
         });
 
-        it('emits "trackingCompleted" event', async () => {
+        it('emits "trackingCompleted" event', () => {
           expect(spies.onTrackingCompleted).to.have.been.calledAfter(spies.onTrackingStarted);
         });
 

package/src/archivist/services/service.js

@@ -28,8 +28,14 @@ export default class Service {
     return history?.find(entry => new Date(date) <= new Date(entry.validUntil)) || currentlyValidTerms;
   }
 
-  getTermsTypes() {
-    return Object.keys(this.terms);
+  getTermsTypes(termsTypes) {
+    let result = Object.keys(this.terms);
+
+    if (termsTypes) {
+      result = result.filter(item => termsTypes.includes(item));
+    }
+
+    return result;
   }
 
   addTerms(terms) {
@@ -54,8 +60,8 @@ export default class Service {
     return this.terms[termsType].history.map(entry => entry.validUntil);
   }
 
-  getNumberOfTerms() {
-    return this.getTermsTypes().length;
+  getNumberOfTerms(termsTypes) {
+    return this.getTermsTypes(termsTypes).length;
   }
 
   hasHistory() {
@@ -63,7 +69,7 @@ export default class Service {
     return Boolean(Object.keys(this.terms).find(termsType => this.terms[termsType].history));
   }
 
-  static getNumberOfTerms(services, servicesIds) {
-    return servicesIds.reduce((acc, serviceId) => acc + services[serviceId].getNumberOfTerms(), 0);
+  static getNumberOfTerms(services, servicesIds, termsTypes) {
+    return servicesIds.reduce((acc, serviceId) => acc + services[serviceId].getNumberOfTerms(termsTypes), 0);
   }
 }