@opentermsarchive/engine 0.26.1 → 0.27.1

package/src/archivist/services/service.js

@@ -1,60 +1,64 @@
 export default class Service {
-  documents = new Map();
+  terms = new Map();
 
   constructor({ id, name }) {
     this.id = id;
     this.name = name;
   }
 
-  getDocumentDeclaration(documentType, date) {
-    if (!this.documents[documentType]) {
+  getTerms(termsType, date) {
+    if (!this.terms[termsType]) {
       return null;
     }
 
-    const { latest: currentlyValidDocumentDeclaration, history } = this.documents[documentType];
+    const { latest: currentlyValidTerms, history } = this.terms[termsType];
 
     if (!date) {
-      return currentlyValidDocumentDeclaration;
+      return currentlyValidTerms;
     }
 
     return (
       history?.find(entry => new Date(date) <= new Date(entry.validUntil))
-      || currentlyValidDocumentDeclaration
+      || currentlyValidTerms
     );
   }
 
-  getDocumentTypes() {
-    return Object.keys(this.documents);
+  getTermsTypes() {
+    return Object.keys(this.terms);
   }
 
-  addDocumentDeclaration(document) {
-    if (!document.service) {
-      document.service = this;
+  addTerms(terms) {
+    if (!terms.service) {
+      terms.service = this;
     }
 
-    this.documents[document.type] = this.documents[document.type] || {};
+    this.terms[terms.type] = this.terms[terms.type] || {};
 
-    if (!document.validUntil) {
-      this.documents[document.type].latest = document;
+    if (!terms.validUntil) {
+      this.terms[terms.type].latest = terms;
 
       return;
     }
 
-    this.documents[document.type].history = this.documents[document.type].history || [];
-    this.documents[document.type].history.push(document);
-    this.documents[document.type].history.sort((a, b) => new Date(a.validUntil) - new Date(b.validUntil));
+    this.terms[terms.type].history = this.terms[terms.type].history || [];
+    this.terms[terms.type].history.push(terms);
+    this.terms[terms.type].history.sort((a, b) => new Date(a.validUntil) - new Date(b.validUntil));
   }
 
-  getHistoryDates(documentType) {
-    return this.documents[documentType].history.map(entry => entry.validUntil);
+  getHistoryDates(termsType) {
+    return this.terms[termsType].history.map(entry => entry.validUntil);
   }
 
-  getNumberOfDocuments() {
-    return this.getDocumentTypes().length;
+  getNumberOfTerms() {
+    return this.getTermsTypes().length;
   }
 
   hasHistory() {
     // If a service is loaded without its history it could return false even if a history declaration file exists.
-    return Boolean(Object.keys(this.documents).find(documentType => this.documents[documentType].history));
+    return Boolean(Object.keys(this.terms).find(termsType => this.terms[termsType].history));
+  }
+
+  static getNumberOfTerms(services, servicesIds) {
+    return servicesIds.reduce((acc, serviceId) => acc + services[serviceId].getNumberOfTerms(), 0);
   }
 }

package/src/archivist/services/service.test.js

@@ -1,135 +1,119 @@
 import chai from 'chai';
 
-import DocumentDeclaration from './documentDeclaration.js';
 import Service from './service.js';
+import Terms from './terms.js';
 
 const { expect } = chai;
 
 describe('Service', () => {
   let subject;
-  const DOCUMENT_TYPE = 'Terms of Service';
+  const TERMS_TYPE = 'Terms of Service';
 
-  describe('#addDocumentDeclaration', () => {
-    let documentDeclaration;
+  describe('#addTerms', () => {
+    let terms;
 
     before(async () => {
-      documentDeclaration = new DocumentDeclaration({
-        type: DOCUMENT_TYPE,
+      terms = new Terms({
+        type: TERMS_TYPE,
         service: subject,
-        pages: [{
-          location: 'https://www.service.example/tos',
-          contentSelectors: 'body',
-        }],
       });
     });
 
-    context('when document declaration has no validity date', () => {
+    context('when terms declaration has no validity date', () => {
       before(async () => {
         subject = new Service({ id: 'serviceID', name: 'serviceName' });
-        subject.addDocumentDeclaration(documentDeclaration);
+        subject.addTerms(terms);
       });
 
-      it('adds the document as the last valid document declaration', async () => {
-        expect(subject.getDocumentDeclaration(DOCUMENT_TYPE)).to.deep.eql(documentDeclaration);
+      it('adds the terms as the last valid terms declaration', async () => {
+        expect(subject.getTerms(TERMS_TYPE)).to.deep.eql(terms);
       });
     });
 
-    context('when document declaration has a validity date', () => {
-      let expiredDocumentDeclaration;
+    context('when terms declaration has a validity date', () => {
+      let expiredTerms;
       const VALIDITY_DATE = new Date('2020-07-22T11:30:21.000Z');
 
       before(async () => {
         subject = new Service({ id: 'serviceID', name: 'serviceName' });
-        expiredDocumentDeclaration = new DocumentDeclaration({
-          type: 'Terms of Service',
+        expiredTerms = new Terms({
+          type: TERMS_TYPE,
           service: subject,
           validUntil: VALIDITY_DATE,
-          pages: [{
-            location: 'https://www.service.example/terms',
-            contentSelectors: 'main',
-          }],
         });
-        subject.addDocumentDeclaration(expiredDocumentDeclaration);
-        subject.addDocumentDeclaration(documentDeclaration);
+        subject.addTerms(expiredTerms);
+        subject.addTerms(terms);
       });
 
-      it('adds the document with the proper validity date', async () => {
-        expect(subject.getDocumentDeclaration(DOCUMENT_TYPE, VALIDITY_DATE)).to.deep.eql(expiredDocumentDeclaration);
+      it('adds the terms with the proper validity date', async () => {
+        expect(subject.getTerms(TERMS_TYPE, VALIDITY_DATE)).to.deep.eql(expiredTerms);
       });
     });
   });
 
-  describe('#getDocumentDeclaration', () => {
+  describe('#getTerms', () => {
     let subject;
 
-    const lastDeclaration = new DocumentDeclaration({
-      type: 'Terms of Service',
-      location: 'https://www.service.example/tos',
-      contentSelectors: 'body',
-    });
+    const lastDeclaration = new Terms({ type: TERMS_TYPE });
 
     context('when there is no history', () => {
       before(async () => {
         subject = new Service({ id: 'serviceID', name: 'serviceName' });
-        subject.addDocumentDeclaration(lastDeclaration);
+        subject.addTerms(lastDeclaration);
       });
 
       context('without given date', () => {
-        it('returns the last document declaration', async () => {
-          expect(subject.getDocumentDeclaration(DOCUMENT_TYPE)).to.eql(lastDeclaration);
+        it('returns the last terms declaration', async () => {
+          expect(subject.getTerms(TERMS_TYPE)).to.eql(lastDeclaration);
         });
       });
 
       context('with a date', () => {
-        it('returns the last document declaration', async () => {
-          expect(subject.getDocumentDeclaration(DOCUMENT_TYPE, '2020-08-21T11:30:21.000Z')).to.eql(lastDeclaration);
+        it('returns the last terms declaration', async () => {
+          expect(subject.getTerms(TERMS_TYPE, '2020-08-21T11:30:21.000Z')).to.eql(lastDeclaration);
         });
       });
     });
 
-    context('when the document has a history', () => {
-      const firstDeclaration = new DocumentDeclaration({
-        type: 'Terms of Service',
-        location: 'https://www.service.example/terms',
-        contentSelectors: 'main',
+    context('when the terms have a history', () => {
+      const firstDeclaration = new Terms({
+        type: TERMS_TYPE,
         validUntil: '2020-07-22T11:30:21.000Z',
       });
 
-      const secondDeclaration = new DocumentDeclaration({
-        type: 'Terms of Service',
-        location: 'https://www.service.example/terms-of-service',
-        contentSelectors: 'main',
+      const secondDeclaration = new Terms({
+        type: TERMS_TYPE,
         validUntil: '2020-08-22T11:30:21.000Z',
       });
 
       before(async () => {
         subject = new Service({ id: 'serviceID', name: 'serviceName' });
-        subject.addDocumentDeclaration(lastDeclaration);
-        subject.addDocumentDeclaration(firstDeclaration);
-        subject.addDocumentDeclaration(secondDeclaration);
+        subject.addTerms(lastDeclaration);
+        subject.addTerms(firstDeclaration);
+        subject.addTerms(secondDeclaration);
       });
 
       context('without given date', () => {
-        it('returns the last document declaration', async () => {
-          expect(subject.getDocumentDeclaration(DOCUMENT_TYPE)).to.eql(lastDeclaration);
+        it('returns the last terms declaration', async () => {
+          expect(subject.getTerms(TERMS_TYPE)).to.eql(lastDeclaration);
         });
       });
 
       context('with a date', () => {
-        it('returns the document declaration according to the given date', async () => {
-          expect(subject.getDocumentDeclaration(DOCUMENT_TYPE, '2020-08-21T11:30:21.000Z')).to.eql(secondDeclaration);
+        it('returns the terms declaration according to the given date', async () => {
+          expect(subject.getTerms(TERMS_TYPE, '2020-08-21T11:30:21.000Z')).to.eql(secondDeclaration);
         });
 
-        context('strictly equal to a document declaration validity date', () => {
-          it('returns the document declaration with the validity date equal to the given date', async () => {
-            expect(subject.getDocumentDeclaration(DOCUMENT_TYPE, secondDeclaration.validUntil)).to.eql(secondDeclaration);
+        context('strictly equal to a terms declaration validity date', () => {
+          it('returns the terms declaration with the validity date equal to the given date', async () => {
+            expect(subject.getTerms(TERMS_TYPE, secondDeclaration.validUntil)).to.eql(secondDeclaration);
           });
         });
       });
     });
   });
 
-  describe('#getDocumentTypes', () => {
+  describe('#getTermsTypes', () => {
     let subject;
     let termsOfServiceDeclaration;
     let privacyPolicyDeclaration;
@@ -137,25 +121,19 @@ describe('Service', () => {
     before(async () => {
       subject = new Service({ id: 'serviceID', name: 'serviceName' });
 
-      termsOfServiceDeclaration = new DocumentDeclaration({
-        type: 'Terms of Service',
-        location: 'https://www.service.example/tos',
-        contentSelectors: 'body',
-      });
+      termsOfServiceDeclaration = new Terms({ type: TERMS_TYPE });
 
-      privacyPolicyDeclaration = new DocumentDeclaration({
+      privacyPolicyDeclaration = new Terms({
         type: 'Privacy Policy',
-        location: 'https://www.service.example/terms',
-        contentSelectors: 'main',
         validUntil: '2020-07-22T11:30:21.000Z',
       });
 
-      subject.addDocumentDeclaration(termsOfServiceDeclaration);
-      subject.addDocumentDeclaration(privacyPolicyDeclaration);
+      subject.addTerms(termsOfServiceDeclaration);
+      subject.addTerms(privacyPolicyDeclaration);
     });
 
     it('returns the service terms types', async () => {
-      expect(subject.getDocumentTypes()).to.have.members([
+      expect(subject.getTermsTypes()).to.have.members([
         termsOfServiceDeclaration.type,
         privacyPolicyDeclaration.type,
       ]);

package/src/archivist/services/{pageDeclaration.js → sourceDocument.js}

@@ -1,19 +1,21 @@
-export default class PageDeclaration {
-  constructor({ location, executeClientScripts, contentSelectors, noiseSelectors, filters }) {
+export default class SourceDocument {
+  constructor({ location, executeClientScripts, contentSelectors, insignificantContentSelectors, filters, content, mimeType }) {
     this.location = location;
     this.executeClientScripts = executeClientScripts;
     this.contentSelectors = contentSelectors;
-    this.noiseSelectors = noiseSelectors;
+    this.insignificantContentSelectors = insignificantContentSelectors;
     this.filters = filters;
+    this.content = content;
+    this.mimeType = mimeType;
     this.id = new URL(location).pathname.split('/').filter(Boolean).join('-');
   }
 
   get cssSelectors() {
-    const { contentSelectors, noiseSelectors } = this;
+    const { contentSelectors, insignificantContentSelectors } = this;
 
     const result = [
-      ...PageDeclaration.extractCssSelectorsFromProperty(contentSelectors),
-      ...PageDeclaration.extractCssSelectorsFromProperty(noiseSelectors),
+      ...SourceDocument.extractCssSelectorsFromProperty(contentSelectors),
+      ...SourceDocument.extractCssSelectorsFromProperty(insignificantContentSelectors),
     ];
 
     return result.filter(selector => selector);
@@ -23,10 +25,10 @@ export default class PageDeclaration {
     if (Array.isArray(property)) {
       return []
         .concat(property)
-        .flatMap(selector => PageDeclaration.extractCssSelectorsFromSelector(selector));
+        .flatMap(selector => SourceDocument.extractCssSelectorsFromSelector(selector));
     }
 
-    return PageDeclaration.extractCssSelectorsFromSelector(property);
+    return SourceDocument.extractCssSelectorsFromSelector(property);
   }
 
   static extractCssSelectorsFromSelector(selector) {
@@ -43,7 +45,7 @@ export default class PageDeclaration {
     return {
       fetch: this.location,
       select: this.contentSelectors,
-      remove: this.noiseSelectors,
+      remove: this.insignificantContentSelectors,
       filter: this.filters ? this.filters.map(filter => filter.name) : undefined,
       executeClientScripts: this.executeClientScripts,
     };

package/src/archivist/services/{pageDeclaration.test.js → sourceDocument.test.js}

@@ -1,17 +1,17 @@
 import chai from 'chai';
 
-import PageDeclaration from './pageDeclaration.js';
+import SourceDocument from './sourceDocument.js';
 
 const { expect } = chai;
 
-describe('PageDeclaration', () => {
+describe('SourceDocument', () => {
   const URL = 'https://www.service.example/terms';
 
   describe('#getCssSelectors', () => {
     context('with "select" property', () => {
       context('with string selector', () => {
         it('extracts selectors', async () => {
-          const result = new PageDeclaration({ location: URL, contentSelectors: 'body' }).cssSelectors;
+          const result = new SourceDocument({ location: URL, contentSelectors: 'body' }).cssSelectors;
 
           expect(result).to.deep.equal(['body']);
         });
@@ -19,7 +19,7 @@ describe('PageDeclaration', () => {
 
       context('with range selector', () => {
         it('extracts selectors', async () => {
-          const result = new PageDeclaration({
+          const result = new SourceDocument({
             location: URL,
             contentSelectors: {
               startBefore: '#startBefore',
@@ -33,7 +33,7 @@ describe('PageDeclaration', () => {
 
       context('with an array of mixed selectors', () => {
         it('extracts selectors', async () => {
-          const result = new PageDeclaration({
+          const result = new SourceDocument({
             location: URL,
             contentSelectors: [
               {
@@ -52,7 +52,7 @@ describe('PageDeclaration', () => {
     context('with "remove" property', () => {
       context('with string selector', () => {
         it('extracts selectors', async () => {
-          const result = new PageDeclaration({ location: URL, noiseSelectors: 'body' }).cssSelectors;
+          const result = new SourceDocument({ location: URL, insignificantContentSelectors: 'body' }).cssSelectors;
 
           expect(result).to.deep.equal(['body']);
         });
@@ -60,9 +60,9 @@ describe('PageDeclaration', () => {
 
       context('with range selector', () => {
         it('extracts selectors', async () => {
-          const result = new PageDeclaration({
+          const result = new SourceDocument({
             location: URL,
-            noiseSelectors: {
+            insignificantContentSelectors: {
               startBefore: '#startBefore',
               endBefore: '#endBefore',
             },
@@ -74,9 +74,9 @@ describe('PageDeclaration', () => {
 
       context('with an array of mixed selectors', () => {
         it('extracts selectors', async () => {
-          const result = new PageDeclaration({
+          const result = new SourceDocument({
             location: URL,
-            noiseSelectors: [
+            insignificantContentSelectors: [
               {
                 startBefore: '#startBefore',
                 endBefore: '#endBefore',
@@ -93,10 +93,10 @@ describe('PageDeclaration', () => {
     context('with both "select" and "remove" property', () => {
       context('with string selector', () => {
         it('extracts selectors', async () => {
-          const result = new PageDeclaration({
+          const result = new SourceDocument({
             location: URL,
             contentSelectors: 'body',
-            noiseSelectors: 'h1',
+            insignificantContentSelectors: 'h1',
           }).cssSelectors;
 
           expect(result).to.deep.equal([ 'body', 'h1' ]);
@@ -105,13 +105,13 @@ describe('PageDeclaration', () => {
 
       context('with range selector', () => {
         it('extracts selectors', async () => {
-          const result = new PageDeclaration({
+          const result = new SourceDocument({
             location: URL,
             contentSelectors: {
               startBefore: '#startBefore',
               endBefore: '#endBefore',
             },
-            noiseSelectors: {
+            insignificantContentSelectors: {
               startBefore: '#startBefore',
               endBefore: '#endBefore',
             },
@@ -128,7 +128,7 @@ describe('PageDeclaration', () => {
 
       context('with an array of mixed selectors', () => {
         it('extracts selectors', async () => {
-          const result = new PageDeclaration({
+          const result = new SourceDocument({
             location: URL,
             contentSelectors: [
               {
@@ -137,7 +137,7 @@ describe('PageDeclaration', () => {
               },
               'body',
             ],
-            noiseSelectors: [
+            insignificantContentSelectors: [
               {
                 startBefore: '#startBefore',
                 endBefore: '#endBefore',
@@ -160,8 +160,8 @@ describe('PageDeclaration', () => {
   });
 
   describe('#toPersistence', () => {
-    it('converts basic page declaration into JSON representation', async () => {
-      const result = new PageDeclaration({
+    it('converts basic source document declarations into JSON representation', async () => {
+      const result = new SourceDocument({
         location: URL,
         contentSelectors: 'body',
       }).toPersistence();
@@ -177,8 +177,8 @@ describe('PageDeclaration', () => {
       expect(result).to.deep.equal(expectedResult);
     });
 
-    it('converts page declaration with all fields to JSON representation', async () => {
-      const result = new PageDeclaration({
+    it('converts full source document declarations to JSON representation', async () => {
+      const result = new SourceDocument({
         location: URL,
         contentSelectors: [
           {
@@ -187,7 +187,7 @@ describe('PageDeclaration', () => {
           },
           'body',
         ],
-        noiseSelectors: [
+        insignificantContentSelectors: [
           {
             startBefore: '#startBefore',
             endBefore: '#endBefore',

package/src/archivist/services/terms.js

@@ -0,0 +1,26 @@
+export default class Terms {
+  constructor({ service, type, sourceDocuments, validUntil }) {
+    this.service = service;
+    this.type = type;
+    this.sourceDocuments = sourceDocuments;
+
+    if (validUntil) {
+      this.validUntil = validUntil;
+    }
+  }
+
+  get hasMultipleSourceDocuments() {
+    return this.sourceDocuments.length > 1;
+  }
+
+  toPersistence() {
+    return {
+      name: this.service.name,
+      documents: {
+        [this.type]: this.hasMultipleSourceDocuments
+          ? { combine: this.sourceDocuments.map(sourceDocument => sourceDocument.toPersistence()) }
+          : this.sourceDocuments[0].toPersistence(),
+      },
+    };
+  }
+}

package/src/archivist/services/{documentDeclaration.test.js → terms.test.js}

@@ -1,15 +1,15 @@
 import chai from 'chai';
 
-import DocumentDeclaration from './documentDeclaration.js';
-import PageDeclaration from './pageDeclaration.js';
+import SourceDocument from './sourceDocument.js';
+import Terms from './terms.js';
 
 const { expect } = chai;
 
-describe('DocumentDeclaration', () => {
+describe('Terms', () => {
   const service = { name: 'Service' };
-  const type = 'Terms of Service';
+  const termsType = 'Terms of Service';
   const URL = 'https://www.service.example/terms';
-  const page1 = new PageDeclaration({
+  const document1 = new SourceDocument({
     location: URL,
     contentSelectors: [
       {
@@ -18,7 +18,7 @@ describe('DocumentDeclaration', () => {
       },
       'body',
     ],
-    noiseSelectors: [
+    insignificantContentSelectors: [
       {
         startBefore: '#startBefore',
         endBefore: '#endBefore',
@@ -26,7 +26,7 @@ describe('DocumentDeclaration', () => {
       'body',
     ],
   });
-  const page1AsJSON = {
+  const document1AsJSON = {
     fetch: URL,
     select: [
       {
@@ -46,12 +46,12 @@ describe('DocumentDeclaration', () => {
     executeClientScripts: undefined,
   };
 
-  const page2 = new PageDeclaration({
+  const document2 = new SourceDocument({
     location: URL,
     contentSelectors: 'body',
   });
 
-  const page2AsJSON = {
+  const document2AsJSON = {
     fetch: URL,
     select: 'body',
     remove: undefined,
@@ -60,23 +60,23 @@ describe('DocumentDeclaration', () => {
   };
 
   describe('#toPersistence', () => {
-    it('converts one page document to JSON representation', async () => {
-      const result = new DocumentDeclaration({ service, type, pages: [page1] }).toPersistence();
+    it('converts terms with single source document to JSON representation', async () => {
+      const result = new Terms({ service, type: termsType, sourceDocuments: [document1] }).toPersistence();
 
       const expectedResult = {
         name: service.name,
-        documents: { [type]: page1AsJSON },
+        documents: { [termsType]: document1AsJSON },
       };
 
       expect(result).to.deep.equal(expectedResult);
     });
 
-    it('converts multipage document to JSON representation', async () => {
-      const result = new DocumentDeclaration({ service, type, pages: [ page1, page2 ] }).toPersistence();
+    it('converts terms with multiple source documents to JSON representation', async () => {
+      const result = new Terms({ service, type: termsType, sourceDocuments: [ document1, document2 ] }).toPersistence();
 
       const expectedResult = {
         name: service.name,
-        documents: { [type]: { combine: [ page1AsJSON, page2AsJSON ] } },
+        documents: { [termsType]: { combine: [ document1AsJSON, document2AsJSON ] } },
       };
 
       expect(result).to.deep.equal(expectedResult);

package/src/exports.js

@@ -1,3 +1,3 @@
-export { default as pageDeclaration } from './archivist/services/pageDeclaration.js';
-export { default as filter } from './archivist/filter/exports.js';
+export { default as sourceDocument } from './archivist/services/sourceDocument.js';
+export { default as extract } from './archivist/extract/exports.js';
 export { default as fetch } from './archivist/fetcher/exports.js';