@opentermsarchive/engine 0.15.0

package/src/archivist/services/pageDeclaration.test.js

@@ -0,0 +1,224 @@
+import chai from 'chai';
+
+import PageDeclaration from './pageDeclaration.js';
+
+const { expect } = chai;
+
+describe('PageDeclaration', () => {
+  const URL = 'https://www.service.example/terms';
+
+  describe('#getCssSelectors', () => {
+    context('with "select" property', () => {
+      context('with string selector', () => {
+        it('extracts selectors', async () => {
+          const result = new PageDeclaration({ location: URL, contentSelectors: 'body' }).cssSelectors;
+
+          expect(result).to.deep.equal(['body']);
+        });
+      });
+
+      context('with range selector', () => {
+        it('extracts selectors', async () => {
+          const result = new PageDeclaration({
+            location: URL,
+            contentSelectors: {
+              startBefore: '#startBefore',
+              endBefore: '#endBefore',
+            },
+          }).cssSelectors;
+
+          expect(result).to.deep.equal([ '#startBefore', '#endBefore' ]);
+        });
+      });
+
+      context('with an array of mixed selectors', () => {
+        it('extracts selectors', async () => {
+          const result = new PageDeclaration({
+            location: URL,
+            contentSelectors: [
+              {
+                startBefore: '#startBefore',
+                endBefore: '#endBefore',
+              },
+              'body',
+            ],
+          }).cssSelectors;
+
+          expect(result).to.deep.equal([ '#startBefore', '#endBefore', 'body' ]);
+        });
+      });
+    });
+
+    context('with "remove" property', () => {
+      context('with string selector', () => {
+        it('extracts selectors', async () => {
+          const result = new PageDeclaration({ location: URL, noiseSelectors: 'body' }).cssSelectors;
+
+          expect(result).to.deep.equal(['body']);
+        });
+      });
+
+      context('with range selector', () => {
+        it('extracts selectors', async () => {
+          const result = new PageDeclaration({
+            location: URL,
+            noiseSelectors: {
+              startBefore: '#startBefore',
+              endBefore: '#endBefore',
+            },
+          }).cssSelectors;
+
+          expect(result).to.deep.equal([ '#startBefore', '#endBefore' ]);
+        });
+      });
+
+      context('with an array of mixed selectors', () => {
+        it('extracts selectors', async () => {
+          const result = new PageDeclaration({
+            location: URL,
+            noiseSelectors: [
+              {
+                startBefore: '#startBefore',
+                endBefore: '#endBefore',
+              },
+              'body',
+            ],
+          }).cssSelectors;
+
+          expect(result).to.deep.equal([ '#startBefore', '#endBefore', 'body' ]);
+        });
+      });
+    });
+
+    context('with both "select" and "remove" property', () => {
+      context('with string selector', () => {
+        it('extracts selectors', async () => {
+          const result = new PageDeclaration({
+            location: URL,
+            contentSelectors: 'body',
+            noiseSelectors: 'h1',
+          }).cssSelectors;
+
+          expect(result).to.deep.equal([ 'body', 'h1' ]);
+        });
+      });
+
+      context('with range selector', () => {
+        it('extracts selectors', async () => {
+          const result = new PageDeclaration({
+            location: URL,
+            contentSelectors: {
+              startBefore: '#startBefore',
+              endBefore: '#endBefore',
+            },
+            noiseSelectors: {
+              startBefore: '#startBefore',
+              endBefore: '#endBefore',
+            },
+          }).cssSelectors;
+
+          expect(result).to.deep.equal([
+            '#startBefore',
+            '#endBefore',
+            '#startBefore',
+            '#endBefore',
+          ]);
+        });
+      });
+
+      context('with an array of mixed selectors', () => {
+        it('extracts selectors', async () => {
+          const result = new PageDeclaration({
+            location: URL,
+            contentSelectors: [
+              {
+                startBefore: '#startBefore',
+                endBefore: '#endBefore',
+              },
+              'body',
+            ],
+            noiseSelectors: [
+              {
+                startBefore: '#startBefore',
+                endBefore: '#endBefore',
+              },
+              'body',
+            ],
+          }).cssSelectors;
+
+          expect(result).to.deep.equal([
+            '#startBefore',
+            '#endBefore',
+            'body',
+            '#startBefore',
+            '#endBefore',
+            'body',
+          ]);
+        });
+      });
+    });
+  });
+
+  describe('#toPersistence', () => {
+    it('converts basic page declaration into JSON representation', async () => {
+      const result = new PageDeclaration({
+        location: URL,
+        contentSelectors: 'body',
+      }).toPersistence();
+
+      const expectedResult = {
+        fetch: URL,
+        select: 'body',
+        remove: undefined,
+        filter: undefined,
+        executeClientScripts: undefined,
+      };
+
+      expect(result).to.deep.equal(expectedResult);
+    });
+
+    it('converts page declaration with all fields to JSON representation', async () => {
+      const result = new PageDeclaration({
+        location: URL,
+        contentSelectors: [
+          {
+            startBefore: '#startBefore',
+            endBefore: '#endBefore',
+          },
+          'body',
+        ],
+        noiseSelectors: [
+          {
+            startBefore: '#startBefore',
+            endBefore: '#endBefore',
+          },
+          'body',
+        ],
+        filters: [function filterSomething() {}],
+        executeClientScripts: true,
+      }).toPersistence();
+
+      const expectedResult = {
+        fetch: URL,
+        select: [
+          {
+            startBefore: '#startBefore',
+            endBefore: '#endBefore',
+          },
+          'body',
+        ],
+        remove: [
+          {
+            startBefore: '#startBefore',
+            endBefore: '#endBefore',
+          },
+          'body',
+        ],
+        filter: ['filterSomething'],
+        executeClientScripts: true,
+      };
+
+      expect(result).to.deep.equal(expectedResult);
+    });
+  });
+});

package/src/archivist/services/service.js

@@ -0,0 +1,60 @@
+export default class Service {
+  documents = new Map();
+
+  constructor({ id, name }) {
+    this.id = id;
+    this.name = name;
+  }
+
+  getDocumentDeclaration(documentType, date) {
+    if (!this.documents[documentType]) {
+      return null;
+    }
+
+    const { latest: currentlyValidDocumentDeclaration, history } = this.documents[documentType];
+
+    if (!date) {
+      return currentlyValidDocumentDeclaration;
+    }
+
+    return (
+      history?.find(entry => new Date(date) <= new Date(entry.validUntil))
+      || currentlyValidDocumentDeclaration
+    );
+  }
+
+  getDocumentTypes() {
+    return Object.keys(this.documents);
+  }
+
+  addDocumentDeclaration(document) {
+    if (!document.service) {
+      document.service = this;
+    }
+
+    this.documents[document.type] = this.documents[document.type] || {};
+
+    if (!document.validUntil) {
+      this.documents[document.type].latest = document;
+
+      return;
+    }
+
+    this.documents[document.type].history = this.documents[document.type].history || [];
+    this.documents[document.type].history.push(document);
+    this.documents[document.type].history.sort((a, b) => new Date(a.validUntil) - new Date(b.validUntil));
+  }
+
+  getHistoryDates(documentType) {
+    return this.documents[documentType].history.map(entry => entry.validUntil);
+  }
+
+  getNumberOfDocuments() {
+    return this.getDocumentTypes().length;
+  }
+
+  hasHistory() {
+    // If a service is loaded without its history it could return false even if a history declaration file exists.
+    return Boolean(Object.keys(this.documents).find(documentType => this.documents[documentType].history));
+  }
+}

package/src/archivist/services/service.test.js

@@ -0,0 +1,164 @@
+import chai from 'chai';
+
+import DocumentDeclaration from './documentDeclaration.js';
+import Service from './service.js';
+
+const { expect } = chai;
+
+describe('Service', () => {
+  let subject;
+  const DOCUMENT_TYPE = 'Terms of Service';
+
+  describe('#addDocumentDeclaration', () => {
+    let documentDeclaration;
+
+    before(async () => {
+      documentDeclaration = new DocumentDeclaration({
+        type: DOCUMENT_TYPE,
+        service: subject,
+        pages: [{
+          location: 'https://www.service.example/tos',
+          contentSelectors: 'body',
+        }],
+      });
+    });
+
+    context('when document declaration has no validity date', () => {
+      before(async () => {
+        subject = new Service({ id: 'serviceID', name: 'serviceName' });
+        subject.addDocumentDeclaration(documentDeclaration);
+      });
+
+      it('adds the document as the last valid document declaration', async () => {
+        expect(subject.getDocumentDeclaration(DOCUMENT_TYPE)).to.deep.eql(documentDeclaration);
+      });
+    });
+
+    context('when document declaration has a validity date', () => {
+      let expiredDocumentDeclaration;
+      const VALIDITY_DATE = new Date('2020-07-22T11:30:21.000Z');
+
+      before(async () => {
+        subject = new Service({ id: 'serviceID', name: 'serviceName' });
+        expiredDocumentDeclaration = new DocumentDeclaration({
+          type: 'Terms of Service',
+          service: subject,
+          validUntil: VALIDITY_DATE,
+          pages: [{
+            location: 'https://www.service.example/terms',
+            contentSelectors: 'main',
+          }],
+        });
+        subject.addDocumentDeclaration(expiredDocumentDeclaration);
+        subject.addDocumentDeclaration(documentDeclaration);
+      });
+
+      it('adds the document with the proper validity date', async () => {
+        expect(subject.getDocumentDeclaration(DOCUMENT_TYPE, VALIDITY_DATE)).to.deep.eql(expiredDocumentDeclaration);
+      });
+    });
+  });
+
+  describe('#getDocumentDeclaration', () => {
+    let subject;
+
+    const lastDeclaration = new DocumentDeclaration({
+      type: 'Terms of Service',
+      location: 'https://www.service.example/tos',
+      contentSelectors: 'body',
+    });
+
+    context('when there is no history', () => {
+      before(async () => {
+        subject = new Service({ id: 'serviceID', name: 'serviceName' });
+        subject.addDocumentDeclaration(lastDeclaration);
+      });
+
+      context('without given date', () => {
+        it('returns the last document declaration', async () => {
+          expect(subject.getDocumentDeclaration(DOCUMENT_TYPE)).to.eql(lastDeclaration);
+        });
+      });
+
+      context('with a date', () => {
+        it('returns the last document declaration', async () => {
+          expect(subject.getDocumentDeclaration(DOCUMENT_TYPE, '2020-08-21T11:30:21.000Z')).to.eql(lastDeclaration);
+        });
+      });
+    });
+
+    context('when the document has a history', () => {
+      const firstDeclaration = new DocumentDeclaration({
+        type: 'Terms of Service',
+        location: 'https://www.service.example/terms',
+        contentSelectors: 'main',
+        validUntil: '2020-07-22T11:30:21.000Z',
+      });
+
+      const secondDeclaration = new DocumentDeclaration({
+        type: 'Terms of Service',
+        location: 'https://www.service.example/terms-of-service',
+        contentSelectors: 'main',
+        validUntil: '2020-08-22T11:30:21.000Z',
+      });
+
+      before(async () => {
+        subject = new Service({ id: 'serviceID', name: 'serviceName' });
+        subject.addDocumentDeclaration(lastDeclaration);
+        subject.addDocumentDeclaration(firstDeclaration);
+        subject.addDocumentDeclaration(secondDeclaration);
+      });
+
+      context('without given date', () => {
+        it('returns the last document declaration', async () => {
+          expect(subject.getDocumentDeclaration(DOCUMENT_TYPE)).to.eql(lastDeclaration);
+        });
+      });
+
+      context('with a date', () => {
+        it('returns the document declaration according to the given date', async () => {
+          expect(subject.getDocumentDeclaration(DOCUMENT_TYPE, '2020-08-21T11:30:21.000Z')).to.eql(secondDeclaration);
+        });
+
+        context('strictly equal to a document declaration validity date', () => {
+          it('returns the document declaration with the validity date equal to the given date', async () => {
+            expect(subject.getDocumentDeclaration(DOCUMENT_TYPE, secondDeclaration.validUntil)).to.eql(secondDeclaration);
+          });
+        });
+      });
+    });
+  });
+
+  describe('#getDocumentTypes', () => {
+    let subject;
+    let termsOfServiceDeclaration;
+    let privacyPolicyDeclaration;
+
+    before(async () => {
+      subject = new Service({ id: 'serviceID', name: 'serviceName' });
+
+      termsOfServiceDeclaration = new DocumentDeclaration({
+        type: 'Terms of Service',
+        location: 'https://www.service.example/tos',
+        contentSelectors: 'body',
+      });
+
+      privacyPolicyDeclaration = new DocumentDeclaration({
+        type: 'Privacy Policy',
+        location: 'https://www.service.example/terms',
+        contentSelectors: 'main',
+        validUntil: '2020-07-22T11:30:21.000Z',
+      });
+
+      subject.addDocumentDeclaration(termsOfServiceDeclaration);
+      subject.addDocumentDeclaration(privacyPolicyDeclaration);
+    });
+
+    it('returns the service document types', async () => {
+      expect(subject.getDocumentTypes()).to.have.members([
+        termsOfServiceDeclaration.type,
+        privacyPolicyDeclaration.type,
+      ]);
+    });
+  });
+});

package/src/exports.js

@@ -0,0 +1,3 @@
+export { default as pageDeclaration } from './archivist/services/pageDeclaration.js';
+export { default as filter } from './archivist/filter/exports.js';
+export { default as fetch } from './archivist/fetcher/exports.js';

package/src/index.js

@@ -0,0 +1,59 @@
+import config from 'config';
+import cron from 'croner';
+
+import Archivist from './archivist/index.js';
+import logger from './logger/index.js';
+import Notifier from './notifier/index.js';
+import Tracker from './tracker/index.js';
+
+export default async function track({ services = [], documentTypes, refilterOnly, schedule }) {
+  const archivist = new Archivist({ recorderConfig: config.get('recorder') });
+
+  archivist.attach(logger);
+
+  await archivist.initialize();
+
+  logger.info('Start Open Terms Archive\n');
+
+  let serviceIds;
+
+  if (services.length) {
+    serviceIds = services.filter(serviceId => {
+      const isServiceDeclared = archivist.serviceDeclarations[serviceId];
+
+      if (!isServiceDeclared) {
+        logger.warn(`Parameter "${serviceId}" was interpreted as a service ID to update, but no matching declaration was found; it will be ignored`);
+      }
+
+      return isServiceDeclared;
+    });
+  }
+
+  await archivist.refilterAndRecord(serviceIds, documentTypes);
+
+  if (refilterOnly) {
+    return;
+  }
+
+  if (process.env.NODE_ENV === 'production') {
+    archivist.attach(new Notifier(archivist.serviceDeclarations));
+  }
+
+  if (process.env.GITHUB_TOKEN) {
+    const tracker = new Tracker(config.get('tracker'));
+
+    await tracker.initialize();
+    archivist.attach(tracker);
+  }
+
+  await archivist.trackChanges(serviceIds, documentTypes);
+
+  if (!schedule) {
+    return;
+  }
+
+  logger.info('The scheduler is running…');
+  logger.info('Documents will be tracked every six hours starting at half past midnight');
+
+  cron('30 */6 * * *', () => archivist.trackChanges(serviceIds, documentTypes));
+}

package/src/logger/README.md

@@ -0,0 +1 @@
+This module is intended to be considered as a simple consumer of the Archivist application's events API and therefore to be extracted from this repository. It is still in this repository to facilitate quick iterations but is expected to be extracted as soon as the Open Terms Archive app is stabilized.

package/src/logger/index.js

@@ -0,0 +1,131 @@
+import os from 'os';
+
+import config from 'config';
+import dotenv from 'dotenv';
+import winston from 'winston';
+import 'winston-mail';
+
+dotenv.config();
+const { combine, timestamp, printf, colorize } = winston.format;
+
+const alignedWithColorsAndTime = combine(
+  colorize(),
+  timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
+  printf(({ level, message, timestamp, serviceId, type, pageId }) => {
+    let prefix = '';
+
+    if (serviceId && type) {
+      prefix = `${serviceId} — ${type}`;
+    }
+
+    if (pageId) {
+      prefix = `${prefix}:${pageId}`;
+    }
+
+    if (prefix.length > 75) {
+      prefix = `${prefix.substring(0, 74)}…`;
+    }
+
+    return `${timestamp} ${level.padEnd(15)} ${prefix.padEnd(75)} ${message}`;
+  }),
+);
+
+const consoleTransport = new winston.transports.Console();
+
+const transports = [consoleTransport];
+
+if (config.get('logger.sendMailOnError')) {
+  const mailerOptions = {
+    to: config.get('logger.sendMailOnError.to'),
+    from: config.get('logger.sendMailOnError.from'),
+    host: config.get('logger.smtp.host'),
+    username: config.get('logger.smtp.username'),
+    password: process.env.SMTP_PASSWORD,
+    ssl: true,
+    timeout: 30 * 1000,
+    formatter: args => args[Object.getOwnPropertySymbols(args)[1]], // Returns the full error message, the same visible in the console. It is referenced in the argument object with a Symbol of which we do not have the reference but we know it is the second one.
+    exitOnError: true,
+  };
+
+  transports.push(new winston.transports.Mail({
+    ...mailerOptions,
+    level: 'error',
+    subject: `[OTA] Error Report — ${os.hostname()}`,
+  }));
+
+  if (config.get('logger.sendMailOnError.sendWarnings')) {
+    transports.push(new winston.transports.Mail({
+      ...mailerOptions,
+      level: 'warn',
+      subject: `[OTA] Inaccessible content — ${os.hostname()}`,
+    }));
+  }
+}
+
+let recordedSnapshotsCount;
+let recordedVersionsCount;
+
+const logger = winston.createLogger({
+  format: alignedWithColorsAndTime,
+  transports,
+  rejectionHandlers: transports,
+});
+
+logger.onFirstSnapshotRecorded = (serviceId, type, pageId, snapshotId) => {
+  logger.info({ message: `Recorded first snapshot with id ${snapshotId}`, serviceId, type, pageId });
+  recordedSnapshotsCount++;
+};
+
+logger.onSnapshotRecorded = (serviceId, type, pageId, snapshotId) => {
+  logger.info({ message: `Recorded snapshot with id ${snapshotId}`, serviceId, type, pageId });
+  recordedSnapshotsCount++;
+};
+
+logger.onSnapshotNotChanged = (serviceId, type, pageId) => {
+  logger.info({ message: 'No changes, did not record snapshot', serviceId, type, pageId });
+};
+
+logger.onFirstVersionRecorded = (serviceId, type, versionId) => {
+  logger.info({ message: `Recorded first version with id ${versionId}`, serviceId, type });
+  recordedVersionsCount++;
+};
+
+logger.onVersionRecorded = (serviceId, type, versionId) => {
+  logger.info({ message: `Recorded version with id ${versionId}`, serviceId, type });
+  recordedVersionsCount++;
+};
+
+logger.onVersionNotChanged = (serviceId, type) => {
+  logger.info({ message: 'No changes after filtering, did not record version', serviceId, type });
+};
+
+logger.onRefilteringStarted = (numberOfServices, numberOfDocuments) => {
+  logger.info(`Examining ${numberOfDocuments} documents from ${numberOfServices} services for refiltering…`);
+  recordedVersionsCount = 0;
+};
+
+logger.onRefilteringCompleted = (numberOfServices, numberOfDocuments) => {
+  logger.info(`Examined ${numberOfDocuments} documents from ${numberOfServices} services for refiltering`);
+  logger.info(`Recorded ${recordedVersionsCount} new versions\n`);
+};
+
+logger.onTrackingStarted = (numberOfServices, numberOfDocuments) => {
+  logger.info(`Tracking changes of ${numberOfDocuments} documents from ${numberOfServices} services…`);
+  recordedSnapshotsCount = 0;
+  recordedVersionsCount = 0;
+};
+
+logger.onTrackingCompleted = (numberOfServices, numberOfDocuments) => {
+  logger.info(`Tracked changes of ${numberOfDocuments} documents from ${numberOfServices} services`);
+  logger.info(`Recorded ${recordedSnapshotsCount} new snapshots and ${recordedVersionsCount} new versions\n`);
+};
+
+logger.onInaccessibleContent = ({ message }, serviceId, type) => {
+  logger.warn({ message, serviceId, type });
+};
+
+logger.onError = (error, serviceId, type, pageId) => {
+  logger.error({ message: error.stack, serviceId, type, pageId });
+};
+
+export default logger;

package/src/main.js

@@ -0,0 +1,18 @@
+import fs from 'fs';
+
+import { program } from 'commander';
+
+import track from './index.js';
+
+const { name, description, version } = JSON.parse(fs.readFileSync(new URL('../package.json', import.meta.url)).toString());
+
+program
+  .name(name)
+  .description(description)
+  .version(version)
+  .option('-s, --services [serviceId...]', 'service IDs of services to handle')
+  .option('-d, --documentTypes [documentType...]', 'document types to handle')
+  .option('-r, --refilter-only', 'only refilter exisiting snapshots with last declarations and engine\'s updates')
+  .option('--schedule', 'schedule automatic document tracking');
+
+track(program.parse(process.argv).opts());

package/src/notifier/README.md

@@ -0,0 +1 @@
+This module is intended to be considered as a simple consumer of the Archivist application's events API and therefore to be extracted from this repository. It is still in this repository to facilitate quick iterations but is expected to be extracted as soon as the Open Terms Archive app is stabilized.