@opentermsarchive/engine 0.15.0

package/src/archivist/services/index.js

@@ -0,0 +1,255 @@
+import fsApi from 'fs';
+import path from 'path';
+import { fileURLToPath, pathToFileURL } from 'url';
+
+import config from 'config';
+
+import DocumentDeclaration from './documentDeclaration.js';
+import PageDeclaration from './pageDeclaration.js';
+import Service from './service.js';
+
+const fs = fsApi.promises;
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const declarationsPath = path.resolve(__dirname, '../../..', config.get('services.declarationsPath'));
+
+export const DOCUMENT_TYPES = JSON.parse(fsApi.readFileSync(path.resolve(__dirname, './documentTypes.json')));
+
+export async function load(servicesIdsToLoad = []) {
+  let servicesIds = await getDeclaredServicesIds();
+
+  if (servicesIdsToLoad.length) {
+    servicesIds = servicesIds.filter(serviceId => serviceId.match(new RegExp(`^${servicesIdsToLoad.join('|')}$`, 'g')));
+  }
+
+  const services = {};
+
+  await Promise.all(servicesIds.map(async serviceId => {
+    const { name, documents: documentsDeclaration } = await loadServiceDeclaration(serviceId);
+
+    const service = new Service({ id: serviceId, name });
+
+    await Promise.all(Object.keys(documentsDeclaration).map(documentType => loadServiceDocument(service, documentType, documentsDeclaration[documentType])));
+
+    services[serviceId] = service;
+  }));
+
+  return services;
+}
+
+async function loadServiceDeclaration(serviceId) {
+  const jsonDeclarationFilePath = path.join(declarationsPath, `${serviceId}.json`);
+  const rawServiceDeclaration = await fs.readFile(jsonDeclarationFilePath);
+
+  try {
+    return JSON.parse(rawServiceDeclaration);
+  } catch (error) {
+    throw new Error(`The "${serviceId}" service declaration is malformed and cannot be parsed in ${jsonDeclarationFilePath}`);
+  }
+}
+
+async function loadServiceFilters(serviceId, filterNames) {
+  if (!filterNames) {
+    return;
+  }
+
+  const filterFilePath = `${serviceId}.filters.js`;
+  const serviceFilters = await import(pathToFileURL(path.join(declarationsPath, filterFilePath))); // eslint-disable-line no-await-in-loop
+
+  return filterNames.map(filterName => serviceFilters[filterName]);
+}
+
+async function loadServiceDocument(service, documentType, documentTypeDeclaration) {
+  const { filter: filterNames, fetch: location, executeClientScripts, select: contentSelectors, remove: noiseSelectors, combine } = documentTypeDeclaration;
+
+  const pages = [];
+
+  const filters = await loadServiceFilters(service.id, filterNames);
+
+  if (!combine) {
+    pages.push(new PageDeclaration({ location, executeClientScripts, contentSelectors, noiseSelectors, filters }));
+  } else {
+    for (const pageDeclaration of combine) {
+      const { filter: pageFilterNames, fetch: pageLocation, executeClientScripts: pageExecuteClientScripts, select: pageContentSelectors, remove: pageNoiseSelectors } = pageDeclaration;
+
+      const pageFilters = await loadServiceFilters(service.id, pageFilterNames); // eslint-disable-line no-await-in-loop
+
+      pages.push(new PageDeclaration({
+        location: pageLocation || location,
+        executeClientScripts: (pageExecuteClientScripts === undefined || pageExecuteClientScripts === null ? executeClientScripts : pageExecuteClientScripts),
+        contentSelectors: pageContentSelectors || contentSelectors,
+        noiseSelectors: pageNoiseSelectors || noiseSelectors,
+        filters: pageFilters || filters,
+      }));
+    }
+  }
+
+  service.addDocumentDeclaration(new DocumentDeclaration({ service, type: documentType, pages }));
+}
+
+async function getDeclaredServicesIds() {
+  const fileNames = await fs.readdir(declarationsPath);
+
+  const servicesFileNames = fileNames.filter(fileName => path.extname(fileName) == '.json' && !fileName.includes('.history.json'));
+
+  return servicesFileNames.map(serviceFileName => path.basename(serviceFileName, '.json'));
+}
+
+export async function loadWithHistory(servicesIds = []) {
+  const services = await load(servicesIds);
+
+  for (const serviceId of Object.keys(services)) {
+    const { declarations, filters } = await loadServiceHistoryFiles(serviceId); // eslint-disable-line no-await-in-loop
+
+    for (const documentType of Object.keys(declarations)) {
+      const documentTypeDeclarationEntries = declarations[documentType];
+      const filterNames = [...new Set(documentTypeDeclarationEntries.flatMap(declaration => declaration.filter))].filter(Boolean);
+      const allHistoryDates = extractHistoryDates({ documentTypeDeclarationEntries, filters, filterNames });
+
+      const latestValidDocumentDeclaration = documentTypeDeclarationEntries.find(entry => !entry.validUntil);
+
+      allHistoryDates.forEach(async date => {
+        const declarationForThisDate = documentTypeDeclarationEntries.find(entry => new Date(date) <= new Date(entry.validUntil)) || latestValidDocumentDeclaration;
+        const { filter: declarationForThisDateFilterNames, combine } = declarationForThisDate;
+
+        const pages = [];
+        let actualFilters;
+
+        if (declarationForThisDateFilterNames) {
+          actualFilters = declarationForThisDateFilterNames.map(filterName => {
+            const currentlyValidFilters = filters[filterName].find(entry => !entry.validUntil);
+            const validFilterForThisDate = filters[filterName].find(entry => new Date(date) <= new Date(entry.validUntil))
+              || currentlyValidFilters;
+
+            return validFilterForThisDate.filter;
+          });
+        }
+
+        if (!combine) {
+          pages.push(new PageDeclaration({
+            location: declarationForThisDate.fetch,
+            executeClientScripts: declarationForThisDate.executeClientScripts,
+            contentSelectors: declarationForThisDate.select,
+            noiseSelectors: declarationForThisDate.remove,
+            filters: actualFilters,
+          }));
+        } else {
+          for (const pageDeclaration of combine) {
+            const { filter: pageFilterNames, fetch: pageLocation, executeClientScripts: pageExecuteClientScripts, select: pageContentSelectors, remove: pageNoiseSelectors } = pageDeclaration;
+
+            const pageFilters = await loadServiceFilters(serviceId, pageFilterNames); // eslint-disable-line no-await-in-loop
+
+            pages.push(new PageDeclaration({
+              location: pageLocation || declarationForThisDate.fetch,
+              executeClientScripts: (pageExecuteClientScripts === undefined || pageExecuteClientScripts === null ? declarationForThisDate.executeClientScripts : pageExecuteClientScripts),
+              contentSelectors: pageContentSelectors || declarationForThisDate.select,
+              noiseSelectors: pageNoiseSelectors || declarationForThisDate.remove,
+              filters: pageFilters || actualFilters,
+            }));
+          }
+        }
+
+        services[serviceId].addDocumentDeclaration(new DocumentDeclaration({
+          service: services[serviceId],
+          type: documentType,
+          pages,
+          validUntil: date,
+        }));
+      });
+    }
+  }
+
+  return services;
+}
+
+function extractHistoryDates({ filters, filterNames, documentTypeDeclarationEntries }) {
+  const allHistoryDates = [];
+
+  Object.keys(filters).forEach(filterName => {
+    if (filterNames.includes(filterName)) {
+      filters[filterName].forEach(({ validUntil }) => allHistoryDates.push(validUntil));
+    }
+  });
+
+  documentTypeDeclarationEntries.forEach(({ validUntil }) => allHistoryDates.push(validUntil));
+
+  const sortedDates = allHistoryDates.sort((a, b) => new Date(a) - new Date(b));
+  const uniqSortedDates = [...new Set(sortedDates)];
+
+  return uniqSortedDates;
+}
+
+function sortHistory(history = {}) {
+  Object.keys(history).forEach(entry => {
+    history[entry].sort((a, b) => new Date(a.validUntil) - new Date(b.validUntil));
+  });
+}
+
+async function loadServiceHistoryFiles(serviceId) {
+  const serviceFileName = path.join(declarationsPath, `${serviceId}.json`);
+  const jsonDeclarationFilePath = await fs.readFile(serviceFileName);
+  let serviceDeclaration;
+
+  try {
+    serviceDeclaration = JSON.parse(jsonDeclarationFilePath);
+  } catch (e) {
+    throw new Error(`The "${path.basename(jsonDeclarationFilePath, '.json')}" service declaration is malformed and cannot be parsed in ${jsonDeclarationFilePath}`);
+  }
+
+  const serviceHistoryFileName = path.join(declarationsPath, `${serviceId}.history.json`);
+  const serviceFiltersFileName = path.join(declarationsPath, `${serviceId}.filters.js`);
+  const serviceFiltersHistoryFileName = path.join(declarationsPath, `${serviceId}.filters.history.js`);
+
+  let serviceHistory = {};
+  const serviceFiltersHistory = {};
+  let serviceFiltersHistoryModule;
+
+  if (await fileExists(serviceHistoryFileName)) {
+    try {
+      serviceHistory = JSON.parse(await fs.readFile(serviceHistoryFileName));
+    } catch (e) {
+      throw new Error(`The "${path.basename(serviceHistoryFileName, '.json')}" service declaration is malformed and cannot be parsed in ${serviceHistoryFileName}`);
+    }
+  }
+
+  Object.keys(serviceDeclaration.documents).forEach(documentType => {
+    serviceHistory[documentType] = serviceHistory[documentType] || [];
+    serviceHistory[documentType].push(serviceDeclaration.documents[documentType]);
+  });
+
+  sortHistory(serviceHistory);
+
+  if (await fileExists(serviceFiltersHistoryFileName)) {
+    serviceFiltersHistoryModule = await import(pathToFileURL(serviceFiltersHistoryFileName));
+    Object.keys(serviceFiltersHistoryModule).forEach(filterName => {
+      serviceFiltersHistory[filterName] = serviceFiltersHistoryModule[filterName];
+    });
+  }
+
+  if (await fileExists(serviceFiltersFileName)) {
+    const serviceFilters = await import(pathToFileURL(serviceFiltersFileName));
+
+    Object.keys(serviceFilters).forEach(filterName => {
+      serviceFiltersHistory[filterName] = serviceFiltersHistory[filterName] || [];
+      serviceFiltersHistory[filterName].push({ filter: serviceFilters[filterName] });
+    });
+  }
+
+  sortHistory(serviceFiltersHistory);
+
+  return {
+    declarations: serviceHistory || {},
+    filters: serviceFiltersHistory || {},
+  };
+}
+
+async function fileExists(filePath) {
+  try {
+    await fs.access(filePath);
+
+    return true;
+  } catch (error) {
+    if (error.code === 'ENOENT') {
+      return false;
+    }
+  }
+}

package/src/archivist/services/index.test.js

@@ -0,0 +1,327 @@
+import chai from 'chai';
+import chaiExclude from 'chai-exclude';
+
+import expectedServices from '../../../test/fixtures/services.js';
+
+import * as services from './index.js';
+
+chai.use(chaiExclude);
+const { expect } = chai;
+
+describe('Services', () => {
+  describe('#load', () => {
+    let result;
+
+    async function validateServiceWithoutHistory(serviceId, expected) {
+      /* eslint-disable no-loop-func */
+      for (const documentType of expected.getDocumentTypes()) {
+        context(`${documentType}`, () => {
+          let actualDocumentDeclaration;
+          let actualFilters;
+          let actualContentSelectors;
+          let actualNoiseSelectors;
+          let actualExecuteClientScripts;
+
+          const expectedDocumentDeclaration = expected.getDocumentDeclaration(documentType);
+
+          const { pages } = expectedDocumentDeclaration;
+
+          pages.forEach((page, index) => {
+            const {
+              filters: expectedFilters,
+              contentSelectors: expectedContentSelectors,
+              noiseSelectors: expectedNoiseSelectors,
+              executeClientScripts: expectedExecuteClientScripts,
+            } = page;
+
+            context(`Page: ${page.id}`, () => {
+              before(() => {
+                actualDocumentDeclaration = result[serviceId].getDocumentDeclaration(documentType);
+                const { pages: actualPages } = actualDocumentDeclaration;
+
+                ({
+                  filters: actualFilters,
+                  contentSelectors: actualContentSelectors,
+                  noiseSelectors: actualNoiseSelectors,
+                  executeClientScripts: actualExecuteClientScripts,
+                } = actualPages[index]);
+              });
+
+              it('has the proper service name', () => {
+                expect(actualDocumentDeclaration.service.name).to.eql(expectedDocumentDeclaration.service.name);
+              });
+
+              it('has the proper document type', () => {
+                expect(actualDocumentDeclaration.type).to.eql(expectedDocumentDeclaration.type);
+              });
+
+              it('has no validity date', () => {
+                expect(actualDocumentDeclaration.validUntil).to.be.undefined;
+              });
+
+              it('has the proper content selectors', async () => {
+                expect(actualContentSelectors).to.equal(expectedContentSelectors);
+              });
+
+              it('has the proper noise selectors', async () => {
+                expect(actualNoiseSelectors).to.equal(expectedNoiseSelectors);
+              });
+
+              it('has the proper executeClientScripts option', async () => {
+                expect(actualExecuteClientScripts).to.equal(expectedExecuteClientScripts);
+              });
+
+              if (expectedFilters) {
+                it('has the proper number of filter functions', async () => {
+                  expect(actualFilters.length).to.equal(expectedFilters.length);
+                });
+
+                for (let indexFilter = 0; indexFilter < expectedFilters.length; indexFilter++) {
+                  it(`has the proper "${expectedFilters[indexFilter].name}" filter function`, async () => {
+                    expect(await actualFilters[indexFilter]()).equal(await expectedFilters[indexFilter]()); // eslint-disable-line no-await-in-loop
+                  });
+                }
+              } else {
+                it('has no filters', () => {
+                  expect(actualFilters).to.be.undefined;
+                });
+              }
+            });
+          });
+        });
+      }
+      /* eslint-enable no-loop-func */
+    }
+
+    before(async () => {
+      result = await services.load();
+    });
+
+    describe('Service A', async () => {
+      await validateServiceWithoutHistory('service_A', expectedServices.service_A);
+    });
+
+    describe('Service B', async () => {
+      await validateServiceWithoutHistory('service_B', expectedServices.service_B);
+    });
+
+    context('when a service has no history', async () => {
+      describe('Service without history', async () => {
+        await validateServiceWithoutHistory('service_without_history', expectedServices.service_without_history);
+      });
+    });
+
+    context('when a service has only history for declarations', async () => {
+      describe('Service with declaration history', async () => {
+        await validateServiceWithoutHistory('service_with_declaration_history', expectedServices.service_with_declaration_history);
+      });
+    });
+
+    context('when a service has only history for filters', async () => {
+      describe('Service with filters history', async () => {
+        await validateServiceWithoutHistory('service_with_filters_history', expectedServices.service_with_filters_history);
+      });
+    });
+
+    context('when a service has histories both for filters and for declarations', async () => {
+      describe('Service with history', async () => {
+        await validateServiceWithoutHistory('service_with_history', expectedServices.service_with_history);
+      });
+    });
+
+    context('when a service has a multipage document', async () => {
+      describe('Service with a multipage document', async () => {
+        await validateServiceWithoutHistory('service_with_multipage_document', expectedServices.service_with_multipage_document);
+      });
+    });
+
+    context('when specifying services to load', async () => {
+      before(async () => {
+        result = await services.load([ 'service_A', 'service_B' ]);
+      });
+
+      it('loads only the given services', async () => {
+        expect(result).to.have.all.keys('service_A', 'service_B');
+      });
+    });
+  });
+
+  describe('#loadWithHistory', () => {
+    let result;
+
+    async function validateServiceWithHistory(serviceId, expected) {
+      /* eslint-disable no-loop-func */
+      for (const documentType of expected.getDocumentTypes()) {
+        context(`${documentType}`, () => {
+          const { history: expectedHistory } = expected.documents[documentType];
+          const expectedHistoryDates = expectedHistory && [ ...expectedHistory.map(entry => entry.validUntil), null ]; // add `null` entry to simulate the still current valid declaration
+
+          let actualDocumentDeclaration;
+          let actualFilters;
+          const expectedDocumentDeclaration = expected.getDocumentDeclaration(documentType);
+
+          const { pages } = expectedDocumentDeclaration;
+
+          before(() => {
+            actualDocumentDeclaration = result[serviceId].getDocumentDeclaration(documentType);
+          });
+
+          it('has the proper service name', () => {
+            expect(actualDocumentDeclaration.service.name).to.eql(expectedDocumentDeclaration.service.name);
+          });
+
+          it('has the proper document type', () => {
+            expect(actualDocumentDeclaration.type).to.eql(expectedDocumentDeclaration.type);
+          });
+
+          pages.forEach((page, index) => {
+            const { filters: expectedFilters } = page;
+
+            context(`${page.id} page`, () => {
+              before(() => {
+                const { pages: actualPages } = actualDocumentDeclaration;
+
+                ({ filters: actualFilters } = actualPages[index]);
+              });
+
+              if (expectedHistoryDates) {
+                for (const date of expectedHistoryDates) {
+                  context(`${date || 'Current'}`, () => {
+                    let actualFiltersForThisDate;
+                    let contentSelectorsForThisDate;
+                    let noiseSelectorsForThisDate;
+                    let actualExecuteClientScriptsForThisDate;
+
+                    const { pages: pagesForThisDate } = expected.getDocumentDeclaration(documentType, date);
+                    const {
+                      filters: expectedFiltersForThisDate,
+                      contentSelectors: expectedContentSelectors,
+                      noiseSelectors: expectedNoiseSelectors,
+                      expectedExecuteClientScripts: expectedExecuteClientScriptsForThisDate,
+                    } = pagesForThisDate[index];
+
+                    before(() => {
+                      const { pages: actualPagesForThisDate } = result[serviceId].getDocumentDeclaration(documentType, date);
+
+                      ({
+                        filters: actualFiltersForThisDate,
+                        contentSelectors: contentSelectorsForThisDate,
+                        noiseSelectors: noiseSelectorsForThisDate,
+                        expectedExecuteClientScripts: actualExecuteClientScriptsForThisDate,
+                      } = actualPagesForThisDate[index]);
+                    });
+
+                    it('has the proper content selectors', async () => {
+                      expect(contentSelectorsForThisDate).to.equal(expectedContentSelectors);
+                    });
+
+                    it('has the proper noise selectors', async () => {
+                      expect(noiseSelectorsForThisDate).to.equal(expectedNoiseSelectors);
+                    });
+
+                    it('has the proper executeClientScripts option', async () => {
+                      expect(actualExecuteClientScriptsForThisDate).to.equal(expectedExecuteClientScriptsForThisDate);
+                    });
+
+                    if (expectedFiltersForThisDate) {
+                      it('has the proper number of filter functions', async () => {
+                        expect(actualFiltersForThisDate.length).to.equal(expectedFiltersForThisDate.length);
+                      });
+
+                      for (let indexFilter = 0; indexFilter < expectedFiltersForThisDate.length; indexFilter++) {
+                        it(`has the proper "${expectedFiltersForThisDate[indexFilter].name}" filter function`, async () => {
+                          expect(await actualFiltersForThisDate[indexFilter]()).equal(await expectedFiltersForThisDate[indexFilter]()); // eslint-disable-line no-await-in-loop
+                        });
+                      }
+                    } else {
+                      it('has no filters', () => {
+                        expect(actualFiltersForThisDate).to.be.undefined;
+                      });
+                    }
+                  });
+                }
+              } else {
+                it('has no history', async () => {
+                  expect(actualDocumentDeclaration.validUntil).to.be.undefined;
+                });
+
+                if (expectedFilters) {
+                  it('has the proper number of filter functions', async () => {
+                    expect(actualFilters.length).to.equal(expectedFilters.length);
+                  });
+
+                  for (
+                    let indexFilter = 0;
+                    indexFilter < expectedFilters.length;
+                    indexFilter++
+                  ) {
+                    it(`has the proper "${expectedFilters[indexFilter].name}" filter function`, async () => {
+                      expect(await actualFilters[indexFilter]()).equal(await expectedFilters[indexFilter]()); // eslint-disable-line no-await-in-loop
+                    });
+                  }
+                } else {
+                  it('has no filters', () => {
+                    expect(actualFilters).to.be.undefined;
+                  });
+                }
+              }
+            });
+          });
+        });
+      }
+      /* eslint-enable no-loop-func */
+    }
+
+    before(async () => {
+      result = await services.loadWithHistory();
+    });
+
+    describe('Service A', async () => {
+      await validateServiceWithHistory('service_A', expectedServices.service_A);
+    });
+
+    describe('Service B', async () => {
+      await validateServiceWithHistory('service_B', expectedServices.service_B);
+    });
+
+    context('when a service has no history', async () => {
+      describe('Service without history', async () => {
+        await validateServiceWithHistory('service_without_history', expectedServices.service_without_history);
+      });
+    });
+
+    context('when a service has only declarations history', async () => {
+      describe('Service with declaration history', async () => {
+        await validateServiceWithHistory('service_with_declaration_history', expectedServices.service_with_declaration_history);
+      });
+    });
+
+    context('when a service has only filters history', async () => {
+      describe('Service with filters history', async () => {
+        await validateServiceWithHistory('service_with_filters_history', expectedServices.service_with_filters_history);
+      });
+    });
+
+    context('when a service has both filters and declarations histories', async () => {
+      describe('Service with history', async () => {
+        await validateServiceWithHistory('service_with_history', expectedServices.service_with_history);
+      });
+    });
+
+    context('when a service has a multipage document', async () => {
+      describe('Service with a multipage document', async () => {
+        await validateServiceWithHistory('service_with_multipage_document', expectedServices.service_with_multipage_document);
+      });
+    });
+
+    context('when specifying services to load', async () => {
+      before(async () => {
+        result = await services.loadWithHistory([ 'service_A', 'service_B' ]);
+      });
+
+      it('loads only the given services', async () => {
+        expect(result).to.have.all.keys('service_A', 'service_B');
+      });
+    });
+  });
+});

package/src/archivist/services/pageDeclaration.js

@@ -0,0 +1,51 @@
+export default class PageDeclaration {
+  constructor({ location, executeClientScripts, contentSelectors, noiseSelectors, filters }) {
+    this.location = location;
+    this.executeClientScripts = executeClientScripts;
+    this.contentSelectors = contentSelectors;
+    this.noiseSelectors = noiseSelectors;
+    this.filters = filters;
+    this.id = new URL(location).pathname.split('/').filter(Boolean).join('-');
+  }
+
+  get cssSelectors() {
+    const { contentSelectors, noiseSelectors } = this;
+
+    const result = [
+      ...PageDeclaration.extractCssSelectorsFromProperty(contentSelectors),
+      ...PageDeclaration.extractCssSelectorsFromProperty(noiseSelectors),
+    ];
+
+    return result.filter(selector => selector);
+  }
+
+  static extractCssSelectorsFromProperty(property) {
+    if (Array.isArray(property)) {
+      return []
+        .concat(property)
+        .flatMap(selector => PageDeclaration.extractCssSelectorsFromSelector(selector));
+    }
+
+    return PageDeclaration.extractCssSelectorsFromSelector(property);
+  }
+
+  static extractCssSelectorsFromSelector(selector) {
+    if (typeof selector === 'object') {
+      const { startBefore, endBefore, startAfter, endAfter } = selector;
+
+      return [ startBefore, endBefore, startAfter, endAfter ].filter(rangeSelector => rangeSelector);
+    }
+
+    return [selector];
+  }
+
+  toPersistence() {
+    return {
+      fetch: this.location,
+      select: this.contentSelectors,
+      remove: this.noiseSelectors,
+      filter: this.filters ? this.filters.map(filter => filter.name) : undefined,
+      executeClientScripts: this.executeClientScripts,
+    };
+  }
+}