@opentermsarchive/engine 1.1.1 → 1.1.3

package/.eslintrc.yaml

@@ -10,6 +10,7 @@ plugins:
   - chai-friendly
   - import
   - json-format
+  - no-only-tests
 rules:
   arrow-parens:
     - error
@@ -101,6 +102,7 @@ rules:
     - error
     - properties: false
   require-await: 1
+  no-only-tests/no-only-tests: error
 
 overrides:
   - files:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "1.1.1",
+  "version": "1.1.3",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {
@@ -65,7 +65,6 @@
     "croner": "^4.3.6",
     "cross-env": "^7.0.3",
     "datauri": "^4.1.0",
-    "deep-diff": "^1.0.2",
     "dotenv": "^10.0.0",
     "eslint": "^8.53.0",
     "eslint-config-airbnb-base": "^15.0.0",
@@ -104,6 +103,7 @@
   "devDependencies": {
     "@commitlint/cli": "^19.0.3",
     "dir-compare": "^4.0.0",
+    "eslint-plugin-no-only-tests": "^3.1.0",
     "keep-a-changelog": "^2.5.3",
     "nock": "^13.2.1",
     "node-stream-zip": "^1.15.0",

package/scripts/declarations/lint/index.mocha.js

@@ -31,7 +31,7 @@ export default async options => {
   if (options.modified) {
     const declarationUtils = new DeclarationUtils(instancePath);
 
-    ({ services: servicesToValidate } = await declarationUtils.getModifiedServiceTermsTypes());
+    ({ services: servicesToValidate } = await declarationUtils.getModifiedServicesAndTermsTypes());
   }
 
   const lintFile = lintAndFixFile(options.fix);

package/scripts/declarations/utils/fixtures/serviceA.json

@@ -0,0 +1,13 @@
+{
+  "name": "Service",
+  "documents": {
+    "Terms of Service": {
+      "fetch": "https://domain.example/tos",
+      "select": "body"
+    },
+    "Privacy Policy": {
+      "fetch": "https://domain.example/privacy",
+      "select": "body"
+    }
+  }
+}

package/scripts/declarations/utils/fixtures/serviceAMultipleTermsUpdated.json

@@ -0,0 +1,20 @@
+{
+  "name": "Service",
+  "documents": {
+    "Terms of Service": {
+      "fetch": "https://domain.example/tos",
+      "select": "body",
+      "remove": "footer"
+    },
+    "Privacy Policy": {
+      "fetch": "https://domain.example/privacy",
+      "select": "body",
+      "remove": "footer"
+    },
+    "Imprint": {
+      "fetch": "https://domain.example/imprint",
+      "select": "body",
+      "remove": "footer"
+    }
+  }
+}

package/scripts/declarations/utils/fixtures/serviceATermsAdded.json

@@ -0,0 +1,17 @@
+{
+  "name": "Service",
+  "documents": {
+    "Terms of Service": {
+      "fetch": "https://domain.example/tos",
+      "select": "body"
+    },
+    "Privacy Policy": {
+      "fetch": "https://domain.example/privacy",
+      "select": "body"
+    },
+    "Imprint": {
+      "fetch": "https://domain.example/imprint",
+      "select": "body"
+    }
+  }
+}

package/scripts/declarations/utils/fixtures/serviceATermsRemoved.json

@@ -0,0 +1,9 @@
+{
+  "name": "Service",
+  "documents": {
+    "Terms of Service": {
+      "fetch": "https://domain.example/tos",
+      "select": "body"
+    }
+  }
+}

package/scripts/declarations/utils/fixtures/serviceATermsUpdated.history.json

@@ -0,0 +1,9 @@
+{
+  "Terms of Service": [
+    {
+      "fetch": "https://domain.example/tos",
+      "select": "body",
+      "validUntil": "2024-03-18T18:30:09Z"
+    }
+  ]
+}

package/scripts/declarations/utils/fixtures/serviceATermsUpdated.json

@@ -0,0 +1,14 @@
+{
+  "name": "Service",
+  "documents": {
+    "Terms of Service": {
+      "fetch": "https://domain.example/tos",
+      "select": "body",
+      "remove": "footer"
+    },
+    "Privacy Policy": {
+      "fetch": "https://domain.example/privacy",
+      "select": "body"
+    }
+  }
+}

package/scripts/declarations/utils/fixtures/serviceB.json

@@ -0,0 +1,9 @@
+{
+  "name": "Service B",
+  "documents": {
+    "Terms of Service": {
+      "fetch": "https://domain.example/tos",
+      "select": "body"
+    }
+  }
+}

package/scripts/declarations/utils/index.js

@@ -1,6 +1,5 @@
 import path from 'path';
 
-import DeepDiff from 'deep-diff';
 import simpleGit from 'simple-git';
 
 export default class DeclarationUtils {
@@ -9,13 +8,17 @@ export default class DeclarationUtils {
     this.defaultBranch = defaultBranch;
   }
 
-  static filePathToServiceId = filePath => path.parse(filePath.replace(/\.history|\.filters/, '')).name;
+  static getServiceIdFromFilePath(filePath) {
+    return path.parse(filePath.replace(/\.history|\.filters/, '')).name;
+  }
 
-  async getJSONFile(path, ref) {
+  async getJSONFromFile(ref, filePath) {
     try {
-      return JSON.parse(await this.git.show([`${ref}:${path}`]));
-    } catch (e) {
-      return {};
+      const fileContent = await this.git.show([`${ref}:${filePath}`]);
+
+      return JSON.parse(fileContent);
+    } catch (error) {
+      // the file does not exist on the requested branch or it is not parsable
     }
   }
 
@@ -24,7 +27,7 @@ export default class DeclarationUtils {
 
     const modifiedFilePaths = (modifiedFilePathsAsString ? modifiedFilePathsAsString.split('\0') : []).filter(str => str !== ''); // split on \0 rather than \n due to the -z option of git diff
 
-    return { modifiedFilePaths, modifiedServicesIds: Array.from(new Set(modifiedFilePaths.map(DeclarationUtils.filePathToServiceId))) };
+    return { modifiedFilePaths, modifiedServicesIds: Array.from(new Set(modifiedFilePaths.map(DeclarationUtils.getServiceIdFromFilePath))) };
   }
 
   async getModifiedServices() {
@@ -33,48 +36,51 @@ export default class DeclarationUtils {
     return modifiedServicesIds;
   }
 
-  async getModifiedServiceTermsTypes() {
-    const { modifiedFilePaths, modifiedServicesIds } = await this.getModifiedData();
+  async getModifiedServicesAndTermsTypes() {
+    const { modifiedFilePaths } = await this.getModifiedData();
     const servicesTermsTypes = {};
 
     await Promise.all(modifiedFilePaths.map(async modifiedFilePath => {
-      const serviceId = DeclarationUtils.filePathToServiceId(modifiedFilePath);
+      const serviceId = DeclarationUtils.getServiceIdFromFilePath(modifiedFilePath);
+
+      if (modifiedFilePath.endsWith('.history.json')) {
+        return; // Assuming history modifications imply corresponding changes in the service declaration and that the analysis of which terms types of this service have changed will be done when analysing the related declaration, no further action is required here
+      }
+
+      if (modifiedFilePath.endsWith('.filters.js')) {
+        const declaration = await this.getJSONFromFile(this.defaultBranch, `declarations/${serviceId}.json`);
 
-      if (!modifiedFilePath.endsWith('.json')) {
-        // Here we should compare AST of both files to detect on which function
-        // change has been made, and then find which terms type depends on this
-        // function.
-        // As this is a complicated process, we will just send back all terms types
-        const declaration = await this.getJSONFile(`declarations/${serviceId}.json`, this.defaultBranch);
+        servicesTermsTypes[serviceId] = Object.keys(declaration.documents); // Considering how rarely filters are used, simply return all term types that could potentially be impacted to spare implementing a function change check
 
-        return Object.keys(declaration.documents);
+        return;
       }
 
-      const defaultFile = await this.getJSONFile(modifiedFilePath, this.defaultBranch);
-      const modifiedFile = await this.getJSONFile(modifiedFilePath, 'HEAD');
+      const originalService = await this.getJSONFromFile(this.defaultBranch, modifiedFilePath);
+      const modifiedService = await this.getJSONFromFile('HEAD', modifiedFilePath);
+      const modifiedServiceTermsTypes = Object.keys(modifiedService.documents);
 
-      const diff = DeepDiff.diff(defaultFile, modifiedFile);
+      if (!originalService) {
+        servicesTermsTypes[serviceId] = modifiedServiceTermsTypes;
 
-      if (!diff) {
-        // This can happen if only a lint has been applied to a document
         return;
       }
 
-      const modifiedTermsTypes = diff.reduce((acc, { path }) => {
-        if (modifiedFilePath.includes('.history')) {
-          acc.add(path[0]);
-        } else if (path[0] == 'documents') {
-          acc.add(path[1]);
-        }
+      const originalServiceTermsTypes = Object.keys(originalService.documents);
 
-        return acc;
-      }, new Set());
+      modifiedServiceTermsTypes.forEach(termsType => {
+        const areTermsAdded = !originalServiceTermsTypes.includes(termsType);
+        const areTermsModified = JSON.stringify(originalService.documents[termsType]) != JSON.stringify(modifiedService.documents[termsType]);
+
+        if (!areTermsAdded && !areTermsModified) {
+          return;
+        }
 
-      servicesTermsTypes[serviceId] = Array.from(new Set([ ...servicesTermsTypes[serviceId] || [], ...modifiedTermsTypes ]));
+        servicesTermsTypes[serviceId] = [termsType].concat(servicesTermsTypes[serviceId] || []);
+      });
     }));
 
     return {
-      services: modifiedServicesIds,
+      services: Object.keys(servicesTermsTypes),
       servicesTermsTypes,
     };
   }

package/scripts/declarations/utils/index.test.js

@@ -0,0 +1,162 @@
+import fs from 'fs/promises';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+import { expect } from 'chai';
+
+import DeclarationUtils from './index.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+let declarationUtils;
+
+const SUBJECT_PATH = path.resolve(__dirname, './test');
+
+const FIXTURES = {
+  serviceA: { path: './fixtures/serviceA.json' },
+  serviceATermsUpdated: { path: './fixtures/serviceATermsUpdated.json' },
+  serviceATermsUpdatedHistory: { path: './fixtures/serviceATermsUpdated.history.json' },
+  serviceAMultipleTermsUpdated: { path: './fixtures/serviceAMultipleTermsUpdated.json' },
+  serviceATermsAdded: { path: './fixtures/serviceATermsAdded.json' },
+  serviceATermsRemoved: { path: './fixtures/serviceATermsRemoved.json' },
+  serviceB: { path: './fixtures/serviceB.json' },
+};
+
+const COMMIT_PATHS = {
+  serviceA: './declarations/ServiceA.json',
+  serviceAHistory: './declarations/ServiceA.history.json',
+  serviceB: './declarations/ServiceB.json',
+};
+
+const commitChanges = async (filePath, content) => {
+  await fs.writeFile(path.resolve(SUBJECT_PATH, filePath), JSON.stringify(content, null, 2));
+  await declarationUtils.git.add(filePath);
+  await declarationUtils.git.commit('Update declarations', filePath);
+};
+
+const removeLatestCommit = async () => {
+  await declarationUtils.git.reset('hard', ['HEAD~1']);
+};
+
+describe('DeclarationUtils', () => {
+  describe('#getModifiedServicesAndTermsTypes', () => {
+    before(async () => {
+      await loadFixtures();
+      await setupRepository();
+    });
+
+    after(() => fs.rm(SUBJECT_PATH, { recursive: true }));
+
+    context('when an existing declaration has been modified', () => {
+      before(async () => {
+        await commitChanges(COMMIT_PATHS.serviceA, FIXTURES.serviceATermsUpdated.content);
+        await commitChanges(COMMIT_PATHS.serviceAHistory, FIXTURES.serviceATermsUpdatedHistory.content);
+      });
+      after(async () => {
+        await removeLatestCommit();
+        await removeLatestCommit();
+      });
+
+      it('returns the service ID and the updated terms type', async () => {
+        expect(await declarationUtils.getModifiedServicesAndTermsTypes()).to.deep.equal({
+          services: ['ServiceA'],
+          servicesTermsTypes: { ServiceA: ['Terms of Service'] },
+        });
+      });
+    });
+
+    context('when new terms are declared in an existing declaration', () => {
+      before(() => commitChanges(COMMIT_PATHS.serviceA, FIXTURES.serviceATermsAdded.content));
+      after(removeLatestCommit);
+
+      it('returns the service ID and the added terms type', async () => {
+        expect(await declarationUtils.getModifiedServicesAndTermsTypes()).to.deep.equal({
+          services: ['ServiceA'],
+          servicesTermsTypes: { ServiceA: ['Imprint'] },
+        });
+      });
+    });
+
+    context('when a new declaration has been added', () => {
+      before(() => commitChanges(COMMIT_PATHS.serviceB, FIXTURES.serviceB.content));
+      after(removeLatestCommit);
+
+      it('returns the added service ID along with the associated terms type', async () => {
+        expect(await declarationUtils.getModifiedServicesAndTermsTypes()).to.deep.equal({
+          services: ['ServiceB'],
+          servicesTermsTypes: { ServiceB: ['Terms of Service'] },
+        });
+      });
+    });
+
+    context('when a declaration has been removed', () => {
+      before(removeLatestCommit);
+      after(async () => {
+        await fs.mkdir(path.resolve(SUBJECT_PATH, './declarations'), { recursive: true });
+        await commitChanges(COMMIT_PATHS.serviceA, FIXTURES.serviceA.content);
+      });
+
+      it('returns no services and no terms types', async () => {
+        expect(await declarationUtils.getModifiedServicesAndTermsTypes()).to.deep.equal({
+          services: [],
+          servicesTermsTypes: {},
+        });
+      });
+    });
+
+    context('when terms are removed from an existing declaration', () => {
+      before(() => commitChanges(COMMIT_PATHS.serviceA, FIXTURES.serviceATermsRemoved.content));
+      after(removeLatestCommit);
+
+      it('returns no services and no terms types', async () => {
+        expect(await declarationUtils.getModifiedServicesAndTermsTypes()).to.deep.equal({
+          services: [],
+          servicesTermsTypes: {},
+        });
+      });
+    });
+
+    context('when there is a combination of an updated declaration and an added declaration', () => {
+      before(async () => {
+        await commitChanges(COMMIT_PATHS.serviceA, FIXTURES.serviceAMultipleTermsUpdated.content);
+        await commitChanges(COMMIT_PATHS.serviceB, FIXTURES.serviceB.content);
+      });
+
+      after(async () => {
+        await removeLatestCommit();
+        await removeLatestCommit();
+      });
+
+      it('returns the services IDs and the updated terms types', async () => {
+        expect(await declarationUtils.getModifiedServicesAndTermsTypes()).to.deep.equal({
+          services: [ 'ServiceA', 'ServiceB' ],
+          servicesTermsTypes: {
+            ServiceA: [ 'Imprint', 'Privacy Policy', 'Terms of Service' ],
+            ServiceB: ['Terms of Service'],
+          },
+        });
+      });
+    });
+  });
+});
+
+async function loadFixtures() {
+  await Promise.all(Object.values(FIXTURES).map(async fixture => {
+    const content = await fs.readFile(path.resolve(__dirname, fixture.path), 'utf-8');
+
+    fixture.content = JSON.parse(content);
+  }));
+}
+
+async function setupRepository() {
+  await fs.mkdir(path.resolve(SUBJECT_PATH, './declarations'), { recursive: true });
+
+  declarationUtils = new DeclarationUtils(SUBJECT_PATH, 'main');
+  await declarationUtils.git.init();
+  await declarationUtils.git.addConfig('user.name', 'Open Terms Archive Testing Bot');
+  await declarationUtils.git.addConfig('user.email', 'testing-bot@opentermsarchive.org');
+  await declarationUtils.git.checkoutLocalBranch('main');
+  await commitChanges('./README.md', 'This directory is auto-generated by test executions and requires cleanup after each test run');
+  await commitChanges(COMMIT_PATHS.serviceA, FIXTURES.serviceA.content);
+  await declarationUtils.git.checkoutLocalBranch('updated');
+}

package/scripts/declarations/validate/index.mocha.js

@@ -37,11 +37,11 @@ export default async options => {
   if (options.modified) {
     const declarationUtils = new DeclarationUtils(instancePath);
 
-    ({ services: servicesToValidate, servicesTermsTypes } = await declarationUtils.getModifiedServiceTermsTypes());
+    ({ services: servicesToValidate, servicesTermsTypes } = await declarationUtils.getModifiedServicesAndTermsTypes());
   }
 
   describe('Service declarations validation', async function () {
-    this.timeout(30000);
+    this.timeout(60000);
     this.slow(SLOW_DOCUMENT_THRESHOLD);
 
     servicesToValidate.forEach(serviceId => {

package/src/archivist/fetcher/index.test.js

@@ -20,7 +20,7 @@ const termsHTML = '<!DOCTYPE html><html><head><meta charset="UTF-8"><title>First
 const termsWithOtherCharsetHTML = '<!DOCTYPE html><html><head><meta http-equiv="Content-Type" content="text/html; charset=windows-1251"><title>TOS на първия доставчик</title></head><body><h1>Условия за ползване</h1><p>Dapibus quis diam sagittis</p></body></html>';
 
 describe('Fetcher', function () {
-  this.timeout(10000);
+  this.timeout(60000);
 
   before(launchHeadlessBrowser);