@opentermsarchive/engine 1.1.1 → 1.1.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "1.1.1",
+  "version": "1.1.2",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {
@@ -65,7 +65,6 @@
     "croner": "^4.3.6",
     "cross-env": "^7.0.3",
     "datauri": "^4.1.0",
-    "deep-diff": "^1.0.2",
     "dotenv": "^10.0.0",
     "eslint": "^8.53.0",
     "eslint-config-airbnb-base": "^15.0.0",

package/scripts/declarations/lint/index.mocha.js

@@ -31,7 +31,7 @@ export default async options => {
   if (options.modified) {
     const declarationUtils = new DeclarationUtils(instancePath);
 
-    ({ services: servicesToValidate } = await declarationUtils.getModifiedServiceTermsTypes());
+    ({ services: servicesToValidate } = await declarationUtils.getModifiedServicesAndTermsTypes());
   }
 
   const lintFile = lintAndFixFile(options.fix);

package/scripts/declarations/utils/fixtures/serviceA.json

@@ -0,0 +1,13 @@
+{
+  "name": "Service",
+  "documents": {
+    "Terms of Service": {
+      "fetch": "https://domain.example/tos",
+      "select": "body"
+    },
+    "Privacy Policy": {
+      "fetch": "https://domain.example/privacy",
+      "select": "body"
+    }
+  }
+}

package/scripts/declarations/utils/fixtures/serviceAMultipleTermsUpdated.json

@@ -0,0 +1,20 @@
+{
+  "name": "Service",
+  "documents": {
+    "Terms of Service": {
+      "fetch": "https://domain.example/tos",
+      "select": "body",
+      "remove": "footer"
+    },
+    "Privacy Policy": {
+      "fetch": "https://domain.example/privacy",
+      "select": "body",
+      "remove": "footer"
+    },
+    "Imprint": {
+      "fetch": "https://domain.example/imprint",
+      "select": "body",
+      "remove": "footer"
+    }
+  }
+}

package/scripts/declarations/utils/fixtures/serviceATermsAdded.json

@@ -0,0 +1,17 @@
+{
+  "name": "Service",
+  "documents": {
+    "Terms of Service": {
+      "fetch": "https://domain.example/tos",
+      "select": "body"
+    },
+    "Privacy Policy": {
+      "fetch": "https://domain.example/privacy",
+      "select": "body"
+    },
+    "Imprint": {
+      "fetch": "https://domain.example/imprint",
+      "select": "body"
+    }
+  }
+}

package/scripts/declarations/utils/fixtures/serviceATermsRemoved.json

@@ -0,0 +1,9 @@
+{
+  "name": "Service",
+  "documents": {
+    "Terms of Service": {
+      "fetch": "https://domain.example/tos",
+      "select": "body"
+    }
+  }
+}

package/scripts/declarations/utils/fixtures/serviceATermsUpdated.json

@@ -0,0 +1,14 @@
+{
+  "name": "Service",
+  "documents": {
+    "Terms of Service": {
+      "fetch": "https://domain.example/tos",
+      "select": "body",
+      "remove": "footer"
+    },
+    "Privacy Policy": {
+      "fetch": "https://domain.example/privacy",
+      "select": "body"
+    }
+  }
+}

package/scripts/declarations/utils/fixtures/serviceB.json

@@ -0,0 +1,9 @@
+{
+  "name": "Service B",
+  "documents": {
+    "Terms of Service": {
+      "fetch": "https://domain.example/tos",
+      "select": "body"
+    }
+  }
+}

package/scripts/declarations/utils/index.js

@@ -1,6 +1,5 @@
 import path from 'path';
 
-import DeepDiff from 'deep-diff';
 import simpleGit from 'simple-git';
 
 export default class DeclarationUtils {
@@ -9,13 +8,17 @@ export default class DeclarationUtils {
     this.defaultBranch = defaultBranch;
   }
 
-  static filePathToServiceId = filePath => path.parse(filePath.replace(/\.history|\.filters/, '')).name;
+  static getServiceIdFromFilePath(filePath) {
+    return path.parse(filePath.replace(/\.history|\.filters/, '')).name;
+  }
 
-  async getJSONFile(path, ref) {
+  async getJSONFromFile(ref, filePath) {
     try {
-      return JSON.parse(await this.git.show([`${ref}:${path}`]));
-    } catch (e) {
-      return {};
+      const fileContent = await this.git.show([`${ref}:${filePath}`]);
+
+      return JSON.parse(fileContent);
+    } catch (error) {
+      // the file does not exist on the requested branch or it is not parsable
     }
   }
 
@@ -24,7 +27,7 @@ export default class DeclarationUtils {
 
     const modifiedFilePaths = (modifiedFilePathsAsString ? modifiedFilePathsAsString.split('\0') : []).filter(str => str !== ''); // split on \0 rather than \n due to the -z option of git diff
 
-    return { modifiedFilePaths, modifiedServicesIds: Array.from(new Set(modifiedFilePaths.map(DeclarationUtils.filePathToServiceId))) };
+    return { modifiedFilePaths, modifiedServicesIds: Array.from(new Set(modifiedFilePaths.map(DeclarationUtils.getServiceIdFromFilePath))) };
   }
 
   async getModifiedServices() {
@@ -33,48 +36,47 @@ export default class DeclarationUtils {
     return modifiedServicesIds;
   }
 
-  async getModifiedServiceTermsTypes() {
-    const { modifiedFilePaths, modifiedServicesIds } = await this.getModifiedData();
+  async getModifiedServicesAndTermsTypes() {
+    const { modifiedFilePaths } = await this.getModifiedData();
     const servicesTermsTypes = {};
 
     await Promise.all(modifiedFilePaths.map(async modifiedFilePath => {
-      const serviceId = DeclarationUtils.filePathToServiceId(modifiedFilePath);
+      const serviceId = DeclarationUtils.getServiceIdFromFilePath(modifiedFilePath);
 
-      if (!modifiedFilePath.endsWith('.json')) {
-        // Here we should compare AST of both files to detect on which function
-        // change has been made, and then find which terms type depends on this
-        // function.
-        // As this is a complicated process, we will just send back all terms types
-        const declaration = await this.getJSONFile(`declarations/${serviceId}.json`, this.defaultBranch);
+      if (modifiedFilePath.endsWith('.filters.js')) {
+        const declaration = await this.getJSONFromFile(this.defaultBranch, `declarations/${serviceId}.json`);
 
-        return Object.keys(declaration.documents);
+        servicesTermsTypes[serviceId] = Object.keys(declaration.documents); // Considering how rarely filters are used, simply return all term types that could potentially be impacted to spare implementing a function change check
+
+        return;
       }
 
-      const defaultFile = await this.getJSONFile(modifiedFilePath, this.defaultBranch);
-      const modifiedFile = await this.getJSONFile(modifiedFilePath, 'HEAD');
+      const originalService = await this.getJSONFromFile(this.defaultBranch, modifiedFilePath);
+      const modifiedService = await this.getJSONFromFile('HEAD', modifiedFilePath);
+      const modifiedServiceTermsTypes = Object.keys(modifiedService.documents);
 
-      const diff = DeepDiff.diff(defaultFile, modifiedFile);
+      if (!originalService) {
+        servicesTermsTypes[serviceId] = modifiedServiceTermsTypes;
 
-      if (!diff) {
-        // This can happen if only a lint has been applied to a document
         return;
       }
 
-      const modifiedTermsTypes = diff.reduce((acc, { path }) => {
-        if (modifiedFilePath.includes('.history')) {
-          acc.add(path[0]);
-        } else if (path[0] == 'documents') {
-          acc.add(path[1]);
-        }
+      const originalServiceTermsTypes = Object.keys(originalService.documents);
+
+      modifiedServiceTermsTypes.forEach(termsType => {
+        const areTermsAdded = !originalServiceTermsTypes.includes(termsType);
+        const areTermsModified = JSON.stringify(originalService.documents[termsType]) != JSON.stringify(modifiedService.documents[termsType]);
 
-        return acc;
-      }, new Set());
+        if (!areTermsAdded && !areTermsModified) {
+          return;
+        }
 
-      servicesTermsTypes[serviceId] = Array.from(new Set([ ...servicesTermsTypes[serviceId] || [], ...modifiedTermsTypes ]));
+        servicesTermsTypes[serviceId] = [termsType].concat(servicesTermsTypes[serviceId] || []);
+      });
     }));
 
     return {
-      services: modifiedServicesIds,
+      services: Object.keys(servicesTermsTypes),
       servicesTermsTypes,
     };
   }

package/scripts/declarations/utils/index.test.js

@@ -0,0 +1,154 @@
+import fs from 'fs/promises';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+import { expect } from 'chai';
+
+import DeclarationUtils from './index.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+let declarationUtils;
+
+const SUBJECT_PATH = path.resolve(__dirname, './test');
+
+const FIXTURES = {
+  serviceA: { path: './fixtures/serviceA.json' },
+  serviceATermsUpdated: { path: './fixtures/serviceATermsUpdated.json' },
+  serviceAMultipleTermsUpdated: { path: './fixtures/serviceAMultipleTermsUpdated.json' },
+  serviceATermsAdded: { path: './fixtures/serviceATermsAdded.json' },
+  serviceATermsRemoved: { path: './fixtures/serviceATermsRemoved.json' },
+  serviceB: { path: './fixtures/serviceB.json' },
+};
+
+const COMMIT_PATHS = {
+  serviceA: './declarations/ServiceA.json',
+  serviceB: './declarations/ServiceB.json',
+};
+
+const commitChanges = async (filePath, content) => {
+  await fs.writeFile(path.resolve(SUBJECT_PATH, filePath), JSON.stringify(content, null, 2));
+  await declarationUtils.git.add(filePath);
+  await declarationUtils.git.commit('Update declarations', filePath);
+};
+
+const removeLatestCommit = async () => {
+  await declarationUtils.git.reset('hard', ['HEAD~1']);
+};
+
+describe.only('DeclarationUtils', () => {
+  describe('#getModifiedServicesAndTermsTypes', () => {
+    before(async () => {
+      await loadFixtures();
+      await setupRepository();
+    });
+
+    after(() => fs.rm(SUBJECT_PATH, { recursive: true }));
+
+    context('when an existing declaration has been modified', () => {
+      before(() => commitChanges(COMMIT_PATHS.serviceA, FIXTURES.serviceATermsUpdated.content));
+      after(removeLatestCommit);
+
+      it('returns the service ID and the updated terms type', async () => {
+        expect(await declarationUtils.getModifiedServicesAndTermsTypes()).to.deep.equal({
+          services: ['ServiceA'],
+          servicesTermsTypes: { ServiceA: ['Terms of Service'] },
+        });
+      });
+    });
+
+    context('when new terms are declared in an existing declaration', () => {
+      before(() => commitChanges(COMMIT_PATHS.serviceA, FIXTURES.serviceATermsAdded.content));
+      after(removeLatestCommit);
+
+      it('returns the service ID and the added terms type', async () => {
+        expect(await declarationUtils.getModifiedServicesAndTermsTypes()).to.deep.equal({
+          services: ['ServiceA'],
+          servicesTermsTypes: { ServiceA: ['Imprint'] },
+        });
+      });
+    });
+
+    context('when a new declaration has been added', () => {
+      before(() => commitChanges(COMMIT_PATHS.serviceB, FIXTURES.serviceB.content));
+      after(removeLatestCommit);
+
+      it('returns the added service ID along with the associated terms type', async () => {
+        expect(await declarationUtils.getModifiedServicesAndTermsTypes()).to.deep.equal({
+          services: ['ServiceB'],
+          servicesTermsTypes: { ServiceB: ['Terms of Service'] },
+        });
+      });
+    });
+
+    context('when a declaration has been removed', () => {
+      before(() => removeLatestCommit(declarationUtils.git));
+      after(async () => {
+        await fs.mkdir(path.resolve(SUBJECT_PATH, './declarations'), { recursive: true });
+        await commitChanges(COMMIT_PATHS.serviceA, FIXTURES.serviceA.content);
+      });
+
+      it('returns no services and no terms types', async () => {
+        expect(await declarationUtils.getModifiedServicesAndTermsTypes()).to.deep.equal({
+          services: [],
+          servicesTermsTypes: {},
+        });
+      });
+    });
+
+    context('when terms are removed from an existing declaration', () => {
+      before(() => commitChanges(COMMIT_PATHS.serviceA, FIXTURES.serviceATermsRemoved.content));
+      after(removeLatestCommit);
+
+      it('returns no services and no terms types', async () => {
+        expect(await declarationUtils.getModifiedServicesAndTermsTypes()).to.deep.equal({
+          services: [],
+          servicesTermsTypes: {},
+        });
+      });
+    });
+
+    context('when there is a combination of an updated declaration and an added declaration', () => {
+      before(async () => {
+        await commitChanges(COMMIT_PATHS.serviceA, FIXTURES.serviceAMultipleTermsUpdated.content);
+        await commitChanges(COMMIT_PATHS.serviceB, FIXTURES.serviceB.content);
+      });
+
+      after(async () => {
+        await removeLatestCommit();
+        await removeLatestCommit();
+      });
+
+      it('returns the services IDs and the updated terms types', async () => {
+        expect(await declarationUtils.getModifiedServicesAndTermsTypes()).to.deep.equal({
+          services: [ 'ServiceA', 'ServiceB' ],
+          servicesTermsTypes: {
+            ServiceA: [ 'Imprint', 'Privacy Policy', 'Terms of Service' ],
+            ServiceB: ['Terms of Service'],
+          },
+        });
+      });
+    });
+  });
+});
+
+async function loadFixtures() {
+  await Promise.all(Object.values(FIXTURES).map(async fixture => {
+    const content = await fs.readFile(path.resolve(__dirname, fixture.path), 'utf-8');
+
+    fixture.content = JSON.parse(content);
+  }));
+}
+
+async function setupRepository() {
+  await fs.mkdir(path.resolve(SUBJECT_PATH, './declarations'), { recursive: true });
+
+  declarationUtils = new DeclarationUtils(SUBJECT_PATH, 'main');
+  await declarationUtils.git.init();
+  await declarationUtils.git.addConfig('user.name', 'Open Terms Archive Testing Bot');
+  await declarationUtils.git.addConfig('user.email', 'testing-bot@opentermsarchive.org');
+  await declarationUtils.git.checkoutLocalBranch('main');
+  await commitChanges('./README.md', 'This directory is auto-generated by test executions and requires cleanup after each test run');
+  await commitChanges(COMMIT_PATHS.serviceA, FIXTURES.serviceA.content);
+  await declarationUtils.git.checkoutLocalBranch('updated');
+}

package/scripts/declarations/validate/index.mocha.js

@@ -37,7 +37,7 @@ export default async options => {
   if (options.modified) {
     const declarationUtils = new DeclarationUtils(instancePath);
 
-    ({ services: servicesToValidate, servicesTermsTypes } = await declarationUtils.getModifiedServiceTermsTypes());
+    ({ services: servicesToValidate, servicesTermsTypes } = await declarationUtils.getModifiedServicesAndTermsTypes());
   }
 
   describe('Service declarations validation', async function () {