@opentermsarchive/engine 7.2.0 → 7.2.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "7.2.0",
+  "version": "7.2.2",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {
@@ -63,7 +63,7 @@
     "chai-as-promised": "^7.1.1",
     "chai-exclude": "^2.1.0",
     "commander": "^9.4.1",
-    "config": "^3.3.6",
+    "config": "^4.1.1",
     "croner": "^8.0.2",
     "cronstrue": "^2.50.0",
     "cross-env": "^7.0.3",

package/scripts/declarations/validate/definitions.js

@@ -38,7 +38,7 @@ const definitions = {
         {
           type: 'object',
           description: 'Filter function with parameters. The key is the filter function name, the value is the parameters.',
-          additionalProperties: false,
+          additionalProperties: true,
           minProperties: 1,
           maxProperties: 1,
         },

package/scripts/declarations/validate/fixtures/invalid/additional-properties.json

@@ -0,0 +1,10 @@
+{
+  "name": "Service with additional properties",
+  "terms": {
+    "Privacy Policy": {
+      "fetch": "https://example.com/privacy",
+      "select": "div.content"
+    }
+  },
+  "unexpectedProperty": "value"
+}

package/scripts/declarations/validate/fixtures/invalid/invalid-filter-object.json

@@ -0,0 +1,17 @@
+{
+  "name": "Service with invalid filter object",
+  "terms": {
+    "Privacy Policy": {
+      "fetch": "https://example.com/privacy",
+      "select": "div.content",
+      "filter": [
+        {
+          "filterName": {
+            "param": "value"
+          },
+          "unexpectedAdditionalProperty": "value"
+        }
+      ]
+    }
+  }
+}

package/scripts/declarations/validate/fixtures/invalid/invalid-terms-type.json

@@ -0,0 +1,9 @@
+{
+  "name": "Service with invalid terms type",
+  "terms": {
+    "Invalid Terms Type": {
+      "fetch": "https://example.com/privacy",
+      "select": "div.content"
+    }
+  }
+}

package/scripts/declarations/validate/fixtures/invalid/invalid-url.json

@@ -0,0 +1,9 @@
+{
+  "name": "Service with invalid URL",
+  "terms": {
+    "Privacy Policy": {
+      "fetch": "not-a-valid-url",
+      "select": "div.content"
+    }
+  }
+}

package/scripts/declarations/validate/fixtures/invalid/missing-fetch.json

@@ -0,0 +1,8 @@
+{
+  "name": "Service without fetch",
+  "terms": {
+    "Privacy Policy": {
+      "select": "div.content"
+    }
+  }
+}

package/scripts/declarations/validate/fixtures/invalid/missing-name.json

@@ -0,0 +1,8 @@
+{
+  "terms": {
+    "Privacy Policy": {
+      "fetch": "https://example.com/privacy",
+      "select": "div.content"
+    }
+  }
+}

package/scripts/declarations/validate/fixtures/invalid/missing-select.json

@@ -0,0 +1,8 @@
+{
+  "name": "Service without select",
+  "terms": {
+    "Privacy Policy": {
+      "fetch": "https://example.com/privacy"
+    }
+  }
+}

package/scripts/declarations/validate/fixtures/invalid/missing-terms.json

@@ -0,0 +1,3 @@
+{
+  "name": "Service without terms"
+}

package/scripts/declarations/validate/fixtures/invalid/non-pdf-url-for-pdf.json

@@ -0,0 +1,8 @@
+{
+  "name": "Service with non-PDF URL for PDF document",
+  "terms": {
+    "Privacy Policy": {
+      "fetch": "https://example.com/privacy.html"
+    }
+  }
+}

package/scripts/declarations/validate/fixtures/invalid-history/invalid-date.json

@@ -0,0 +1,9 @@
+{
+  "Privacy Policy": [
+    {
+      "fetch": "https://example.com/privacy",
+      "select": "div.content",
+      "validUntil": "2023-12-31"
+    }
+  ]
+}

package/scripts/declarations/validate/fixtures/invalid-history/missing-valid-until.json

@@ -0,0 +1,8 @@
+{
+  "Privacy Policy": [
+    {
+      "fetch": "https://example.com/privacy",
+      "select": "div.content"
+    }
+  ]
+}

package/scripts/declarations/validate/fixtures/valid/complex-service.json

@@ -0,0 +1,30 @@
+{
+  "name": "Complex service",
+  "terms": {
+    "Privacy Policy": {
+      "fetch": "https://example.com/privacy",
+      "select": [
+        "div.content",
+        ".privacy-section"
+      ],
+      "filter": [
+        "removeLinks",
+        {
+          "customTransform": {
+            "options": {
+              "cleanup": true
+            }
+          }
+        }
+      ],
+      "remove": [
+        ".ads",
+        ".tracking"
+      ],
+      "executeClientScripts": true
+    },
+    "Terms of Service": {
+      "fetch": "https://example.com/terms.pdf"
+    }
+  }
+}

package/scripts/declarations/validate/fixtures/valid/minimal-service.json

@@ -0,0 +1,9 @@
+{
+  "name": "Minimal service",
+  "terms": {
+    "Privacy Policy": {
+      "fetch": "https://example.com/privacy",
+      "select": "div.content"
+    }
+  }
+}

package/scripts/declarations/validate/fixtures/valid/multiple-source-documents.json

@@ -0,0 +1,24 @@
+{
+  "name": "Multiple source documents service",
+  "terms": {
+    "Privacy Policy": {
+      "combine": [
+        {
+          "fetch": "https://example.com/privacy1",
+          "select": "div.content",
+          "filter": [
+            "removeLinks"
+          ]
+        },
+        {
+          "fetch": "https://example.com/privacy2",
+          "select": ".privacy-section"
+        }
+      ],
+      "select": "body",
+      "filter": [
+        "trimWhitespace"
+      ]
+    }
+  }
+}

package/scripts/declarations/validate/fixtures/valid/pdf-document.json

@@ -0,0 +1,8 @@
+{
+  "name": "Service with PDF terms",
+  "terms": {
+    "Privacy Policy": {
+      "fetch": "https://example.com/privacy.pdf"
+    }
+  }
+}

package/scripts/declarations/validate/fixtures/valid/range-selectors.json

@@ -0,0 +1,19 @@
+{
+  "name": "Service with range selectors",
+  "terms": {
+    "Privacy Policy": {
+      "fetch": "https://example.com/privacy",
+      "select": [
+        "div.content",
+        {
+          "startBefore": "h1",
+          "endAfter": "footer"
+        },
+        {
+          "startAfter": "h2",
+          "endBefore": ".end-marker"
+        }
+      ]
+    }
+  }
+}

package/scripts/declarations/validate/fixtures/valid/service-with-filters.json

@@ -0,0 +1,22 @@
+{
+  "name": "Service with filters",
+  "terms": {
+    "Privacy Policy": {
+      "fetch": "https://example.com/privacy",
+      "select": "div.content",
+      "filter": [
+        "removeLinks",
+        {
+          "customFilter": {
+            "param": "value"
+          }
+        },
+        {
+          "anotherFilter": {
+            "setting": true
+          }
+        }
+      ]
+    }
+  }
+}

package/scripts/declarations/validate/fixtures/valid-history/pdf-history.json

@@ -0,0 +1,8 @@
+{
+  "Privacy Policy": [
+    {
+      "fetch": "https://example.com/privacy.pdf",
+      "validUntil": "2023-12-31T23:59:59Z"
+    }
+  ]
+}

package/scripts/declarations/validate/fixtures/valid-history/simple-history.json

@@ -0,0 +1,9 @@
+{
+  "Privacy Policy": [
+    {
+      "fetch": "https://example.com/privacy",
+      "select": "div.content",
+      "validUntil": "2023-12-31T23:59:59Z"
+    }
+  ]
+}

package/scripts/declarations/validate/schemas.test.js

@@ -0,0 +1,84 @@
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+import Ajv from 'ajv';
+import addFormats from 'ajv-formats';
+import { expect } from 'chai';
+
+import serviceHistorySchema from './service.history.schema.js';
+import serviceSchema from './service.schema.js';
+
+describe('Schema validation', () => {
+  let validator;
+  const __dirname = path.dirname(fileURLToPath(import.meta.url));
+  const fixturesDir = path.join(__dirname, 'fixtures');
+
+  before(() => {
+    validator = new Ajv({ allErrors: true });
+    addFormats(validator);
+  });
+
+  const loadFixture = filePath => {
+    const content = fs.readFileSync(filePath, 'utf8');
+
+    return JSON.parse(content);
+  };
+
+  const getJsonFiles = dir => {
+    const files = fs.readdirSync(dir);
+
+    return files.filter(file => file.endsWith('.json'));
+  };
+
+  const validateDeclaration = (schema, declaration, file, shouldBeValid) => {
+    const valid = validator.validate(schema, declaration);
+
+    if (!valid && shouldBeValid) {
+      console.error(`Validation errors for ${file}:`, validator.errors);
+    }
+
+    return valid;
+  };
+
+  const createValidationTests = (schema, directory, shouldBeValid) => {
+    const files = getJsonFiles(directory);
+    const action = shouldBeValid ? 'validates' : 'rejects';
+
+    files.forEach(file => {
+      it(`${action} ${file}`, () => {
+        const filePath = path.join(directory, file);
+        const declaration = loadFixture(filePath);
+        const valid = validateDeclaration(schema, declaration, file, shouldBeValid);
+
+        expect(valid, `${file} should be ${shouldBeValid ? 'valid' : 'invalid'}`).to.equal(shouldBeValid);
+      });
+    });
+  };
+
+  describe('Service schema', () => {
+    const validDir = path.join(fixturesDir, 'valid');
+    const invalidDir = path.join(fixturesDir, 'invalid');
+
+    describe('with valid declarations', () => {
+      createValidationTests(serviceSchema, validDir, true);
+    });
+
+    describe('with invalid declarations', () => {
+      createValidationTests(serviceSchema, invalidDir, false);
+    });
+  });
+
+  describe('Service history schema', () => {
+    const validHistoryDir = path.join(fixturesDir, 'valid-history');
+    const invalidHistoryDir = path.join(fixturesDir, 'invalid-history');
+
+    describe('with valid history declarations', () => {
+      createValidationTests(serviceHistorySchema, validHistoryDir, true);
+    });
+
+    describe('with invalid history declarations', () => {
+      createValidationTests(serviceHistorySchema, invalidHistoryDir, false);
+    });
+  });
+});