@opentermsarchive/engine 15.0.0 → 15.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "15.0.0",
+  "version": "15.2.0",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {

package/src/archivist/extract/exposedFilters.js

@@ -1,4 +1,35 @@
-export function removeQueryParams(webPageDOM, paramsToRemove = []) {
+const DEFAULT_TRACKING_QUERY_PARAMS = Object.freeze([
+  'utm_source', // Campaign parameters, see https://en.wikipedia.org/wiki/UTM_parameters
+  'utm_medium',
+  'utm_campaign',
+  'utm_term',
+  'utm_content',
+  'fbclid', // Click and user identifiers, see https://github.com/privacytests/privacytests/blob/dda473a462e5c37f9f2c8b2367fbc5f834796be4/live/results.js#L31-L65
+  'gclid',
+  'msclkid',
+  'mc_eid',
+  'dclid',
+  'oly_anon_id',
+  'oly_enc_id',
+  '_openstat',
+  'vero_conv',
+  'vero_id',
+  'wickedid',
+  'yclid',
+  '__s',
+  'rb_clickid',
+  's_cid',
+  'ml_subscriber',
+  'ml_subscriber_hash',
+  '_hsenc',
+  '__hssc',
+  '__hstc',
+  '__hsfp',
+  'hsCtaTracking',
+  'mkt_tok',
+]);
+
+export function removeQueryParams(webPageDOM, paramsToRemove = DEFAULT_TRACKING_QUERY_PARAMS) {
   const normalizedParams = Array.isArray(paramsToRemove) ? paramsToRemove : [paramsToRemove];
 
   if (!normalizedParams.length) {
@@ -26,3 +57,18 @@ export function removeQueryParams(webPageDOM, paramsToRemove = []) {
     }
   }
 }
+
+const SPACE_SEPARATORS = /\p{Zs}/gu;
+
+export function convertSpacesToStandard(webPageDOM) {
+  const walker = webPageDOM.createTreeWalker(webPageDOM.body, webPageDOM.defaultView.NodeFilter.SHOW_TEXT);
+
+  for (let node = walker.nextNode(); node; node = walker.nextNode()) {
+    const original = node.nodeValue;
+    const normalized = original.replace(SPACE_SEPARATORS, ' ');
+
+    if (normalized !== original) {
+      node.nodeValue = normalized;
+    }
+  }
+}

package/src/archivist/extract/exposedFilters.test.js

@@ -1,7 +1,7 @@
 import { expect } from 'chai';
 
 import createWebPageDOM from './dom.js';
-import { removeQueryParams } from './exposedFilters.js';
+import { convertSpacesToStandard, removeQueryParams } from './exposedFilters.js';
 
 describe('exposedFilters', () => {
   let webPageDOM;
@@ -51,6 +51,26 @@ describe('exposedFilters', () => {
       });
     });
 
+    describe('without parameters', () => {
+      let link;
+
+      before(() => {
+        link = webPageDOM.createElement('a');
+        link.setAttribute('href', 'https://example.com/page?utm_source=newsletter&utm_campaign=spring&fbclid=abc&gclid=def&mkt_tok=ghi&keep=value');
+        webPageDOM.body.appendChild(link);
+      });
+
+      after(() => {
+        link.remove();
+      });
+
+      it('removes well-known tracking parameters', () => {
+        removeQueryParams(webPageDOM);
+
+        expect(link.getAttribute('href')).to.equal('https://example.com/page?keep=value');
+      });
+    });
+
     describe('with string parameter', () => {
       let link;
 
@@ -265,4 +285,52 @@ fetch(trackingUrl);
       });
     });
   });
+
+  describe('#convertSpacesToStandard', () => {
+    describe('with Unicode space separators in text', () => {
+      let element;
+
+      before(() => {
+        element = webPageDOM.createElement('p');
+        element.textContent = 'a\u00A0b\u202Fc\u2009d\u3000e';
+        webPageDOM.body.appendChild(element);
+
+        convertSpacesToStandard(webPageDOM);
+      });
+
+      after(() => {
+        element.remove();
+      });
+
+      it('replaces them with a regular space', () => {
+        expect(element.textContent).to.equal('a b c d e');
+      });
+    });
+
+    describe('with Unicode space separators in an attribute', () => {
+      let element;
+      const className = 'label\u00A0primary';
+
+      before(() => {
+        element = webPageDOM.createElement('a');
+        element.setAttribute('class', className);
+        element.textContent = 'read\u00A0the\u00A0policy';
+        webPageDOM.body.appendChild(element);
+
+        convertSpacesToStandard(webPageDOM);
+      });
+
+      after(() => {
+        element.remove();
+      });
+
+      it('leaves attribute values untouched', () => {
+        expect(element.getAttribute('class')).to.equal(className);
+      });
+
+      it('replaces them in the text content', () => {
+        expect(element.textContent).to.equal('read the policy');
+      });
+    });
+  });
 });

package/src/archivist/fetcher/fullDomFetcher.test.js

@@ -79,7 +79,7 @@ describe('Full DOM Fetcher', function () {
   });
 
   describe('#fetch', () => {
-    const config = { navigationTimeout: 1000, waitForElementsTimeout: 1000, language: 'en' };
+    const config = { navigationTimeout: 5000, waitForElementsTimeout: 5000, language: 'en' };
 
     it('waits for dynamically injected elements to appear in the DOM', async () => {
       const result = await fetch(`http://127.0.0.1:${SERVER_PORT}/dynamic`, ['.dynamic'], config);

package/src/archivist/index.test.js

@@ -35,7 +35,7 @@ function resetGitRepositories() {
 }
 
 describe('Archivist', function () {
-  this.timeout(10000);
+  this.timeout(30000);
 
   const SERVICE_A_ID = 'service·A';
   const SERVICE_A_TYPE = 'Terms of Service';

package/src/archivist/services/index.js

@@ -106,7 +106,7 @@ function createWrappedFilter(baseFunction, filterName, filterParams) {
     return;
   }
 
-  if (filterParams) {
+  if (filterParams || exposedFilters[filterName]) { // Built-in filters always receive their parameters before the context, even when none are declared
     const wrappedFilter = (webPageDOM, context) => baseFunction(webPageDOM, filterParams, context);
 
     Object.defineProperty(wrappedFilter, 'name', { value: filterName });

package/src/archivist/services/index.test.js

@@ -6,6 +6,7 @@ import sinon from 'sinon';
 import sinonChai from 'sinon-chai';
 
 import expectedServices from '../../../test/fixtures/services.js';
+import createWebPageDOM from '../extract/dom.js';
 import * as exposedFilters from '../extract/exposedFilters.js';
 
 import Service from './service.js';
@@ -272,7 +273,18 @@ describe('Services', () => {
       const filterName = filterNames[0];
       const result = getServiceFilters({}, [filterName]);
 
-      expect(result).to.deep.equal([exposedFilters[filterName]]);
+      expect(result).to.have.length(1);
+      expect(result[0]).to.be.a('function');
+      expect(result[0].name).to.equal(filterName);
+    });
+
+    it('calls exposedFilters with their default parameters when declared by string name', () => {
+      const webPageDOM = createWebPageDOM('<!DOCTYPE html><html><body><a href="https://example.com/page?fbclid=abc&keep=value"></a></body></html>', 'https://example.com');
+      const [removeQueryParams] = getServiceFilters({}, ['removeQueryParams']);
+
+      removeQueryParams(webPageDOM, { fetch: 'https://example.com', select: [], remove: [], filter: ['removeQueryParams'] });
+
+      expect(webPageDOM.querySelector('a').getAttribute('href')).to.equal('https://example.com/page?keep=value');
     });
 
     it('returns filters from serviceFilters by string name', () => {
@@ -371,7 +383,7 @@ describe('Services', () => {
         expect(sourceDocument.filters).to.be.an('array');
         expect(sourceDocument.filters).to.have.length(2);
         expect(sourceDocument.filters[0]).to.be.a('function');
-        expect(sourceDocument.filters[0]).to.equal(exposedFilters[realFilterNames[0]]);
+        expect(sourceDocument.filters[0].name).to.equal(realFilterNames[0]);
         expect(sourceDocument.filters[1]).to.be.a('function');
         expect(sourceDocument.filters[1].name).to.equal('removePrintButton');
       });
@@ -431,7 +443,7 @@ describe('Services', () => {
         expect(secondSourceDocument.filters).to.be.an('array');
         expect(secondSourceDocument.filters).to.have.length(2);
         expect(secondSourceDocument.filters[0]).to.be.a('function');
-        expect(secondSourceDocument.filters[0].name).to.equal('removeQueryParams');
+        expect(secondSourceDocument.filters[0].name).to.equal(realFilterNames[0]);
         expect(secondSourceDocument.filters[1]).to.be.a('function');
         expect(secondSourceDocument.filters[1].name).to.equal('removePrintButton');
       });