@opentermsarchive/engine 15.0.0 → 15.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "15.0.0",
+  "version": "15.1.0",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {

package/src/archivist/extract/exposedFilters.js

@@ -26,3 +26,18 @@ export function removeQueryParams(webPageDOM, paramsToRemove = []) {
     }
   }
 }
+
+const SPACE_SEPARATORS = /\p{Zs}/gu;
+
+export function convertSpacesToStandard(webPageDOM) {
+  const walker = webPageDOM.createTreeWalker(webPageDOM.body, webPageDOM.defaultView.NodeFilter.SHOW_TEXT);
+
+  for (let node = walker.nextNode(); node; node = walker.nextNode()) {
+    const original = node.nodeValue;
+    const normalized = original.replace(SPACE_SEPARATORS, ' ');
+
+    if (normalized !== original) {
+      node.nodeValue = normalized;
+    }
+  }
+}

package/src/archivist/extract/exposedFilters.test.js

@@ -1,7 +1,7 @@
 import { expect } from 'chai';
 
 import createWebPageDOM from './dom.js';
-import { removeQueryParams } from './exposedFilters.js';
+import { convertSpacesToStandard, removeQueryParams } from './exposedFilters.js';
 
 describe('exposedFilters', () => {
   let webPageDOM;
@@ -265,4 +265,52 @@ fetch(trackingUrl);
       });
     });
   });
+
+  describe('#convertSpacesToStandard', () => {
+    describe('with Unicode space separators in text', () => {
+      let element;
+
+      before(() => {
+        element = webPageDOM.createElement('p');
+        element.textContent = 'a\u00A0b\u202Fc\u2009d\u3000e';
+        webPageDOM.body.appendChild(element);
+
+        convertSpacesToStandard(webPageDOM);
+      });
+
+      after(() => {
+        element.remove();
+      });
+
+      it('replaces them with a regular space', () => {
+        expect(element.textContent).to.equal('a b c d e');
+      });
+    });
+
+    describe('with Unicode space separators in an attribute', () => {
+      let element;
+      const className = 'label\u00A0primary';
+
+      before(() => {
+        element = webPageDOM.createElement('a');
+        element.setAttribute('class', className);
+        element.textContent = 'read\u00A0the\u00A0policy';
+        webPageDOM.body.appendChild(element);
+
+        convertSpacesToStandard(webPageDOM);
+      });
+
+      after(() => {
+        element.remove();
+      });
+
+      it('leaves attribute values untouched', () => {
+        expect(element.getAttribute('class')).to.equal(className);
+      });
+
+      it('replaces them in the text content', () => {
+        expect(element.textContent).to.equal('read the policy');
+      });
+    });
+  });
 });

package/src/archivist/fetcher/fullDomFetcher.test.js

@@ -79,7 +79,7 @@ describe('Full DOM Fetcher', function () {
   });
 
   describe('#fetch', () => {
-    const config = { navigationTimeout: 1000, waitForElementsTimeout: 1000, language: 'en' };
+    const config = { navigationTimeout: 5000, waitForElementsTimeout: 5000, language: 'en' };
 
     it('waits for dynamically injected elements to appear in the DOM', async () => {
       const result = await fetch(`http://127.0.0.1:${SERVER_PORT}/dynamic`, ['.dynamic'], config);

package/src/archivist/index.test.js

@@ -35,7 +35,7 @@ function resetGitRepositories() {
 }
 
 describe('Archivist', function () {
-  this.timeout(10000);
+  this.timeout(30000);
 
   const SERVICE_A_ID = 'service·A';
   const SERVICE_A_TYPE = 'Terms of Service';

package/src/archivist/services/index.test.js

@@ -431,7 +431,7 @@ describe('Services', () => {
         expect(secondSourceDocument.filters).to.be.an('array');
         expect(secondSourceDocument.filters).to.have.length(2);
         expect(secondSourceDocument.filters[0]).to.be.a('function');
-        expect(secondSourceDocument.filters[0].name).to.equal('removeQueryParams');
+        expect(secondSourceDocument.filters[0].name).to.equal(realFilterNames[0]);
         expect(secondSourceDocument.filters[1]).to.be.a('function');
         expect(secondSourceDocument.filters[1].name).to.equal('removePrintButton');
       });