@opentermsarchive/engine 7.1.0 → 7.2.0

package/README.md

@@ -4,6 +4,12 @@ This codebase is a Node.js module enabling downloading, archiving and publishing
 
 For documentation, visit [docs.opentermsarchive.org](https://docs.opentermsarchive.org/)
 
+## Testing
+
+Use `npm test` to run all tests.
+
+Use `npm run test:only <file.test.js>... [--watch]` to run specific test files. The `--watch` option enables running those tests each time a file changes.
+
 - - -
 
 ## Contribute

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "7.1.0",
+  "version": "7.2.0",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {
@@ -45,6 +45,7 @@
     "start:api": "node bin/ota.js serve",
     "start:scheduler": "npm start -- --schedule",
     "test": "cross-env NODE_ENV=test mocha --recursive \"./src/**/*.test.js\" \"./scripts/**/*.test.js\" --exit",
+    "test:only": "cross-env NODE_ENV=test mocha --recursive",
     "posttest": "npm run lint",
     "test:debug": "npm run test -- --inspect-brk --exit"
   },
@@ -122,6 +123,6 @@
     "@opentermsarchive/terms-types": "^2.0.0"
   },
   "engines": {
-    "node": ">=16.0.0"
+    "node": ">=16.0.0 < 23.0.0"
   }
 }

package/scripts/declarations/validate/definitions.js

@@ -29,9 +29,20 @@ const definitions = {
   filters: {
     type: 'array',
     items: {
-      type: 'string',
-      pattern: '^.+$',
-      description: 'Filter function name',
+      oneOf: [
+        {
+          type: 'string',
+          pattern: '^.+$',
+          description: 'Filter function name',
+        },
+        {
+          type: 'object',
+          description: 'Filter function with parameters. The key is the filter function name, the value is the parameters.',
+          additionalProperties: false,
+          minProperties: 1,
+          maxProperties: 1,
+        },
+      ],
     },
   },
   validUntil: {

package/scripts/declarations/validate/index.mocha.js

@@ -7,6 +7,7 @@ import { expect } from 'chai';
 import config from 'config';
 import jsonSourceMap from 'json-source-map';
 
+import * as exposedFilters from '../../../src/archivist/extract/exposedFilters.js';
 import extract from '../../../src/archivist/extract/index.js';
 import fetch, { launchHeadlessBrowser, stopHeadlessBrowser } from '../../../src/archivist/fetcher/index.js';
 import * as services from '../../../src/archivist/services/index.js';
@@ -76,6 +77,24 @@ export default async options => {
           });
         }
 
+        it('filters do not use reserved names', async () => {
+          const filtersFilePath = path.join(declarationsPath, `${serviceId}.filters.js`);
+
+          if (!fsApi.existsSync(filtersFilePath)) {
+            return; // Skip if no filters file exists
+          }
+
+          const serviceFilters = await services.loadServiceFilters(serviceId);
+          const reservedFilterNames = Object.keys(exposedFilters);
+          const serviceFilterNames = Object.keys(serviceFilters);
+
+          const conflictingNames = serviceFilterNames.filter(name => reservedFilterNames.includes(name));
+
+          if (conflictingNames.length) {
+            throw new Error(`Service filter file "${serviceId}.filters.js" declares filters with names used by built-in filters: "${conflictingNames.join('", "')}". Rename these filters to avoid a collision.`);
+          }
+        });
+
         if (!schemaOnly && service) {
           service.getTermsTypes()
             .filter(termsType => {

package/src/archivist/extract/dom.js

@@ -0,0 +1,75 @@
+import jsdom from 'jsdom';
+
+export default function createWebPageDOM(content, location) {
+  const { document } = new jsdom.JSDOM(content, {
+    url: location,
+    virtualConsole: new jsdom.VirtualConsole(),
+  }).window;
+
+  return Object.assign(document, {
+    select(contentSelectors) {
+      const result = document.createDocumentFragment();
+      let hasContent = false;
+
+      [].concat(contentSelectors).forEach(selector => {
+        if (typeof selector === 'object') {
+          const rangeSelection = this.selectRange(selector);
+          const clonedContent = rangeSelection.cloneContents();
+
+          if (clonedContent.hasChildNodes()) {
+            result.appendChild(clonedContent);
+            hasContent = true;
+          }
+        } else {
+          const elements = document.querySelectorAll(selector);
+
+          if (elements.length > 0) {
+            elements.forEach(element => result.appendChild(element.cloneNode(true)));
+            hasContent = true;
+          }
+        }
+      });
+
+      return hasContent ? result : null;
+    },
+
+    remove(insignificantContentSelectors) {
+      const rangeSelections = [];
+      const nodes = [];
+
+      [].concat(insignificantContentSelectors).forEach(selector => {
+        if (typeof selector === 'object') {
+          rangeSelections.push(this.selectRange(selector));
+        } else {
+          nodes.push(...document.querySelectorAll(selector));
+        }
+      });
+
+      nodes.forEach(node => node.remove());
+      rangeSelections.forEach(rangeSelection => rangeSelection.deleteContents());
+
+      return this;
+    },
+
+    selectRange(rangeSelector) {
+      const { startBefore, startAfter, endBefore, endAfter } = rangeSelector;
+
+      const selection = document.createRange();
+      const startNode = document.querySelector(startBefore || startAfter);
+      const endNode = document.querySelector(endBefore || endAfter);
+
+      if (!startNode) {
+        throw new Error(`The "start" selector has no match in document in: ${JSON.stringify(rangeSelector)}`);
+      }
+
+      if (!endNode) {
+        throw new Error(`The "end" selector has no match in document in: ${JSON.stringify(rangeSelector)}`);
+      }
+
+      selection[startBefore ? 'setStartBefore' : 'setStartAfter'](startNode);
+      selection[endBefore ? 'setEndBefore' : 'setEndAfter'](endNode);
+
+      return selection;
+    },
+  });
+}

package/src/archivist/extract/dom.test.js

@@ -0,0 +1,207 @@
+import { expect } from 'chai';
+
+import createWebPageDOM from './dom.js';
+
+describe('createWebPageDOM', () => {
+  const sampleHTML = `
+    <!DOCTYPE html>
+    <html>
+      <head>
+        <title>Test Document</title>
+      </head>
+      <body>
+        <header id="header">
+          <h1>Main Title</h1>
+          <nav class="navigation">
+            <a href="/home">Home</a>
+            <a href="/about">About</a>
+          </nav>
+        </header>
+        <main>
+          <article id="content">
+            <p class="introduction">Introduction paragraph</p>
+            <p class="central">Central paragraph</p>
+            <p class="conclusion">Conclusion paragraph</p>
+          </article>
+          <aside class="sidebar">
+            <div class="widget">Widget content</div>
+          </aside>
+        </main>
+        <footer id="footer">
+          <p>Footer content</p>
+        </footer>
+      </body>
+    </html>
+  `;
+  const location = 'https://example.com/test';
+  let document;
+
+  before(() => {
+    document = createWebPageDOM(sampleHTML, location);
+  });
+
+  it('creates a DOM document from HTML content', () => {
+    expect(document.documentElement.tagName).to.equal('HTML');
+  });
+
+  it('sets the document location', () => {
+    expect(document.location.href).to.equal(location);
+  });
+
+  it('provides access to the DOM API', () => {
+    const title = document.querySelector('title');
+
+    expect(title.textContent).to.equal('Test Document');
+  });
+
+  describe('#select', () => {
+    it('returns elements using CSS selectors', () => {
+      const fragment = document.select('p.introduction');
+      const paragraph = fragment.querySelector('p');
+
+      expect(paragraph.textContent).to.equal('Introduction paragraph');
+    });
+
+    it('returns multiple elements using CSS selectors', () => {
+      const fragment = document.select('p');
+      const paragraphs = fragment.querySelectorAll('p');
+
+      expect(paragraphs.length).to.equal(4);
+    });
+
+    it('returns elements using an array of CSS selectors', () => {
+      const fragment = document.select([ 'h1', '.introduction' ]);
+      const heading = fragment.querySelector('h1');
+      const paragraph = fragment.querySelector('p');
+
+      expect(heading.textContent).to.equal('Main Title');
+      expect(paragraph.textContent).to.equal('Introduction paragraph');
+    });
+
+    it('returns content using a range selector object', () => {
+      const rangeSelector = {
+        startAfter: '.introduction',
+        endBefore: '.conclusion',
+      };
+      const fragment = document.select(rangeSelector);
+      const paragraph = fragment.querySelector('p');
+
+      expect(paragraph.textContent).to.equal('Central paragraph');
+    });
+
+    it('returns null when the selector matches no element', () => {
+      const result = document.select('.nonexistent');
+
+      expect(result).to.be.null;
+    });
+  });
+
+  describe('#remove', () => {
+    let testDocument;
+
+    beforeEach(() => {
+      testDocument = createWebPageDOM(sampleHTML, location);
+    });
+
+    it('removes elements using CSS selectors', () => {
+      testDocument.remove('.sidebar');
+      const sidebar = testDocument.querySelector('.sidebar');
+
+      expect(sidebar).to.be.null;
+    });
+
+    it('removes multiple elements using CSS selectors', () => {
+      testDocument.remove('p');
+      const paragraphs = testDocument.querySelectorAll('p');
+
+      expect(paragraphs.length).to.equal(0);
+    });
+
+    it('removes elements using an array of CSS selectors', () => {
+      testDocument.remove([ 'nav', '.widget' ]);
+      const nav = testDocument.querySelector('nav');
+      const widget = testDocument.querySelector('.widget');
+
+      expect(nav).to.be.null;
+      expect(widget).to.be.null;
+    });
+
+    it('removes content using a range selector object', () => {
+      const rangeSelector = {
+        startAfter: '.introduction',
+        endBefore: '.conclusion',
+      };
+
+      testDocument.remove(rangeSelector);
+      const bodyParagraph = testDocument.querySelector('.central');
+
+      expect(bodyParagraph).to.be.null;
+    });
+  });
+
+  describe('#selectRange', () => {
+    it('creates a range using startAfter and endBefore', () => {
+      const rangeSelector = {
+        startAfter: '.introduction',
+        endBefore: '.conclusion',
+      };
+      const range = document.selectRange(rangeSelector);
+      const fragment = range.cloneContents();
+      const paragraph = fragment.querySelector('p');
+
+      expect(paragraph.textContent).to.equal('Central paragraph');
+    });
+
+    it('creates a range using startBefore and endAfter', () => {
+      const rangeSelector = {
+        startBefore: '.central',
+        endAfter: '.central',
+      };
+      const range = document.selectRange(rangeSelector);
+      const fragment = range.cloneContents();
+      const paragraph = fragment.querySelector('p');
+
+      expect(paragraph.textContent).to.equal('Central paragraph');
+    });
+
+    it('throws a clear error when the startBefore selector has no match', () => {
+      const rangeSelector = {
+        startBefore: '.nonexistent',
+        endBefore: '.conclusion',
+      };
+
+      expect(() => document.selectRange(rangeSelector)).to.throw('"start" selector has no match');
+      expect(() => document.selectRange(rangeSelector)).to.throw(JSON.stringify(rangeSelector));
+    });
+
+    it('throws a clear error when the startAfter selector has no match', () => {
+      const rangeSelector = {
+        startAfter: '.nonexistent',
+        endBefore: '.conclusion',
+      };
+
+      expect(() => document.selectRange(rangeSelector)).to.throw('"start" selector has no match');
+      expect(() => document.selectRange(rangeSelector)).to.throw(JSON.stringify(rangeSelector));
+    });
+
+    it('throws a clear error when the endBefore selector has no match', () => {
+      const rangeSelector = {
+        startAfter: '.introduction',
+        endBefore: '.nonexistent',
+      };
+
+      expect(() => document.selectRange(rangeSelector)).to.throw('"end" selector has no match');
+      expect(() => document.selectRange(rangeSelector)).to.throw(JSON.stringify(rangeSelector));
+    });
+
+    it('throws a clear error when the endAfter selector has no match', () => {
+      const rangeSelector = {
+        startAfter: '.introduction',
+        endAfter: '.nonexistent',
+      };
+
+      expect(() => document.selectRange(rangeSelector)).to.throw('"end" selector has no match');
+      expect(() => document.selectRange(rangeSelector)).to.throw(JSON.stringify(rangeSelector));
+    });
+  });
+});

package/src/archivist/extract/exposedFilters.js

@@ -0,0 +1,25 @@
+export function removeQueryParams(webPageDOM, paramsToRemove = []) {
+  if (typeof paramsToRemove === 'string') {
+    paramsToRemove = [paramsToRemove];
+  }
+
+  if (!paramsToRemove.length) {
+    return;
+  }
+
+  const elements = [
+    ...webPageDOM.querySelectorAll('a[href]'),
+    ...webPageDOM.querySelectorAll('img[src]'),
+  ];
+
+  elements.forEach(element => {
+    try {
+      const url = new URL(element.href || element.src);
+
+      paramsToRemove.forEach(param => url.searchParams.delete(param));
+      element[element.tagName === 'A' ? 'href' : 'src'] = url.toString();
+    } catch (error) {
+      // ignore if the element has not a valid URL
+    }
+  });
+}

package/src/archivist/extract/exposedFilters.test.js

@@ -0,0 +1,208 @@
+import { expect } from 'chai';
+
+import createWebPageDOM from './dom.js';
+import { removeQueryParams } from './exposedFilters.js';
+
+describe('exposedFilters', () => {
+  let webPageDOM;
+
+  before(() => {
+    webPageDOM = createWebPageDOM('<!DOCTYPE html><html><body></body></html>');
+  });
+
+  describe('#removeQueryParams', () => {
+    describe('from links', () => {
+      let link;
+
+      before(() => {
+        link = webPageDOM.createElement('a');
+        link.href = 'https://example.com/page?utm_source=test&keep=value';
+        webPageDOM.body.appendChild(link);
+      });
+
+      after(() => {
+        link.remove();
+      });
+
+      it('removes the specified query parameters', () => {
+        removeQueryParams(webPageDOM, ['utm_source']);
+
+        expect(link.href).to.equal('https://example.com/page?keep=value');
+      });
+    });
+
+    describe('from images', () => {
+      let img;
+
+      before(() => {
+        img = webPageDOM.createElement('img');
+        img.src = 'https://example.com/image.jpg?width=100&keep=value';
+        webPageDOM.body.appendChild(img);
+      });
+
+      after(() => {
+        img.remove();
+      });
+
+      it('removes the specified query parameters', () => {
+        removeQueryParams(webPageDOM, ['width']);
+
+        expect(img.src).to.equal('https://example.com/image.jpg?keep=value');
+      });
+    });
+
+    describe('with string parameter', () => {
+      let link;
+
+      before(() => {
+        link = webPageDOM.createElement('a');
+        link.href = 'https://example.com/page?utm_source=test&keep=value';
+        webPageDOM.body.appendChild(link);
+      });
+
+      after(() => {
+        link.remove();
+      });
+
+      it('removes a single query parameter passed as string', () => {
+        removeQueryParams(webPageDOM, 'utm_source');
+
+        expect(link.href).to.equal('https://example.com/page?keep=value');
+      });
+    });
+
+    describe('with empty parameters', () => {
+      let link;
+
+      before(() => {
+        link = webPageDOM.createElement('a');
+        link.href = 'https://example.com/page?utm_source=test&keep=value';
+        webPageDOM.body.appendChild(link);
+      });
+
+      after(() => {
+        link.remove();
+      });
+
+      it('leaves the URL unchanged', () => {
+        removeQueryParams(webPageDOM, []);
+
+        expect(link.href).to.equal('https://example.com/page?utm_source=test&keep=value');
+      });
+    });
+
+    describe('with invalid URLs', () => {
+      let link;
+
+      before(() => {
+        link = webPageDOM.createElement('a');
+        link.href = 'ht^THIS_IS_WRONG^tp://example.com?utm_source=test';
+        webPageDOM.body.appendChild(link);
+      });
+
+      after(() => {
+        link.remove();
+      });
+
+      it('ignores elements with invalid URLs', () => {
+        removeQueryParams(webPageDOM, ['utm_source']);
+
+        expect(link.href).to.equal('ht^THIS_IS_WRONG^tp://example.com?utm_source=test');
+      });
+    });
+
+    describe('with multiple parameters', () => {
+      let link;
+
+      before(() => {
+        link = webPageDOM.createElement('a');
+        link.href = 'https://example.com/page?utm_source=test&utm_medium=email&keep=value&remove=me';
+        webPageDOM.body.appendChild(link);
+      });
+
+      after(() => {
+        link.remove();
+      });
+
+      it('removes all specified query parameters', () => {
+        removeQueryParams(webPageDOM, [ 'utm_source', 'utm_medium', 'remove' ]);
+
+        expect(link.href).to.equal('https://example.com/page?keep=value');
+      });
+    });
+
+    describe('with duplicate parameters', () => {
+      let link;
+
+      before(() => {
+        link = webPageDOM.createElement('a');
+        link.href = 'https://example.com/test?utm_source=to_remove_1&keep=true&utm_source=to_remove_2';
+        webPageDOM.body.appendChild(link);
+      });
+
+      after(() => {
+        link.remove();
+      });
+
+      it('removes all instances of duplicate query parameters', () => {
+        removeQueryParams(webPageDOM, ['utm_source']);
+
+        expect(link.href).to.equal('https://example.com/test?keep=true');
+      });
+    });
+
+    describe('textual content preservation', () => {
+      let codeElement;
+      let paragraphElement;
+      let preElement;
+
+      before(() => {
+        codeElement = webPageDOM.createElement('code');
+        codeElement.textContent = 'https://example.com/track?utm_source=newsletter&utm_campaign=winter';
+        webPageDOM.body.appendChild(codeElement);
+
+        paragraphElement = webPageDOM.createElement('p');
+        paragraphElement.textContent = 'When users click on links with utm_source=email or utm_medium=social, we track their behavior using https://analytics.example.com?utm_source=website&session_id=abc123.';
+        webPageDOM.body.appendChild(paragraphElement);
+
+        preElement = webPageDOM.createElement('pre');
+        preElement.textContent = `
+// Example tracking implementation
+const trackingUrl = 'https://tracker.com/pixel?utm_source=app&user_id=123';
+fetch(trackingUrl);
+        `;
+        webPageDOM.body.appendChild(preElement);
+      });
+
+      after(() => {
+        codeElement.remove();
+        paragraphElement.remove();
+        preElement.remove();
+      });
+
+      it('preserves code element URLs with tracking parameters', () => {
+        const originalCodeContent = codeElement.textContent;
+
+        removeQueryParams(webPageDOM, [ 'utm_source', 'utm_campaign', 'utm_medium', 'session_id', 'user_id' ]);
+
+        expect(codeElement.textContent).to.equal(originalCodeContent);
+      });
+
+      it('preserves paragraph element URLs with tracking parameters', () => {
+        const originalParagraphContent = paragraphElement.textContent;
+
+        removeQueryParams(webPageDOM, [ 'utm_source', 'utm_campaign', 'utm_medium', 'session_id', 'user_id' ]);
+
+        expect(paragraphElement.textContent).to.equal(originalParagraphContent);
+      });
+
+      it('preserves preformatted element URLs with tracking parameters', () => {
+        const originalPreContent = preElement.textContent;
+
+        removeQueryParams(webPageDOM, [ 'utm_source', 'utm_campaign', 'utm_medium', 'session_id', 'user_id' ]);
+
+        expect(preElement.textContent).to.equal(originalPreContent);
+      });
+    });
+  });
+});

package/src/archivist/extract/filter.js

@@ -0,0 +1,59 @@
+export const LINKS_TO_CONVERT_SELECTOR = 'a[href]:not([href^="#"]):not([href=""])';
+
+export default async function filter(webPageDOM, sourceDocument) {
+  await applyCustomFilters(webPageDOM, sourceDocument);
+  convertRelativeURLsToAbsolute(webPageDOM, sourceDocument.location);
+  discardNonTextualElements(webPageDOM);
+  cleanEmailProtectedLinks(webPageDOM);
+
+  return webPageDOM;
+}
+
+async function applyCustomFilters(webPageDOM, sourceDocument) {
+  const { location, contentSelectors = [], insignificantContentSelectors = [], filters: serviceSpecificFilters = [] } = sourceDocument;
+
+  for (const filterFunction of serviceSpecificFilters) {
+    try {
+      await filterFunction(webPageDOM, {
+        fetch: location,
+        select: contentSelectors,
+        remove: insignificantContentSelectors,
+        filter: serviceSpecificFilters.map(filter => filter.name),
+      });
+    } catch (error) {
+      throw new Error(`The filter function "${filterFunction.name}" failed: ${error}`);
+    }
+  }
+}
+
+function convertRelativeURLsToAbsolute(webPageDOM, baseURL) {
+  Array.from(webPageDOM.querySelectorAll(LINKS_TO_CONVERT_SELECTOR)).forEach(link => {
+    try {
+      link.href = new URL(link.href, baseURL).href;
+    } catch (error) {
+      // Leave the URL as is if it's invalid in the source document and can't be converted to an absolute URL
+    }
+  });
+}
+
+function discardNonTextualElements(webPageDOM) {
+  webPageDOM.querySelectorAll('script, style').forEach(node => node.remove());
+}
+
+function cleanEmailProtectedLinks(webPageDOM) {
+  webPageDOM.querySelectorAll('a[href*="/email-protection"]').forEach(node => {
+    const replacement = webPageDOM.createElement('a');
+    const [href] = node.href.split('#');
+
+    Array.from(node.attributes).forEach(attr => {
+      if (attr.name === 'href') {
+        replacement.setAttribute('href', href);
+      } else {
+        replacement.setAttribute(attr.name, attr.value);
+      }
+    });
+
+    replacement.innerHTML = '[email protected]';
+    node.parentNode.replaceChild(replacement, node);
+  });
+}