@opentermsarchive/engine 14.1.0 → 15.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "14.1.0",
+  "version": "15.1.0",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {

package/src/archivist/extract/exposedFilters.js

@@ -26,3 +26,18 @@ export function removeQueryParams(webPageDOM, paramsToRemove = []) {
     }
   }
 }
+
+const SPACE_SEPARATORS = /\p{Zs}/gu;
+
+export function convertSpacesToStandard(webPageDOM) {
+  const walker = webPageDOM.createTreeWalker(webPageDOM.body, webPageDOM.defaultView.NodeFilter.SHOW_TEXT);
+
+  for (let node = walker.nextNode(); node; node = walker.nextNode()) {
+    const original = node.nodeValue;
+    const normalized = original.replace(SPACE_SEPARATORS, ' ');
+
+    if (normalized !== original) {
+      node.nodeValue = normalized;
+    }
+  }
+}

package/src/archivist/extract/exposedFilters.test.js

@@ -1,7 +1,7 @@
 import { expect } from 'chai';
 
 import createWebPageDOM from './dom.js';
-import { removeQueryParams } from './exposedFilters.js';
+import { convertSpacesToStandard, removeQueryParams } from './exposedFilters.js';
 
 describe('exposedFilters', () => {
   let webPageDOM;
@@ -265,4 +265,52 @@ fetch(trackingUrl);
       });
     });
   });
+
+  describe('#convertSpacesToStandard', () => {
+    describe('with Unicode space separators in text', () => {
+      let element;
+
+      before(() => {
+        element = webPageDOM.createElement('p');
+        element.textContent = 'a\u00A0b\u202Fc\u2009d\u3000e';
+        webPageDOM.body.appendChild(element);
+
+        convertSpacesToStandard(webPageDOM);
+      });
+
+      after(() => {
+        element.remove();
+      });
+
+      it('replaces them with a regular space', () => {
+        expect(element.textContent).to.equal('a b c d e');
+      });
+    });
+
+    describe('with Unicode space separators in an attribute', () => {
+      let element;
+      const className = 'label\u00A0primary';
+
+      before(() => {
+        element = webPageDOM.createElement('a');
+        element.setAttribute('class', className);
+        element.textContent = 'read\u00A0the\u00A0policy';
+        webPageDOM.body.appendChild(element);
+
+        convertSpacesToStandard(webPageDOM);
+      });
+
+      after(() => {
+        element.remove();
+      });
+
+      it('leaves attribute values untouched', () => {
+        expect(element.getAttribute('class')).to.equal(className);
+      });
+
+      it('replaces them in the text content', () => {
+        expect(element.textContent).to.equal('read the policy');
+      });
+    });
+  });
 });

package/src/archivist/fetcher/fullDomFetcher.test.js

@@ -79,7 +79,7 @@ describe('Full DOM Fetcher', function () {
   });
 
   describe('#fetch', () => {
-    const config = { navigationTimeout: 1000, waitForElementsTimeout: 1000, language: 'en' };
+    const config = { navigationTimeout: 5000, waitForElementsTimeout: 5000, language: 'en' };
 
     it('waits for dynamically injected elements to appear in the DOM', async () => {
       const result = await fetch(`http://127.0.0.1:${SERVER_PORT}/dynamic`, ['.dynamic'], config);

package/src/archivist/index.test.js

@@ -35,7 +35,7 @@ function resetGitRepositories() {
 }
 
 describe('Archivist', function () {
-  this.timeout(10000);
+  this.timeout(30000);
 
   const SERVICE_A_ID = 'service·A';
   const SERVICE_A_TYPE = 'Terms of Service';

package/src/archivist/recorder/repositories/git/dataMapper.js

@@ -30,6 +30,10 @@ const MULTIPLE_SOURCE_DOCUMENTS_PREFIX = 'This version was recorded after extrac
 
 export const COMMIT_MESSAGE_PREFIXES_REGEXP = new RegExp(`^(${Object.values(COMMIT_MESSAGE_PREFIXES).join('|')})`);
 
+export function isTechnicalUpgrade(message) {
+  return message.startsWith(COMMIT_MESSAGE_PREFIXES.technicalUpgrade) || message.startsWith(COMMIT_MESSAGE_PREFIXES.deprecated_refilter);
+}
+
 export function toPersistence(record, snapshotIdentiferTemplate) {
   const { serviceId, termsType, documentId, snapshotIds = [], mimeType, metadata } = record;
 

package/src/archivist/recorder/repositories/git/git.js

@@ -22,7 +22,7 @@ export default class Git {
 
     this.git = simpleGit(this.path, {
       trimmed: true,
-      maxConcurrentProcesses: 1,
+      maxConcurrentProcesses: 1, // Concurrent runs on the same repository race the index and the commit-graph and can corrupt them
     });
 
     await this.git.init();
@@ -106,7 +106,8 @@ export default class Git {
 
       return commits;
     } catch (error) {
-      if (/unknown revision or path not in the working tree|does not have any commits yet/.test(error.message)) {
+      // `bad object` is raised for a well-formed but absent object ID; like an unknown revision, it means "no match" rather than a hard failure
+      if (/unknown revision or path not in the working tree|does not have any commits yet|bad object/.test(error.message)) {
         return [];
       }
 
@@ -132,15 +133,24 @@ export default class Git {
     await fs.rm(path.join(this.path, '.git', 'objects', 'info', 'commit-graph.lock'), { force: true }); // Remove a leftover commit-graph lock from a previous `commit-graph write` that was killed mid-write (e.g. the process was terminated during a deploy or restart). The commit-graph is a disposable cache rebuilt by `writeCommitGraph`, so clearing a stale lock is safe and prevents every subsequent run from failing.
     await this.git.reset('hard');
 
-    return this.git.clean('f', '-d');
+    return this.git.clean('f', '-d'); // Force-remove untracked files (`f`) and untracked directories (`-d`)
   }
 
   getFullHash(shortHash) {
-    return this.git.show([ shortHash, '--pretty=%H', '-s' ]);
+    return this.git.show([
+      shortHash,
+      '--pretty=%H', // Print the full 40-character commit hash
+      '-s', // Suppress the diff output, only the formatted hash is wanted
+    ]);
   }
 
   restore(path, commit) {
-    return this.git.raw([ 'restore', '-s', commit, '--', path ]);
+    return this.git.raw([
+      'restore',
+      '-s', commit, // Take the file contents from this specific commit rather than from the index
+      '--', // Everything after is a pathspec, not a revision or option
+      path,
+    ]);
   }
 
   async destroyHistory() {
@@ -154,14 +164,86 @@ export default class Git {
   }
 
   async listFiles(path) {
-    return (await this.git.raw([ 'ls-files', path ])).split('\n');
+    return (await this.git.raw([ 'ls-files', '--', path ])).split('\n'); // Everything after "--" is a pathspec, not a revision or option
   }
 
   async writeCommitGraph() {
-    await this.git.raw([ 'commit-graph', 'write', '--reachable', '--changed-paths' ]);
+    await this.git.raw([
+      'commit-graph',
+      'write',
+      '--reachable', // Cover every commit reachable from the refs, so the whole history is indexed
+      '--changed-paths', // Also store the changed-path Bloom filters that speed up path-limited log/diff
+    ]);
   }
 
   async updateCommitGraph() {
-    await this.git.raw([ 'commit-graph', 'write', '--reachable', '--changed-paths', '--append' ]);
+    await this.git.raw([
+      'commit-graph',
+      'write',
+      '--reachable',
+      '--changed-paths',
+      '--append', // Extend the existing commit-graph instead of rewriting it in full
+    ]);
+  }
+
+  async listPathRevisions(pathFilter) {
+    let output;
+
+    try {
+      // Ordering and technical-upgrade filtering are done by the caller in memory, so `--author-date-order`/`--grep`/`--name-only` are deliberately omitted to keep this walk lean
+      output = await this.git.raw([
+        'log',
+        '--no-merges',
+        '--format=%H%x09%at%x09%s', // Tab-separated hash, author date and subject: the minimum needed to order versions and detect technical upgrades, with no diff or message body loaded
+        '--', // Everything after is a pathspec, never a revision or an option
+        pathFilter,
+      ]);
+    } catch (error) {
+      if (/unknown revision or path not in the working tree|does not have any commits yet/.test(error.message)) {
+        return [];
+      }
+
+      throw error;
+    }
+
+    if (!output) {
+      return [];
+    }
+
+    return output.trim().split('\n').filter(Boolean).map(line => {
+      const [ hash, timestamp, ...subjectParts ] = line.split('\t');
+
+      return { hash, timestamp: parseInt(timestamp, 10), subject: subjectParts.join('\t') };
+    });
+  }
+
+  async getDiffStats(commitHash) {
+    const output = await this.git.raw([
+      'show',
+      '--numstat', // Report added/deleted line counts per file as tab-separated numbers, instead of a textual diff
+      '--format=', // Drop the commit header so the output holds only the numstat lines
+      commitHash,
+    ]);
+
+    let additions = 0;
+    let deletions = 0;
+
+    for (const line of output.trim().split('\n')) {
+      if (!line) {
+        continue;
+      }
+
+      const [ added, deleted ] = line.split('\t');
+
+      // Binary files show '-' for additions/deletions
+      if (added !== '-') {
+        additions += parseInt(added, 10);
+      }
+      if (deleted !== '-') {
+        deletions += parseInt(deleted, 10);
+      }
+    }
+
+    return { additions, deletions };
   }
 }

package/src/archivist/recorder/repositories/git/index.js

@@ -15,6 +15,25 @@ import Git from './git.js';
 
 const fs = fsApi.promises;
 
+const RECORD_ID_REGEXP = /^[0-9a-f]{7,40}$/i; // Git commit SHA 7 (abbreviated) to 40 (full) hexadecimal characters. Prevent value such as `--output=…` to be parsed as a command-line option
+
+const CONTROL_CHARACTERS_REGEXP = /\p{Cc}/u; // Matches any Unicode "control" character: the C0 range (U+0000 to U+001F), DEL (U+007F) and the C1 range (U+0080 to U+009F), i.e. 65 non-printable characters including NUL. The `u` flag is required for the `\p{...}` property escape to be recognised, otherwise the pattern would match the literal text `p{Cc}`. Legitimate service IDs, terms types and document IDs never contain these, and NUL in particular can truncate a value once it reaches git or the filesystem, so any segment holding one is rejected.
+
+// Keeps hostile values from reaching git, where a pathspec that resolves outside the repository (such as `../foo/*`) aborts with an error that exposes the repository location.
+function isPlainPathSegment(segment) {
+  return segment.length > 0
+    && segment !== '.'
+    && segment !== '..'
+    && !segment.includes('/')
+    && !segment.includes('\\')
+    && !CONTROL_CHARACTERS_REGEXP.test(segment);
+}
+
+function canMatchRecordFilePath(...pathSegments) {
+  // A non-string segment means "not provided" (`undefined`, or `false` for an absent document ID) and constrains nothing
+  return pathSegments.every(segment => typeof segment !== 'string' || isPlainPathSegment(segment));
+}
+
 export default class GitRepository extends RepositoryInterface {
   constructor({ path, author, publish, snapshotIdentiferTemplate }) {
     super();
@@ -64,6 +83,10 @@ export default class GitRepository extends RepositoryInterface {
   }
 
   async findLatest(serviceId, termsType, documentId) {
+    if (!canMatchRecordFilePath(serviceId, termsType, documentId)) {
+      return null;
+    }
+
     const matchingFilesPaths = await this.git.listFiles(DataMapper.generateFilePath(serviceId, termsType, documentId));
 
     if (!matchingFilesPaths.length) {
@@ -76,35 +99,95 @@ export default class GitRepository extends RepositoryInterface {
   }
 
   async findByDate(serviceId, termsType, date, documentId) {
+    if (!canMatchRecordFilePath(serviceId, termsType, documentId)) {
+      return null;
+    }
+
     const filePath = DataMapper.generateFilePath(serviceId, termsType, documentId);
-    const commit = await this.git.getCommit([ `--until=${date?.toISOString()}`, filePath ]);
+    const commit = await this.git.getCommit([ `--until=${date?.toISOString()}`, '--', filePath ]);
 
     return this.#toDomain(commit);
   }
 
   async findById(recordId) {
-    const commit = await this.git.getCommit([recordId]);
+    if (!RECORD_ID_REGEXP.test(recordId)) {
+      return null;
+    }
+
+    const commit = await this.git.getCommit([ '--end-of-options', recordId ]); // `--end-of-options` forces git to treat `recordId` as a revision, never as an option: a second line of defence that keeps the lookup safe from argument injection even if the format guard above is ever relaxed
 
     return this.#toDomain(commit);
   }
 
+  async findMetadataById(recordId) {
+    if (!RECORD_ID_REGEXP.test(recordId)) {
+      return null;
+    }
+
+    const commit = await this.git.getCommit([ '--end-of-options', recordId ]); // `--end-of-options` forces git to treat `recordId` as a revision, never as an option: a second line of defence that keeps the lookup safe from argument injection even if the format guard above is ever relaxed
+
+    return this.#toDomain(commit, { deferContentLoading: true });
+  }
+
   async findAll({ limit, offset, includeTechnicalUpgrades = true } = {}) {
     return Promise.all((await this.#getCommits({ limit, offset, includeTechnicalUpgrades })).map(commit => this.#toDomain(commit, { deferContentLoading: true })));
   }
 
+  async findByServiceAndTermsType(serviceId, termsType, { limit, offset, includeTechnicalUpgrades = true } = {}) {
+    if (!canMatchRecordFilePath(serviceId, termsType)) {
+      return [];
+    }
+
+    const pathPattern = DataMapper.generateFilePath(serviceId, termsType);
+
+    return Promise.all((await this.#getCommits({ pathFilter: pathPattern, limit, offset, includeTechnicalUpgrades })).map(commit => this.#toDomain(commit, { deferContentLoading: true })));
+  }
+
   async findByService(serviceId, { limit, offset, includeTechnicalUpgrades = true } = {}) {
+    if (!canMatchRecordFilePath(serviceId)) {
+      return [];
+    }
+
     const pathPattern = DataMapper.generateFilePath(serviceId);
 
     return Promise.all((await this.#getCommits({ pathFilter: pathPattern, limit, offset, includeTechnicalUpgrades })).map(commit => this.#toDomain(commit, { deferContentLoading: true })));
   }
 
-  async findByServiceAndTermsType(serviceId, termsType, { limit, offset, includeTechnicalUpgrades = true } = {}) {
+  async getNavigationIds(serviceId, termsType, versionId, { includeTechnicalUpgrades = true } = {}) {
+    if (!canMatchRecordFilePath(serviceId, termsType)) {
+      return { first: null, prev: null, next: null, last: null };
+    }
+
     const pathPattern = DataMapper.generateFilePath(serviceId, termsType);
+    let revisions = await this.git.listPathRevisions(pathPattern); // single lean walk of the terms history
 
-    return Promise.all((await this.#getCommits({ pathFilter: pathPattern, limit, offset, includeTechnicalUpgrades })).map(commit => this.#toDomain(commit, { deferContentLoading: true })));
+    if (!includeTechnicalUpgrades) {
+      revisions = revisions.filter(revision => !DataMapper.isTechnicalUpgrade(revision.subject));
+    }
+
+    // Deterministic total order: most recent first, commit SHA as a stable tiebreaker for versions sharing the same fetch date (git stores second precision).
+    // prev/next are then adjacent entries in this single order, so navigation always round-trips, unlike the previous chronological/topological mix.
+    revisions.sort((a, b) => b.timestamp - a.timestamp || (a.hash < b.hash ? -1 : 1));
+
+    const index = revisions.findIndex(revision => revision.hash === versionId);
+
+    if (index === -1) {
+      return { first: null, prev: null, next: null, last: null };
+    }
+
+    return {
+      last: revisions[0].hash,
+      first: revisions[revisions.length - 1].hash,
+      next: index > 0 ? revisions[index - 1].hash : null,
+      prev: index < revisions.length - 1 ? revisions[index + 1].hash : null,
+    };
   }
 
   async count(serviceId, termsType) {
+    if (!canMatchRecordFilePath(serviceId, termsType)) {
+      return 0;
+    }
+
     const grepOptions = Object.values(DataMapper.COMMIT_MESSAGE_PREFIXES).map(prefix => `--grep=${prefix}`);
     const pathOptions = [];
 
@@ -158,6 +241,10 @@ export default class GitRepository extends RepositoryInterface {
     record.content = pdfBuffer;
   }
 
+  getDiffStats(recordId) {
+    return this.git.getDiffStats(recordId);
+  }
+
   async #getCommits({ pathFilter, reverse = false, limit, offset, includeTechnicalUpgrades = true } = {}) {
     const prefixes = includeTechnicalUpgrades
       ? DataMapper.COMMIT_MESSAGE_PREFIXES