@opentermsarchive/engine 16.0.1 → 16.1.0

package/src/git/pathSegment.js

@@ -0,0 +1,18 @@
+const CONTROL_CHARACTERS_REGEXP = /\p{Cc}/u; // Matches any Unicode "control" character: the C0 range (U+0000 to U+001F), DEL (U+007F) and the C1 range (U+0080 to U+009F), i.e. 65 non-printable characters including NUL. The `u` flag is required for the `\p{...}` property escape to be recognised, otherwise the pattern would match the literal text `p{Cc}`. Legitimate service IDs, terms types and document IDs never contain these, and NUL in particular can truncate a value once it reaches git or the filesystem, so any segment holding one is rejected.
+
+const WINDOWS_RESERVED_CHARACTERS_REGEXP = /[:"<>|*?]/; // Forbidden in Windows file names, so a segment holding one cannot exist on every supported platform; the service ID guidelines exclude them for the same reason
+
+// Keeps hostile values from reaching git, where a pathspec that resolves outside the repository (such as `../foo/*`) aborts with an error that exposes the repository location.
+export function isPlainPathSegment(segment) {
+  return segment.length > 0
+    && segment !== '.'
+    && segment !== '..'
+    && !segment.includes('/')
+    && !segment.includes('\\')
+    && !CONTROL_CHARACTERS_REGEXP.test(segment);
+}
+
+// Accepts only segments that are usable as a file name on every supported platform. As git C-quotes only double-quotes, backslashes and control characters in the paths it prints (see core.quotePath), all rejected here, an accepted segment also never comes back quoted from a git log.
+export function isPortableFileName(segment) {
+  return isPlainPathSegment(segment) && !WINDOWS_RESERVED_CHARACTERS_REGEXP.test(segment);
+}

package/src/git/pathSegment.test.js

@@ -0,0 +1,33 @@
+import { expect } from 'chai';
+
+import { isPlainPathSegment, isPortableFileName } from './pathSegment.js';
+
+describe('PathSegment', () => {
+  describe('#isPlainPathSegment', () => {
+    [ 'Facebook', 'Terms of Service', 'Facebook v2.0', 'service·A' ].forEach(segment => {
+      it(`accepts ${JSON.stringify(segment)}`, () => {
+        expect(isPlainPathSegment(segment)).to.be.true;
+      });
+    });
+
+    [ '', '.', '..', 'a/b', 'a\\b', 'with\0null', 'line\nbreak' ].forEach(segment => {
+      it(`rejects ${JSON.stringify(segment)}`, () => {
+        expect(isPlainPathSegment(segment)).to.be.false;
+      });
+    });
+  });
+
+  describe('#isPortableFileName', () => {
+    [ 'Facebook', 'Terms of Service', 'Booking.com', 'Yahoo!', 'service·A' ].forEach(segment => {
+      it(`accepts ${JSON.stringify(segment)}`, () => {
+        expect(isPortableFileName(segment)).to.be.true;
+      });
+    });
+
+    [ 'Service "A"', 're:start', 'a<b', 'a>b', 'a|b', 'a*b', 'a?b', '', '.', '..', 'a/b', 'a\\b', 'with\0null' ].forEach(segment => {
+      it(`rejects ${JSON.stringify(segment)}`, () => {
+        expect(isPortableFileName(segment)).to.be.false;
+      });
+    });
+  });
+});

package/src/index.js

@@ -12,10 +12,11 @@ import Reporter from './reporter/index.js';
 const require = createRequire(import.meta.url);
 const { version: PACKAGE_VERSION } = require('../package.json');
 
-async function initialize(services) {
+async function initialize(services, { trackingResultsConfig } = {}) {
   const archivist = new Archivist({
     recorderConfig: config.get('@opentermsarchive/engine.recorder'),
     fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
+    trackingResultsConfig,
   });
 
   archivist.attach(logger);
@@ -28,7 +29,7 @@ async function initialize(services) {
   logger.info(`Start engine v${PACKAGE_VERSION}${collectionName}\n`);
 
   if (services?.length) {
-    services = services.filter(serviceId => {
+    services = [...new Set(services)].filter(serviceId => { // Deduplicated so that a service ID given twice is not tracked twice in the same run
       const isServiceDeclared = archivist.services[serviceId];
 
       if (!isServiceDeclared) {
@@ -43,7 +44,7 @@ async function initialize(services) {
 }
 
 export default async function track({ services, types, schedule }) {
-  const { archivist, services: filteredServices } = await initialize(services);
+  const { archivist, services: filteredServices } = await initialize(services, { trackingResultsConfig: config.get('@opentermsarchive/engine.tracking-results') });
 
   // Technical upgrade pass: apply changes from engine, dependency, or declaration upgrades.
   // This regenerates versions from existing snapshots with updated extraction logic.
@@ -84,7 +85,7 @@ export default async function track({ services, types, schedule }) {
 }
 
 export async function applyTechnicalUpgrades({ services, types }) {
-  const { archivist, services: filteredServices } = await initialize(services);
+  const { archivist, services: filteredServices } = await initialize(services); // Without tracking-results: technical upgrades never record any, and initializing the module would finalize as crashed the run that a tracking process may have in progress
 
   await archivist.applyTechnicalUpgrades({ services: filteredServices, types });
 }

package/src/reporter/index.js

@@ -144,9 +144,14 @@ No changes were found in the last run, so no new version has been recorded.`,
     const latestDeclarationLink = `[Latest declaration](${declarationFileUrl})`;
     const latestVersionLink = `[Latest version](${this.reporter.generateVersionURL(terms.service.name, terms.type)})`;
     const snapshotsBaseUrl = this.reporter.generateSnapshotsBaseUrl(terms.service.name, terms.type);
-    const latestSnapshotsLink = terms.hasMultipleSourceDocuments
-      ? `Latest snapshots:\n  - ${terms.sourceDocuments.map(sourceDocument => `[${sourceDocument.id}](${snapshotsBaseUrl}.%20#${sourceDocument.id}.${mime.getExtension(sourceDocument.mimeType)})`).join('\n  - ')}`
-      : `[Latest snapshot](${snapshotsBaseUrl}.${mime.getExtension(terms.sourceDocuments[0].mimeType)})`;
+    const recordedSourceDocuments = terms.sourceDocuments.filter(sourceDocument => sourceDocument.snapshotId); // A source document that has never been recorded has no snapshot file to link to, and its unknown MIME type would produce a link to a nonexistent ".null" file
+    let latestSnapshotsLink = '';
+
+    if (recordedSourceDocuments.length) {
+      latestSnapshotsLink = terms.hasMultipleSourceDocuments
+        ? `Latest snapshots:\n  - ${recordedSourceDocuments.map(sourceDocument => `[${sourceDocument.id}](${snapshotsBaseUrl}.%20#${sourceDocument.id}.${mime.getExtension(sourceDocument.mimeType)})`).join('\n  - ')}`
+        : `[Latest snapshot](${snapshotsBaseUrl}.${mime.getExtension(recordedSourceDocuments[0].mimeType)})`;
+    }
 
     /* eslint-disable no-irregular-whitespace */
     return `
@@ -185,7 +190,7 @@ If the source documents are accessible in a browser but fetching them always fai
 
 - ${latestDeclarationLink}
 ${this.repositories.versions ? `- ${latestVersionLink}` : ''}
-${this.repositories.snapshots ? `- ${latestSnapshotsLink}` : ''} 
+${this.repositories.snapshots && latestSnapshotsLink ? `- ${latestSnapshotsLink}` : ''}
 `;
   /* eslint-enable no-irregular-whitespace */
   }

package/src/reporter/index.test.js

@@ -64,17 +64,21 @@ describe('Reporter', () => {
   describe('#generateDescription', () => {
     const buildReporter = () => new Reporter({
       type: 'github',
-      repositories: { declarations: 'OpenTermsArchive/test-declarations' },
+      repositories: { declarations: 'OpenTermsArchive/test-declarations', snapshots: 'OpenTermsArchive/test-snapshots' },
     });
 
-    const buildTerms = ({ sourceCount = 1 } = {}) => {
-      const sourceDocuments = Array.from({ length: sourceCount }, (_, index) => ({
-        id: `source-${index}`,
-        location: `https://example.com/source-${index}`,
-        mimeType: 'text/html',
-        snapshotId: `snapshot-${index}`,
-        toPersistence: () => ({ fetch: `https://example.com/source-${index}` }),
-      }));
+    const buildTerms = ({ sourceCount = 1, hasSnapshots = true, withoutSnapshotIndexes = [] } = {}) => {
+      const sourceDocuments = Array.from({ length: sourceCount }, (_, index) => {
+        const hasSnapshot = hasSnapshots && !withoutSnapshotIndexes.includes(index); // A document that has never been recorded has neither a snapshot ID nor an observed MIME type
+
+        return {
+          id: `source-${index}`,
+          location: `https://example.com/source-${index}`,
+          mimeType: hasSnapshot ? 'text/html' : null,
+          snapshotId: hasSnapshot ? `snapshot-${index}` : null,
+          toPersistence: () => ({ fetch: `https://example.com/source-${index}` }),
+        };
+      });
 
       return {
         service: { id: 'TestService', name: 'TestService' },
@@ -94,6 +98,40 @@ describe('Reporter', () => {
 
     const error = { reasons: ['HTTP code 404'] };
 
+    context('when the source documents have been recorded as snapshots', () => {
+      it('mentions that the missed versions might be recovered', () => {
+        const description = buildReporter().generateDescription({ error, terms: buildTerms() });
+
+        expect(description).to.include('it might still be possible to recover the missed versions');
+      });
+    });
+
+    context('when the source documents could not be recorded as snapshots', () => {
+      it('does not suggest that the missed versions might be recovered', () => {
+        const description = buildReporter().generateDescription({ error, terms: buildTerms({ hasSnapshots: false }) });
+
+        expect(description).to.include('has not been recorded as a snapshot');
+        expect(description).to.not.include('it might still be possible to recover the missed versions');
+      });
+
+      it('omits the snapshot link, which would point to a nonexistent file', () => {
+        const description = buildReporter().generateDescription({ error, terms: buildTerms({ hasSnapshots: false }) });
+
+        expect(description).to.not.include('Latest snapshot');
+        expect(description).to.not.include('.null');
+      });
+    });
+
+    context('when a source document of combined terms has never been recorded as a snapshot', () => {
+      it('omits its snapshot link and keeps the ones of the recorded documents', () => {
+        const description = buildReporter().generateDescription({ error, terms: buildTerms({ sourceCount: 2, withoutSnapshotIndexes: [1] }) });
+
+        expect(description).to.include('#source-0.html'); // The `#<id>` fragment only appears in snapshot links, unlike the document locations listed in the accessibility checklist
+        expect(description).to.not.include('#source-1');
+        expect(description).to.not.include('.null');
+      });
+    });
+
     context('when the terms has a single source document', () => {
       it('deep-links to the contribution tool with the serialized declaration as the edit target', () => {
         const description = buildReporter().generateDescription({ error, terms: buildTerms({ sourceCount: 1 }) });

package/src/archivist/recorder/repositories/git/git.test.js

@@ -1,114 +0,0 @@
-import fs from 'fs/promises';
-import path from 'path';
-import { fileURLToPath } from 'url';
-
-import { expect } from 'chai';
-import config from 'config';
-
-import Git from './git.js';
-
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-const RECORDER_PATH = path.resolve(__dirname, '../../../../../', config.get('@opentermsarchive/engine.recorder.versions.storage.git.path'));
-
-describe('Git', () => {
-  const DEFAULT_CONTENT = 'default content';
-  const DEFAULT_COMMIT_MESSAGE = 'default commit message';
-  let subject;
-
-  before(() => {
-    subject = new Git({
-      path: RECORDER_PATH,
-      author: {
-        name: config.get('@opentermsarchive/engine.recorder.versions.storage.git.author.name'),
-        email: config.get('@opentermsarchive/engine.recorder.versions.storage.git.author.email'),
-      },
-    });
-
-    return subject.initialize();
-  });
-
-  describe('#commit', () => {
-    const expectedFilePath = `${RECORDER_PATH}/test.md`;
-
-    let commitId;
-
-    before(async () => {
-      await fs.writeFile(expectedFilePath, DEFAULT_CONTENT);
-
-      await subject.add(expectedFilePath);
-      commitId = await subject.commit({ filePath: expectedFilePath, message: DEFAULT_COMMIT_MESSAGE });
-    });
-
-    after(() => subject.destroyHistory());
-
-    it('returns a full length SHA1 commit ID', () => {
-      expect(commitId).to.match(/\b[0-9a-f]{40}\b/);
-    });
-
-    context('when stage area is dirty', () => {
-      const expectedFileName = 'file-to-commit.md';
-      const expectedFilePath = `${RECORDER_PATH}/${expectedFileName}`;
-      const unwantedFilePath = `${RECORDER_PATH}/unwanted-file.md`;
-      let commit;
-      let committedFiles;
-      let committedFileName;
-
-      before(async () => {
-        await fs.writeFile(expectedFilePath, DEFAULT_CONTENT);
-        await fs.writeFile(unwantedFilePath, DEFAULT_CONTENT);
-
-        await subject.add(expectedFilePath);
-        await subject.add(unwantedFilePath);
-
-        const commitId = await subject.commit({ filePath: expectedFilePath, message: DEFAULT_COMMIT_MESSAGE });
-
-        commit = await subject.getCommit([commitId]);
-
-        if (!commit) {
-          return;
-        }
-
-        ({ files: committedFiles } = commit.diff);
-        ([{ file: committedFileName }] = committedFiles);
-      });
-
-      after(() => subject.destroyHistory());
-
-      it('commits the specified file', () => {
-        expect(committedFileName).to.equal(expectedFileName);
-      });
-
-      it('commits only one file', () => {
-        expect(committedFiles).to.have.lengthOf(1);
-      });
-    });
-  });
-
-  describe('#cleanUp', () => {
-    context('when a commit-graph lock has been left behind by an interrupted process', () => {
-      const infoDirectoryPath = path.join(RECORDER_PATH, '.git', 'objects', 'info');
-      const lockFilePath = path.join(infoDirectoryPath, 'commit-graph.lock');
-
-      before(async () => {
-        const filePath = `${RECORDER_PATH}/file-to-clean.md`;
-
-        await fs.writeFile(filePath, DEFAULT_CONTENT);
-        await subject.add(filePath);
-        await subject.commit({ filePath, message: DEFAULT_COMMIT_MESSAGE });
-
-        await fs.mkdir(infoDirectoryPath, { recursive: true });
-        await fs.writeFile(lockFilePath, '');
-
-        await subject.cleanUp();
-      });
-
-      after(() => subject.destroyHistory());
-
-      it('removes the stale commit-graph lock', async () => {
-        const infoDirectoryContent = await fs.readdir(infoDirectoryPath);
-
-        expect(infoDirectoryContent).to.not.include('commit-graph.lock');
-      });
-    });
-  });
-});