@opentermsarchive/engine 16.0.2 → 16.1.1

package/src/index.js

@@ -12,10 +12,11 @@ import Reporter from './reporter/index.js';
 const require = createRequire(import.meta.url);
 const { version: PACKAGE_VERSION } = require('../package.json');
 
-async function initialize(services) {
+async function initialize(services, { trackingResultsConfig } = {}) {
   const archivist = new Archivist({
     recorderConfig: config.get('@opentermsarchive/engine.recorder'),
     fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
+    trackingResultsConfig,
   });
 
   archivist.attach(logger);
@@ -28,7 +29,7 @@ async function initialize(services) {
   logger.info(`Start engine v${PACKAGE_VERSION}${collectionName}\n`);
 
   if (services?.length) {
-    services = services.filter(serviceId => {
+    services = [...new Set(services)].filter(serviceId => { // Deduplicated so that a service ID given twice is not tracked twice in the same run
       const isServiceDeclared = archivist.services[serviceId];
 
       if (!isServiceDeclared) {
@@ -43,7 +44,7 @@ async function initialize(services) {
 }
 
 export default async function track({ services, types, schedule }) {
-  const { archivist, services: filteredServices } = await initialize(services);
+  const { archivist, services: filteredServices } = await initialize(services, { trackingResultsConfig: config.get('@opentermsarchive/engine.tracking-results') });
 
   // Technical upgrade pass: apply changes from engine, dependency, or declaration upgrades.
   // This regenerates versions from existing snapshots with updated extraction logic.
@@ -84,7 +85,7 @@ export default async function track({ services, types, schedule }) {
 }
 
 export async function applyTechnicalUpgrades({ services, types }) {
-  const { archivist, services: filteredServices } = await initialize(services);
+  const { archivist, services: filteredServices } = await initialize(services); // Without tracking-results: technical upgrades never record any, and initializing the module would finalize as crashed the run that a tracking process may have in progress
 
   await archivist.applyTechnicalUpgrades({ services: filteredServices, types });
 }

package/src/reporter/factory.js

@@ -4,9 +4,9 @@ import GitLab from './gitlab/index.js';
 export function createReporter(config) {
   switch (config.type) {
   case 'github':
-    return new GitHub(config.repositories.declarations);
+    return new GitHub(config.repositories);
   case 'gitlab':
-    return new GitLab(config.repositories.declarations, config.baseURL, config.apiBaseURL);
+    return new GitLab(config.repositories, config.baseURL, config.apiBaseURL);
   default:
     throw new Error(`Unsupported reporter type: ${config.type}`);
   }

package/src/reporter/github/index.js

@@ -13,7 +13,7 @@ export default class GitHub {
   static ISSUE_STATE_ALL = 'all';
   static MAX_LABEL_DESCRIPTION_LENGTH = 100;
 
-  constructor(repository) {
+  constructor(repositories) {
     const { version } = require('../../../package.json');
 
     this.octokit = new Octokit({
@@ -25,7 +25,9 @@ export default class GitHub {
       },
     });
 
-    const [ owner, repo ] = repository.split('/');
+    this.repositories = repositories; // Each generated link points into its own repository, while the issues live in the declarations one
+
+    const [ owner, repo ] = repositories.declarations.split('/');
 
     this.commonParams = { owner, repo };
 
@@ -272,14 +274,14 @@ export default class GitHub {
   }
 
   generateDeclarationURL(serviceName) {
-    return `https://github.com/${this.commonParams.owner}/${this.commonParams.repo}/blob/main/declarations/${encodeURIComponent(serviceName)}.json`;
+    return `https://github.com/${this.repositories.declarations}/blob/main/declarations/${encodeURIComponent(serviceName)}.json`;
   }
 
   generateVersionURL(serviceName, termsType) {
-    return `https://github.com/${this.commonParams.owner}/${this.commonParams.repo}/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}.md`;
+    return `https://github.com/${this.repositories.versions}/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}.md`;
   }
 
   generateSnapshotsBaseUrl(serviceName, termsType) {
-    return `https://github.com/${this.commonParams.owner}/${this.commonParams.repo}/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}`;
+    return `https://github.com/${this.repositories.snapshots}/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}`;
   }
 }

package/src/reporter/github/index.test.js

@@ -10,12 +10,13 @@ describe('GitHub', function () {
 
   let MANAGED_LABELS;
   let github;
+  const REPOSITORIES = { declarations: 'owner/repo', versions: 'owner/versions-repo', snapshots: 'owner/snapshots-repo' };
   const EXISTING_OPEN_ISSUE = { number: 1, title: 'Opened issue', description: 'Issue description', state: GitHub.ISSUE_STATE_OPEN, labels: [{ name: LABELS.HTTP_403.name }] };
   const EXISTING_CLOSED_ISSUE = { number: 2, title: 'Closed issue', description: 'Issue description', state: GitHub.ISSUE_STATE_CLOSED, labels: [{ name: LABELS.EMPTY_CONTENT.name }] };
 
   before(async () => {
     MANAGED_LABELS = Object.values(LABELS);
-    github = new GitHub('owner/repo');
+    github = new GitHub(REPOSITORIES);
     nock('https://api.github.com')
       .get('/repos/owner/repo/issues')
       .query(true)
@@ -518,4 +519,18 @@ describe('GitHub', function () {
       });
     });
   });
+
+  describe('URL generation', () => {
+    it('links the declaration into the declarations repository', () => {
+      expect(github.generateDeclarationURL('Service A')).to.equal('https://github.com/owner/repo/blob/main/declarations/Service%20A.json');
+    });
+
+    it('links the version into the versions repository', () => {
+      expect(github.generateVersionURL('Service A', 'Terms of Service')).to.equal('https://github.com/owner/versions-repo/blob/main/Service%20A/Terms%20of%20Service.md');
+    });
+
+    it('links the snapshots into the snapshots repository', () => {
+      expect(github.generateSnapshotsBaseUrl('Service A', 'Terms of Service')).to.equal('https://github.com/owner/snapshots-repo/blob/main/Service%20A/Terms%20of%20Service');
+    });
+  });
 });

package/src/reporter/gitlab/index.js

@@ -14,8 +14,8 @@ export default class GitLab {
   static ISSUE_STATE_ALL = 'all';
   static MAX_LABEL_DESCRIPTION_LENGTH = 255;
 
-  constructor(repository, baseURL = BASE_URL, apiBaseURL = API_BASE_URL) {
-    this.repositoryPath = repository;
+  constructor(repositories, baseURL = BASE_URL, apiBaseURL = API_BASE_URL) {
+    this.repositories = repositories; // Each generated link points into its own repository, while the issues live in the declarations one
     this.projectId = null;
     this.baseURL = baseURL;
     this.apiBaseURL = apiBaseURL;
@@ -26,7 +26,7 @@ export default class GitLab {
 
     try {
       const response = await nodeFetch(
-        `${this.apiBaseURL}/projects/${encodeURIComponent(this.repositoryPath)}`,
+        `${this.apiBaseURL}/projects/${encodeURIComponent(this.repositories.declarations)}`,
         options,
       );
 
@@ -478,15 +478,15 @@ export default class GitLab {
   }
 
   generateDeclarationURL(serviceName) {
-    return `${this.baseURL}/${this.repositoryPath}/-/blob/main/declarations/${encodeURIComponent(serviceName)}.json`;
+    return `${this.baseURL}/${this.repositories.declarations}/-/blob/main/declarations/${encodeURIComponent(serviceName)}.json`;
   }
 
   generateVersionURL(serviceName, termsType) {
-    return `${this.baseURL}/${this.repositoryPath}/-/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}.md`;
+    return `${this.baseURL}/${this.repositories.versions}/-/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}.md`;
   }
 
   generateSnapshotsBaseUrl(serviceName, termsType) {
-    return `${this.baseURL}/${this.repositoryPath}/-/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}`;
+    return `${this.baseURL}/${this.repositories.snapshots}/-/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}`;
   }
 
   // GitLab API responses are not cached unlike GitHub, so this method only exists to satisfy the Reporter interface contract

package/src/reporter/gitlab/index.test.js

@@ -11,10 +11,11 @@ describe('GitLab', function () {
   let MANAGED_LABELS;
   let gitlab;
   const PROJECT_ID = '4';
+  const REPOSITORIES = { declarations: 'owner/repo', versions: 'owner/versions-repo', snapshots: 'owner/snapshots-repo' };
 
   before(() => {
     MANAGED_LABELS = Object.values(LABELS);
-    gitlab = new GitLab('owner/repo');
+    gitlab = new GitLab(REPOSITORIES);
   });
 
   describe('#initialize', () => {
@@ -767,4 +768,26 @@ describe('GitLab', function () {
       });
     });
   });
+
+  describe('URL generation', () => {
+    it('links the declaration into the declarations repository', () => {
+      expect(gitlab.generateDeclarationURL('Service A')).to.equal('https://gitlab.com/owner/repo/-/blob/main/declarations/Service%20A.json');
+    });
+
+    it('links the version into the versions repository', () => {
+      expect(gitlab.generateVersionURL('Service A', 'Terms of Service')).to.equal('https://gitlab.com/owner/versions-repo/-/blob/main/Service%20A/Terms%20of%20Service.md');
+    });
+
+    it('links the snapshots into the snapshots repository', () => {
+      expect(gitlab.generateSnapshotsBaseUrl('Service A', 'Terms of Service')).to.equal('https://gitlab.com/owner/snapshots-repo/-/blob/main/Service%20A/Terms%20of%20Service');
+    });
+
+    context('with a custom base URL', () => {
+      it('links into the custom instance', () => {
+        const customGitlab = new GitLab(REPOSITORIES, 'https://gitlab.example.test');
+
+        expect(customGitlab.generateVersionURL('Service A', 'Terms of Service')).to.equal('https://gitlab.example.test/owner/versions-repo/-/blob/main/Service%20A/Terms%20of%20Service.md');
+      });
+    });
+  });
 });

package/src/reporter/index.js

@@ -144,9 +144,14 @@ No changes were found in the last run, so no new version has been recorded.`,
     const latestDeclarationLink = `[Latest declaration](${declarationFileUrl})`;
     const latestVersionLink = `[Latest version](${this.reporter.generateVersionURL(terms.service.name, terms.type)})`;
     const snapshotsBaseUrl = this.reporter.generateSnapshotsBaseUrl(terms.service.name, terms.type);
-    const latestSnapshotsLink = terms.hasMultipleSourceDocuments
-      ? `Latest snapshots:\n  - ${terms.sourceDocuments.map(sourceDocument => `[${sourceDocument.id}](${snapshotsBaseUrl}.%20#${sourceDocument.id}.${mime.getExtension(sourceDocument.mimeType)})`).join('\n  - ')}`
-      : `[Latest snapshot](${snapshotsBaseUrl}.${mime.getExtension(terms.sourceDocuments[0].mimeType)})`;
+    const recordedSourceDocuments = terms.sourceDocuments.filter(sourceDocument => sourceDocument.snapshotId); // A source document that has never been recorded has no snapshot file to link to, and its unknown MIME type would produce a link to a nonexistent ".null" file
+    let latestSnapshotsLink = '';
+
+    if (recordedSourceDocuments.length) {
+      latestSnapshotsLink = terms.hasMultipleSourceDocuments
+        ? `Latest snapshots:\n  - ${recordedSourceDocuments.map(sourceDocument => `[${sourceDocument.id}](${snapshotsBaseUrl}.%20#${sourceDocument.id}.${mime.getExtension(sourceDocument.mimeType)})`).join('\n  - ')}`
+        : `[Latest snapshot](${snapshotsBaseUrl}.${mime.getExtension(recordedSourceDocuments[0].mimeType)})`;
+    }
 
     /* eslint-disable no-irregular-whitespace */
     return `
@@ -185,7 +190,7 @@ If the source documents are accessible in a browser but fetching them always fai
 
 - ${latestDeclarationLink}
 ${this.repositories.versions ? `- ${latestVersionLink}` : ''}
-${this.repositories.snapshots ? `- ${latestSnapshotsLink}` : ''} 
+${this.repositories.snapshots && latestSnapshotsLink ? `- ${latestSnapshotsLink}` : ''}
 `;
   /* eslint-enable no-irregular-whitespace */
   }

package/src/reporter/index.test.js

@@ -64,17 +64,21 @@ describe('Reporter', () => {
   describe('#generateDescription', () => {
     const buildReporter = () => new Reporter({
       type: 'github',
-      repositories: { declarations: 'OpenTermsArchive/test-declarations' },
+      repositories: { declarations: 'OpenTermsArchive/test-declarations', versions: 'OpenTermsArchive/test-versions', snapshots: 'OpenTermsArchive/test-snapshots' },
     });
 
-    const buildTerms = ({ sourceCount = 1 } = {}) => {
-      const sourceDocuments = Array.from({ length: sourceCount }, (_, index) => ({
-        id: `source-${index}`,
-        location: `https://example.com/source-${index}`,
-        mimeType: 'text/html',
-        snapshotId: `snapshot-${index}`,
-        toPersistence: () => ({ fetch: `https://example.com/source-${index}` }),
-      }));
+    const buildTerms = ({ sourceCount = 1, hasSnapshots = true, withoutSnapshotIndexes = [] } = {}) => {
+      const sourceDocuments = Array.from({ length: sourceCount }, (_, index) => {
+        const hasSnapshot = hasSnapshots && !withoutSnapshotIndexes.includes(index); // A document that has never been recorded has neither a snapshot ID nor an observed MIME type
+
+        return {
+          id: `source-${index}`,
+          location: `https://example.com/source-${index}`,
+          mimeType: hasSnapshot ? 'text/html' : null,
+          snapshotId: hasSnapshot ? `snapshot-${index}` : null,
+          toPersistence: () => ({ fetch: `https://example.com/source-${index}` }),
+        };
+      });
 
       return {
         service: { id: 'TestService', name: 'TestService' },
@@ -94,6 +98,40 @@ describe('Reporter', () => {
 
     const error = { reasons: ['HTTP code 404'] };
 
+    context('when the source documents have been recorded as snapshots', () => {
+      it('mentions that the missed versions might be recovered', () => {
+        const description = buildReporter().generateDescription({ error, terms: buildTerms() });
+
+        expect(description).to.include('it might still be possible to recover the missed versions');
+      });
+    });
+
+    context('when the source documents could not be recorded as snapshots', () => {
+      it('does not suggest that the missed versions might be recovered', () => {
+        const description = buildReporter().generateDescription({ error, terms: buildTerms({ hasSnapshots: false }) });
+
+        expect(description).to.include('has not been recorded as a snapshot');
+        expect(description).to.not.include('it might still be possible to recover the missed versions');
+      });
+
+      it('omits the snapshot link, which would point to a nonexistent file', () => {
+        const description = buildReporter().generateDescription({ error, terms: buildTerms({ hasSnapshots: false }) });
+
+        expect(description).to.not.include('Latest snapshot');
+        expect(description).to.not.include('.null');
+      });
+    });
+
+    context('when a source document of combined terms has never been recorded as a snapshot', () => {
+      it('omits its snapshot link and keeps the ones of the recorded documents', () => {
+        const description = buildReporter().generateDescription({ error, terms: buildTerms({ sourceCount: 2, withoutSnapshotIndexes: [1] }) });
+
+        expect(description).to.include('#source-0.html'); // The `#<id>` fragment only appears in snapshot links, unlike the document locations listed in the accessibility checklist
+        expect(description).to.not.include('#source-1');
+        expect(description).to.not.include('.null');
+      });
+    });
+
     context('when the terms has a single source document', () => {
       it('deep-links to the contribution tool with the serialized declaration as the edit target', () => {
         const description = buildReporter().generateDescription({ error, terms: buildTerms({ sourceCount: 1 }) });
@@ -121,5 +159,32 @@ describe('Reporter', () => {
         expect(description.length).to.be.lessThan(65000);
       });
     });
+
+    context('references links', () => {
+      it('links the latest version into the versions repository', () => {
+        const description = buildReporter().generateDescription({ error, terms: buildTerms() });
+
+        expect(description).to.include('github.com/OpenTermsArchive/test-versions/blob/main/TestService/Terms%20of%20Service.md');
+      });
+
+      it('links the latest snapshots into the snapshots repository', () => {
+        const description = buildReporter().generateDescription({ error, terms: buildTerms() });
+
+        expect(description).to.include('github.com/OpenTermsArchive/test-snapshots/blob/main/TestService/Terms%20of%20Service');
+        expect(description).to.not.include('test-declarations/blob/main/TestService/Terms%20of%20Service'); // A version or snapshot link into the declarations repository is the regression this guards against
+      });
+
+      context('when only the declarations repository is configured', () => {
+        it('omits the version and snapshots links instead of generating broken ones', () => {
+          const reporter = new Reporter({ type: 'github', repositories: { declarations: 'OpenTermsArchive/test-declarations' } });
+
+          const description = reporter.generateDescription({ error, terms: buildTerms() });
+
+          expect(description).to.not.include('Latest version');
+          expect(description).to.not.include('Latest snapshot');
+          expect(description).to.not.include('undefined');
+        });
+      });
+    });
   });
 });

package/src/archivist/recorder/repositories/git/git.test.js

@@ -1,114 +0,0 @@
-import fs from 'fs/promises';
-import path from 'path';
-import { fileURLToPath } from 'url';
-
-import { expect } from 'chai';
-import config from 'config';
-
-import Git from './git.js';
-
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-const RECORDER_PATH = path.resolve(__dirname, '../../../../../', config.get('@opentermsarchive/engine.recorder.versions.storage.git.path'));
-
-describe('Git', () => {
-  const DEFAULT_CONTENT = 'default content';
-  const DEFAULT_COMMIT_MESSAGE = 'default commit message';
-  let subject;
-
-  before(() => {
-    subject = new Git({
-      path: RECORDER_PATH,
-      author: {
-        name: config.get('@opentermsarchive/engine.recorder.versions.storage.git.author.name'),
-        email: config.get('@opentermsarchive/engine.recorder.versions.storage.git.author.email'),
-      },
-    });
-
-    return subject.initialize();
-  });
-
-  describe('#commit', () => {
-    const expectedFilePath = `${RECORDER_PATH}/test.md`;
-
-    let commitId;
-
-    before(async () => {
-      await fs.writeFile(expectedFilePath, DEFAULT_CONTENT);
-
-      await subject.add(expectedFilePath);
-      commitId = await subject.commit({ filePath: expectedFilePath, message: DEFAULT_COMMIT_MESSAGE });
-    });
-
-    after(() => subject.destroyHistory());
-
-    it('returns a full length SHA1 commit ID', () => {
-      expect(commitId).to.match(/\b[0-9a-f]{40}\b/);
-    });
-
-    context('when stage area is dirty', () => {
-      const expectedFileName = 'file-to-commit.md';
-      const expectedFilePath = `${RECORDER_PATH}/${expectedFileName}`;
-      const unwantedFilePath = `${RECORDER_PATH}/unwanted-file.md`;
-      let commit;
-      let committedFiles;
-      let committedFileName;
-
-      before(async () => {
-        await fs.writeFile(expectedFilePath, DEFAULT_CONTENT);
-        await fs.writeFile(unwantedFilePath, DEFAULT_CONTENT);
-
-        await subject.add(expectedFilePath);
-        await subject.add(unwantedFilePath);
-
-        const commitId = await subject.commit({ filePath: expectedFilePath, message: DEFAULT_COMMIT_MESSAGE });
-
-        commit = await subject.getCommit([commitId]);
-
-        if (!commit) {
-          return;
-        }
-
-        ({ files: committedFiles } = commit.diff);
-        ([{ file: committedFileName }] = committedFiles);
-      });
-
-      after(() => subject.destroyHistory());
-
-      it('commits the specified file', () => {
-        expect(committedFileName).to.equal(expectedFileName);
-      });
-
-      it('commits only one file', () => {
-        expect(committedFiles).to.have.lengthOf(1);
-      });
-    });
-  });
-
-  describe('#cleanUp', () => {
-    context('when a commit-graph lock has been left behind by an interrupted process', () => {
-      const infoDirectoryPath = path.join(RECORDER_PATH, '.git', 'objects', 'info');
-      const lockFilePath = path.join(infoDirectoryPath, 'commit-graph.lock');
-
-      before(async () => {
-        const filePath = `${RECORDER_PATH}/file-to-clean.md`;
-
-        await fs.writeFile(filePath, DEFAULT_CONTENT);
-        await subject.add(filePath);
-        await subject.commit({ filePath, message: DEFAULT_COMMIT_MESSAGE });
-
-        await fs.mkdir(infoDirectoryPath, { recursive: true });
-        await fs.writeFile(lockFilePath, '');
-
-        await subject.cleanUp();
-      });
-
-      after(() => subject.destroyHistory());
-
-      it('removes the stale commit-graph lock', async () => {
-        const infoDirectoryContent = await fs.readdir(infoDirectoryPath);
-
-        expect(infoDirectoryContent).to.not.include('commit-graph.lock');
-      });
-    });
-  });
-});