@opentermsarchive/engine 16.1.0 → 16.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "16.1.0",
+  "version": "16.2.0",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {

package/src/git/index.js

@@ -5,6 +5,7 @@ import simpleGit from 'simple-git';
 
 import { GitObjectNotFoundError } from './errors.js';
 import { parseTrailers, formatTrailers } from './trailers.js';
+import { acquireWriterLock, WriterLockHeldError } from './writerLock.js';
 
 export { GitObjectNotFoundError } from './errors.js';
 
@@ -12,6 +13,8 @@ process.env.LC_ALL = 'en_GB'; // Ensure git messages will be in English as some
 
 const fs = fsApi.promises;
 
+const WRITER_LOCK_FILE_NAME = 'ota-writer.lock';
+
 const OBJECT_NOT_FOUND_MESSAGES = /bad object|not a tree|invalid object name|unknown revision|does not exist|exists on disk, but not in/i;
 
 export default class Git {
@@ -51,6 +54,7 @@ export default class Git {
 
     this.#connect();
     await this.git.init();
+    await this.#acquireWriterLock();
 
     const configFile = path.resolve(this.path, '.git', 'config'); // Anchored to an absolute path: git resolves a relative `--file` argument against its own cwd (the repository), not against process.cwd, so a relative repository path would silently point the write at a nonexistent nested location
 
@@ -69,6 +73,20 @@ export default class Git {
       .raw([ 'config', '--file', configFile, 'gc.writeCommitGraph', 'false' ]); // Prevent automatic `git gc` from also writing the commit-graph: the engine writes it explicitly (see `writeCommitGraph`/`updateCommitGraph`), and a concurrent gc write races those, which can leave a stale `commit-graph.lock` and make subsequent operations fail
   }
 
+  async #acquireWriterLock() { // Held until the process exits, so that no other process writes to the repository meanwhile: concurrent writers race the working tree and the index, and the clean up of one discards the uncommitted changes of the other
+    const lockFilePath = path.join(this.path, '.git', WRITER_LOCK_FILE_NAME);
+
+    try {
+      await acquireWriterLock(lockFilePath); // Acquired again silently by the same process, as repositories are initialized again at each tracking run
+    } catch (error) {
+      if (!(error instanceof WriterLockHeldError)) {
+        throw error;
+      }
+
+      throw new Error(`Repository ${this.path} is already being written by the process ${error.holderPid}; stop it before starting another writer, such as the tracker or the technical upgrades, or remove ${lockFilePath} if that process is not an Open Terms Archive writer`);
+    }
+  }
+
   open() {
     if (!fsApi.existsSync(path.join(this.path, '.git'))) {
       throw new Error(`Repository ${this.path} does not exist, it has to be created by a writer such as the tracker first`); // Without this check, git would silently walk up to an enclosing repository, such as the collection one
@@ -183,6 +201,7 @@ export default class Git {
   }
 
   async cleanUp() {
+    await fs.rm(path.join(this.path, '.git', 'index.lock'), { force: true }); // Remove a leftover index lock from a Git operation that was killed mid-write, which would otherwise make every subsequent reset, add and commit fail. Safe as the writer lock guarantees that no other process writes to the repository, and readers never lock the index
     await fs.rm(path.join(this.path, '.git', 'objects', 'info', 'commit-graph.lock'), { force: true }); // Remove a leftover commit-graph lock from a previous `commit-graph write` that was killed mid-write (e.g. the process was terminated during a deploy or restart). The commit-graph is a disposable cache rebuilt by `writeCommitGraph`, so clearing a stale lock is safe and prevents every subsequent run from failing.
     await this.git.reset('hard');
 

package/src/git/index.test.js

@@ -1,13 +1,17 @@
+import { spawn } from 'child_process';
 import fs from 'fs/promises';
 import os from 'os';
 import path from 'path';
 import { fileURLToPath } from 'url';
 
-import { expect } from 'chai';
+import { expect, use } from 'chai';
+import chaiAsPromised from 'chai-as-promised';
 import config from 'config';
 
 import Git, { GitObjectNotFoundError } from './index.js';
 
+use(chaiAsPromised);
+
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const RECORDER_PATH = path.resolve(__dirname, '../../', config.get('@opentermsarchive/engine.recorder.versions.storage.git.path'));
 
@@ -28,6 +32,51 @@ describe('Git', () => {
     return subject.initialize();
   });
 
+  describe('#initialize', () => {
+    const AUTHOR = { name: 'Writer lock tester', email: 'writer-lock@example.com' };
+    let repositoryPath;
+    let lockFilePath;
+
+    beforeEach(async () => {
+      repositoryPath = await fs.mkdtemp(path.join(os.tmpdir(), 'ota-writer-lock-'));
+      lockFilePath = path.join(repositoryPath, '.git', 'ota-writer.lock');
+    });
+
+    afterEach(() => fs.rm(repositoryPath, { recursive: true, force: true }));
+
+    it('locks the repository with the PID of the current process', async () => {
+      await new Git({ path: repositoryPath, author: AUTHOR }).initialize();
+
+      expect(await fs.readFile(lockFilePath, 'utf8')).to.equal(String(process.pid));
+    });
+
+    it('can be initialized again by the same process', async () => {
+      await new Git({ path: repositoryPath, author: AUTHOR }).initialize();
+
+      await expect(new Git({ path: repositoryPath, author: AUTHOR }).initialize()).to.be.fulfilled;
+    });
+
+    context('when another running process holds the lock', () => {
+      let otherProcess;
+
+      beforeEach(async () => {
+        otherProcess = spawn(process.execPath, [ '-e', 'setInterval(() => {}, 1000)' ]);
+        await fs.mkdir(path.dirname(lockFilePath), { recursive: true });
+        await fs.writeFile(lockFilePath, String(otherProcess.pid));
+      });
+
+      afterEach(() => otherProcess.kill());
+
+      it('rejects with an error naming the holder', async () => {
+        await expect(new Git({ path: repositoryPath, author: AUTHOR }).initialize()).to.be.rejectedWith(`already being written by the process ${otherProcess.pid}`);
+      });
+
+      it('names the lock file in the error', async () => {
+        await expect(new Git({ path: repositoryPath, author: AUTHOR }).initialize()).to.be.rejectedWith(lockFilePath);
+      });
+    });
+  });
+
   describe('#commit', () => {
     const expectedFilePath = `${RECORDER_PATH}/test.md`;
 
@@ -136,6 +185,36 @@ describe('Git', () => {
         expect(infoDirectoryContent).to.not.include('commit-graph.lock');
       });
     });
+    context('when an index lock has been left behind by an interrupted process', () => {
+      const indexLockFilePath = path.join(RECORDER_PATH, '.git', 'index.lock');
+
+      before(async () => {
+        const filePath = `${RECORDER_PATH}/file-to-clean.md`;
+
+        await fs.writeFile(filePath, DEFAULT_CONTENT);
+        await subject.add(filePath);
+        await subject.commit({ filePath, message: DEFAULT_COMMIT_MESSAGE });
+
+        await fs.writeFile(indexLockFilePath, '');
+
+        await subject.cleanUp();
+      });
+
+      after(() => subject.destroyHistory());
+
+      it('removes the stale index lock', async () => {
+        await expect(fs.access(indexLockFilePath)).to.be.rejected;
+      });
+
+      it('leaves the repository writable', async () => {
+        const filePath = `${RECORDER_PATH}/file-after-clean.md`;
+
+        await fs.writeFile(filePath, DEFAULT_CONTENT);
+        await subject.add(filePath);
+
+        expect(await subject.commit({ filePath, message: DEFAULT_COMMIT_MESSAGE })).to.be.a('string');
+      });
+    });
   });
 
   describe('.getHeadSha', () => {

package/src/git/writerLock.js

@@ -0,0 +1,101 @@
+import fsApi from 'fs';
+import os from 'os';
+
+const fs = fsApi.promises;
+
+const heldLocks = new Set();
+
+process.on('exit', () => heldLocks.forEach(release)); // A process killed without exiting leaves its lock behind, which the next taker detects as stale
+
+export class WriterLockHeldError extends Error {
+  constructor(lockFilePath, holderPid) {
+    super(`Lock ${lockFilePath} is held by the process ${holderPid}`);
+    this.name = 'WriterLockHeldError';
+    this.lockFilePath = lockFilePath;
+    this.holderPid = holderPid;
+  }
+}
+
+export async function acquireWriterLock(lockFilePath) { // Held until the process exits; a process already holding the lock acquires it again silently
+  try {
+    await fs.writeFile(lockFilePath, String(process.pid), { flag: 'wx' }); // Exclusive creation, so that only one of two processes starting at the same time gets the lock
+  } catch (error) {
+    if (error.code !== 'EEXIST') {
+      throw error;
+    }
+
+    const { holderPid, writtenAt } = await read(lockFilePath);
+
+    if (holderPid === process.pid) { // Left by a previous process that had the same PID, such as the same container restarted after a kill
+      heldLocks.add(lockFilePath);
+
+      return;
+    }
+
+    if (isAlive(holderPid) && writtenAt > Date.now() - os.uptime() * 1000) { // A lock written before the last boot is stale, even if its PID has since been reused by another process
+      throw new WriterLockHeldError(lockFilePath, holderPid);
+    }
+
+    await takeOver(lockFilePath);
+
+    return acquireWriterLock(lockFilePath);
+  }
+
+  heldLocks.add(lockFilePath);
+}
+
+async function read(lockFilePath) {
+  try {
+    const [ content, { mtimeMs }] = await Promise.all([ fs.readFile(lockFilePath, 'utf8'), fs.stat(lockFilePath) ]);
+
+    return { holderPid: Number(content), writtenAt: mtimeMs };
+  } catch (error) {
+    if (error.code !== 'ENOENT') {
+      throw error;
+    }
+
+    return { holderPid: null, writtenAt: 0 }; // The holder released the lock in the meantime, so it is taken over as a stale one
+  }
+}
+
+async function takeOver(lockFilePath) {
+  const staleLockFilePath = `${lockFilePath}.${process.pid}`;
+
+  try {
+    await fs.rename(lockFilePath, staleLockFilePath); // Atomic, so that only one of two processes taking over the same stale lock at the same time succeeds; the other finds the fresh lock of the winner when it retries
+  } catch (error) {
+    if (error.code !== 'ENOENT') {
+      throw error;
+    }
+  }
+
+  await fs.rm(staleLockFilePath, { force: true });
+}
+
+function release(lockFilePath) { // Synchronous, as asynchronous work is not run on exit
+  let holder;
+
+  try {
+    holder = fsApi.readFileSync(lockFilePath, 'utf8');
+  } catch {
+    return; // Already released, for example along with the directory that held it
+  }
+
+  if (holder === String(process.pid)) { // Leave the lock to a process that took it over, which happens when the clock was set forward after this process wrote it
+    fsApi.rmSync(lockFilePath, { force: true });
+  }
+}
+
+function isAlive(pid) {
+  if (!Number.isInteger(pid) || pid <= 0) { // An empty or corrupted lock file, left by a process killed while creating it
+    return false;
+  }
+
+  try {
+    process.kill(pid, 0); // Signal 0 only checks that the process exists
+
+    return true;
+  } catch (error) {
+    return error.code === 'EPERM'; // The process exists but belongs to another user
+  }
+}

package/src/git/writerLock.test.js

@@ -0,0 +1,148 @@
+import { execFile, spawn } from 'child_process';
+import fs from 'fs/promises';
+import os from 'os';
+import path from 'path';
+import { fileURLToPath, pathToFileURL } from 'url';
+import { promisify } from 'util';
+
+import { expect, use } from 'chai';
+import chaiAsPromised from 'chai-as-promised';
+
+import { acquireWriterLock, WriterLockHeldError } from './writerLock.js';
+
+use(chaiAsPromised);
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+describe('WriterLock', () => {
+  describe('#acquireWriterLock', () => {
+    let directory;
+    let lockFilePath;
+
+    beforeEach(async () => {
+      directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ota-writer-lock-'));
+      lockFilePath = path.join(directory, 'writer.lock');
+    });
+
+    afterEach(() => fs.rm(directory, { recursive: true, force: true }));
+
+    async function writeLock(content, { date } = {}) {
+      await fs.writeFile(lockFilePath, content);
+
+      if (date) {
+        await fs.utimes(lockFilePath, date, date);
+      }
+    }
+
+    it('writes the PID of the current process in the lock file', async () => {
+      await acquireWriterLock(lockFilePath);
+
+      expect(await fs.readFile(lockFilePath, 'utf8')).to.equal(String(process.pid));
+    });
+
+    it('can be acquired again by the same process', async () => {
+      await acquireWriterLock(lockFilePath);
+
+      await expect(acquireWriterLock(lockFilePath)).to.be.fulfilled;
+    });
+
+    context('when another running process holds the lock', () => {
+      let otherProcess;
+
+      beforeEach(async () => {
+        otherProcess = spawn(process.execPath, [ '-e', 'setInterval(() => {}, 1000)' ]);
+        await writeLock(String(otherProcess.pid));
+      });
+
+      afterEach(() => otherProcess.kill());
+
+      it('rejects with an error naming the holder and the lock file', async () => {
+        const error = await acquireWriterLock(lockFilePath).catch(error => error);
+
+        expect(error).to.be.an.instanceOf(WriterLockHeldError);
+        expect(error.holderPid).to.equal(otherProcess.pid);
+        expect(error.lockFilePath).to.equal(lockFilePath);
+      });
+
+      it('leaves the lock to its holder', async () => {
+        await acquireWriterLock(lockFilePath).catch(() => {});
+
+        expect(await fs.readFile(lockFilePath, 'utf8')).to.equal(String(otherProcess.pid));
+      });
+
+      context('when the lock was written before the last boot', () => {
+        beforeEach(() => writeLock(String(otherProcess.pid), { date: new Date(Date.now() - (os.uptime() + 60) * 1000) }));
+
+        it('takes over the lock', async () => {
+          await acquireWriterLock(lockFilePath);
+
+          expect(await fs.readFile(lockFilePath, 'utf8')).to.equal(String(process.pid));
+        });
+      });
+    });
+
+    context('when the lock was left by a process that no longer runs', () => {
+      beforeEach(async () => {
+        const { pid } = await new Promise(resolve => {
+          const exitedProcess = spawn(process.execPath, [ '-e', '' ]);
+
+          exitedProcess.on('exit', () => resolve(exitedProcess));
+        });
+
+        await writeLock(String(pid));
+      });
+
+      it('takes over the lock', async () => {
+        await acquireWriterLock(lockFilePath);
+
+        expect(await fs.readFile(lockFilePath, 'utf8')).to.equal(String(process.pid));
+      });
+
+      it('leaves no copy of the stale lock behind', async () => {
+        await acquireWriterLock(lockFilePath);
+
+        expect(await fs.readdir(directory)).to.deep.equal(['writer.lock']);
+      });
+    });
+
+    context('when the lock file is empty', () => {
+      beforeEach(() => writeLock(''));
+
+      it('takes over the lock', async () => {
+        await acquireWriterLock(lockFilePath);
+
+        expect(await fs.readFile(lockFilePath, 'utf8')).to.equal(String(process.pid));
+      });
+    });
+
+    context('when the process exits', function () {
+      this.timeout(10000); // Starts a separate Node.js process
+
+      function runInSeparateProcess(script) {
+        return promisify(execFile)(process.execPath, [ '--input-type=module', '-e', `import { acquireWriterLock } from ${JSON.stringify(pathToFileURL(path.join(__dirname, 'writerLock.js')).href)}; ${script}` ]); // A file URL rather than a path: on Windows, an absolute path is parsed by the ESM loader as a URL with the drive letter as scheme, which it refuses
+      }
+
+      it('releases the lock', async () => {
+        await runInSeparateProcess(`await acquireWriterLock(${JSON.stringify(lockFilePath)});`);
+
+        await expect(fs.access(lockFilePath)).to.be.rejected;
+      });
+
+      context('when the lock was left by a previous process with the same PID', () => {
+        it('releases the lock', async () => {
+          await runInSeparateProcess(`import fs from 'fs/promises'; await fs.writeFile(${JSON.stringify(lockFilePath)}, String(process.pid)); await acquireWriterLock(${JSON.stringify(lockFilePath)});`);
+
+          await expect(fs.access(lockFilePath)).to.be.rejected;
+        });
+      });
+
+      context('when another process took the lock over meanwhile', () => {
+        it('leaves the lock to that process', async () => {
+          await runInSeparateProcess(`import fs from 'fs/promises'; await acquireWriterLock(${JSON.stringify(lockFilePath)}); await fs.writeFile(${JSON.stringify(lockFilePath)}, '${process.pid}');`);
+
+          expect(await fs.readFile(lockFilePath, 'utf8')).to.equal(String(process.pid));
+        });
+      });
+    });
+  });
+});

package/src/index.js

@@ -85,7 +85,7 @@ export default async function track({ services, types, schedule }) {
 }
 
 export async function applyTechnicalUpgrades({ services, types }) {
-  const { archivist, services: filteredServices } = await initialize(services); // Without tracking-results: technical upgrades never record any, and initializing the module would finalize as crashed the run that a tracking process may have in progress
+  const { archivist, services: filteredServices } = await initialize(services); // Without tracking-results, as technical upgrades never record any
 
   await archivist.applyTechnicalUpgrades({ services: filteredServices, types });
 }

package/src/reporter/factory.js

@@ -4,9 +4,9 @@ import GitLab from './gitlab/index.js';
 export function createReporter(config) {
   switch (config.type) {
   case 'github':
-    return new GitHub(config.repositories.declarations);
+    return new GitHub(config.repositories);
   case 'gitlab':
-    return new GitLab(config.repositories.declarations, config.baseURL, config.apiBaseURL);
+    return new GitLab(config.repositories, config.baseURL, config.apiBaseURL);
   default:
     throw new Error(`Unsupported reporter type: ${config.type}`);
   }

package/src/reporter/github/index.js

@@ -13,7 +13,7 @@ export default class GitHub {
   static ISSUE_STATE_ALL = 'all';
   static MAX_LABEL_DESCRIPTION_LENGTH = 100;
 
-  constructor(repository) {
+  constructor(repositories) {
     const { version } = require('../../../package.json');
 
     this.octokit = new Octokit({
@@ -25,7 +25,9 @@ export default class GitHub {
       },
     });
 
-    const [ owner, repo ] = repository.split('/');
+    this.repositories = repositories; // Each generated link points into its own repository, while the issues live in the declarations one
+
+    const [ owner, repo ] = repositories.declarations.split('/');
 
     this.commonParams = { owner, repo };
 
@@ -272,14 +274,14 @@ export default class GitHub {
   }
 
   generateDeclarationURL(serviceName) {
-    return `https://github.com/${this.commonParams.owner}/${this.commonParams.repo}/blob/main/declarations/${encodeURIComponent(serviceName)}.json`;
+    return `https://github.com/${this.repositories.declarations}/blob/main/declarations/${encodeURIComponent(serviceName)}.json`;
   }
 
   generateVersionURL(serviceName, termsType) {
-    return `https://github.com/${this.commonParams.owner}/${this.commonParams.repo}/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}.md`;
+    return `https://github.com/${this.repositories.versions}/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}.md`;
   }
 
   generateSnapshotsBaseUrl(serviceName, termsType) {
-    return `https://github.com/${this.commonParams.owner}/${this.commonParams.repo}/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}`;
+    return `https://github.com/${this.repositories.snapshots}/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}`;
   }
 }

package/src/reporter/github/index.test.js

@@ -10,12 +10,13 @@ describe('GitHub', function () {
 
   let MANAGED_LABELS;
   let github;
+  const REPOSITORIES = { declarations: 'owner/repo', versions: 'owner/versions-repo', snapshots: 'owner/snapshots-repo' };
   const EXISTING_OPEN_ISSUE = { number: 1, title: 'Opened issue', description: 'Issue description', state: GitHub.ISSUE_STATE_OPEN, labels: [{ name: LABELS.HTTP_403.name }] };
   const EXISTING_CLOSED_ISSUE = { number: 2, title: 'Closed issue', description: 'Issue description', state: GitHub.ISSUE_STATE_CLOSED, labels: [{ name: LABELS.EMPTY_CONTENT.name }] };
 
   before(async () => {
     MANAGED_LABELS = Object.values(LABELS);
-    github = new GitHub('owner/repo');
+    github = new GitHub(REPOSITORIES);
     nock('https://api.github.com')
       .get('/repos/owner/repo/issues')
       .query(true)
@@ -518,4 +519,18 @@ describe('GitHub', function () {
       });
     });
   });
+
+  describe('URL generation', () => {
+    it('links the declaration into the declarations repository', () => {
+      expect(github.generateDeclarationURL('Service A')).to.equal('https://github.com/owner/repo/blob/main/declarations/Service%20A.json');
+    });
+
+    it('links the version into the versions repository', () => {
+      expect(github.generateVersionURL('Service A', 'Terms of Service')).to.equal('https://github.com/owner/versions-repo/blob/main/Service%20A/Terms%20of%20Service.md');
+    });
+
+    it('links the snapshots into the snapshots repository', () => {
+      expect(github.generateSnapshotsBaseUrl('Service A', 'Terms of Service')).to.equal('https://github.com/owner/snapshots-repo/blob/main/Service%20A/Terms%20of%20Service');
+    });
+  });
 });

package/src/reporter/gitlab/index.js

@@ -14,8 +14,8 @@ export default class GitLab {
   static ISSUE_STATE_ALL = 'all';
   static MAX_LABEL_DESCRIPTION_LENGTH = 255;
 
-  constructor(repository, baseURL = BASE_URL, apiBaseURL = API_BASE_URL) {
-    this.repositoryPath = repository;
+  constructor(repositories, baseURL = BASE_URL, apiBaseURL = API_BASE_URL) {
+    this.repositories = repositories; // Each generated link points into its own repository, while the issues live in the declarations one
     this.projectId = null;
     this.baseURL = baseURL;
     this.apiBaseURL = apiBaseURL;
@@ -26,7 +26,7 @@ export default class GitLab {
 
     try {
       const response = await nodeFetch(
-        `${this.apiBaseURL}/projects/${encodeURIComponent(this.repositoryPath)}`,
+        `${this.apiBaseURL}/projects/${encodeURIComponent(this.repositories.declarations)}`,
         options,
       );
 
@@ -478,15 +478,15 @@ export default class GitLab {
   }
 
   generateDeclarationURL(serviceName) {
-    return `${this.baseURL}/${this.repositoryPath}/-/blob/main/declarations/${encodeURIComponent(serviceName)}.json`;
+    return `${this.baseURL}/${this.repositories.declarations}/-/blob/main/declarations/${encodeURIComponent(serviceName)}.json`;
   }
 
   generateVersionURL(serviceName, termsType) {
-    return `${this.baseURL}/${this.repositoryPath}/-/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}.md`;
+    return `${this.baseURL}/${this.repositories.versions}/-/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}.md`;
   }
 
   generateSnapshotsBaseUrl(serviceName, termsType) {
-    return `${this.baseURL}/${this.repositoryPath}/-/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}`;
+    return `${this.baseURL}/${this.repositories.snapshots}/-/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}`;
   }
 
   // GitLab API responses are not cached unlike GitHub, so this method only exists to satisfy the Reporter interface contract

package/src/reporter/gitlab/index.test.js

@@ -11,10 +11,11 @@ describe('GitLab', function () {
   let MANAGED_LABELS;
   let gitlab;
   const PROJECT_ID = '4';
+  const REPOSITORIES = { declarations: 'owner/repo', versions: 'owner/versions-repo', snapshots: 'owner/snapshots-repo' };
 
   before(() => {
     MANAGED_LABELS = Object.values(LABELS);
-    gitlab = new GitLab('owner/repo');
+    gitlab = new GitLab(REPOSITORIES);
   });
 
   describe('#initialize', () => {
@@ -767,4 +768,26 @@ describe('GitLab', function () {
       });
     });
   });
+
+  describe('URL generation', () => {
+    it('links the declaration into the declarations repository', () => {
+      expect(gitlab.generateDeclarationURL('Service A')).to.equal('https://gitlab.com/owner/repo/-/blob/main/declarations/Service%20A.json');
+    });
+
+    it('links the version into the versions repository', () => {
+      expect(gitlab.generateVersionURL('Service A', 'Terms of Service')).to.equal('https://gitlab.com/owner/versions-repo/-/blob/main/Service%20A/Terms%20of%20Service.md');
+    });
+
+    it('links the snapshots into the snapshots repository', () => {
+      expect(gitlab.generateSnapshotsBaseUrl('Service A', 'Terms of Service')).to.equal('https://gitlab.com/owner/snapshots-repo/-/blob/main/Service%20A/Terms%20of%20Service');
+    });
+
+    context('with a custom base URL', () => {
+      it('links into the custom instance', () => {
+        const customGitlab = new GitLab(REPOSITORIES, 'https://gitlab.example.test');
+
+        expect(customGitlab.generateVersionURL('Service A', 'Terms of Service')).to.equal('https://gitlab.example.test/owner/versions-repo/-/blob/main/Service%20A/Terms%20of%20Service.md');
+      });
+    });
+  });
 });

package/src/reporter/index.test.js

@@ -64,7 +64,7 @@ describe('Reporter', () => {
   describe('#generateDescription', () => {
     const buildReporter = () => new Reporter({
       type: 'github',
-      repositories: { declarations: 'OpenTermsArchive/test-declarations', snapshots: 'OpenTermsArchive/test-snapshots' },
+      repositories: { declarations: 'OpenTermsArchive/test-declarations', versions: 'OpenTermsArchive/test-versions', snapshots: 'OpenTermsArchive/test-snapshots' },
     });
 
     const buildTerms = ({ sourceCount = 1, hasSnapshots = true, withoutSnapshotIndexes = [] } = {}) => {
@@ -159,5 +159,32 @@ describe('Reporter', () => {
         expect(description.length).to.be.lessThan(65000);
       });
     });
+
+    context('references links', () => {
+      it('links the latest version into the versions repository', () => {
+        const description = buildReporter().generateDescription({ error, terms: buildTerms() });
+
+        expect(description).to.include('github.com/OpenTermsArchive/test-versions/blob/main/TestService/Terms%20of%20Service.md');
+      });
+
+      it('links the latest snapshots into the snapshots repository', () => {
+        const description = buildReporter().generateDescription({ error, terms: buildTerms() });
+
+        expect(description).to.include('github.com/OpenTermsArchive/test-snapshots/blob/main/TestService/Terms%20of%20Service');
+        expect(description).to.not.include('test-declarations/blob/main/TestService/Terms%20of%20Service'); // A version or snapshot link into the declarations repository is the regression this guards against
+      });
+
+      context('when only the declarations repository is configured', () => {
+        it('omits the version and snapshots links instead of generating broken ones', () => {
+          const reporter = new Reporter({ type: 'github', repositories: { declarations: 'OpenTermsArchive/test-declarations' } });
+
+          const description = reporter.generateDescription({ error, terms: buildTerms() });
+
+          expect(description).to.not.include('Latest version');
+          expect(description).to.not.include('Latest snapshot');
+          expect(description).to.not.include('undefined');
+        });
+      });
+    });
   });
 });