@opentermsarchive/engine 16.1.1 → 16.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "16.1.1",
+  "version": "16.2.0",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {

package/src/git/index.js

@@ -5,6 +5,7 @@ import simpleGit from 'simple-git';
 
 import { GitObjectNotFoundError } from './errors.js';
 import { parseTrailers, formatTrailers } from './trailers.js';
+import { acquireWriterLock, WriterLockHeldError } from './writerLock.js';
 
 export { GitObjectNotFoundError } from './errors.js';
 
@@ -12,6 +13,8 @@ process.env.LC_ALL = 'en_GB'; // Ensure git messages will be in English as some
 
 const fs = fsApi.promises;
 
+const WRITER_LOCK_FILE_NAME = 'ota-writer.lock';
+
 const OBJECT_NOT_FOUND_MESSAGES = /bad object|not a tree|invalid object name|unknown revision|does not exist|exists on disk, but not in/i;
 
 export default class Git {
@@ -51,6 +54,7 @@ export default class Git {
 
     this.#connect();
     await this.git.init();
+    await this.#acquireWriterLock();
 
     const configFile = path.resolve(this.path, '.git', 'config'); // Anchored to an absolute path: git resolves a relative `--file` argument against its own cwd (the repository), not against process.cwd, so a relative repository path would silently point the write at a nonexistent nested location
 
@@ -69,6 +73,20 @@ export default class Git {
       .raw([ 'config', '--file', configFile, 'gc.writeCommitGraph', 'false' ]); // Prevent automatic `git gc` from also writing the commit-graph: the engine writes it explicitly (see `writeCommitGraph`/`updateCommitGraph`), and a concurrent gc write races those, which can leave a stale `commit-graph.lock` and make subsequent operations fail
   }
 
+  async #acquireWriterLock() { // Held until the process exits, so that no other process writes to the repository meanwhile: concurrent writers race the working tree and the index, and the clean up of one discards the uncommitted changes of the other
+    const lockFilePath = path.join(this.path, '.git', WRITER_LOCK_FILE_NAME);
+
+    try {
+      await acquireWriterLock(lockFilePath); // Acquired again silently by the same process, as repositories are initialized again at each tracking run
+    } catch (error) {
+      if (!(error instanceof WriterLockHeldError)) {
+        throw error;
+      }
+
+      throw new Error(`Repository ${this.path} is already being written by the process ${error.holderPid}; stop it before starting another writer, such as the tracker or the technical upgrades, or remove ${lockFilePath} if that process is not an Open Terms Archive writer`);
+    }
+  }
+
   open() {
     if (!fsApi.existsSync(path.join(this.path, '.git'))) {
       throw new Error(`Repository ${this.path} does not exist, it has to be created by a writer such as the tracker first`); // Without this check, git would silently walk up to an enclosing repository, such as the collection one
@@ -183,6 +201,7 @@ export default class Git {
   }
 
   async cleanUp() {
+    await fs.rm(path.join(this.path, '.git', 'index.lock'), { force: true }); // Remove a leftover index lock from a Git operation that was killed mid-write, which would otherwise make every subsequent reset, add and commit fail. Safe as the writer lock guarantees that no other process writes to the repository, and readers never lock the index
     await fs.rm(path.join(this.path, '.git', 'objects', 'info', 'commit-graph.lock'), { force: true }); // Remove a leftover commit-graph lock from a previous `commit-graph write` that was killed mid-write (e.g. the process was terminated during a deploy or restart). The commit-graph is a disposable cache rebuilt by `writeCommitGraph`, so clearing a stale lock is safe and prevents every subsequent run from failing.
     await this.git.reset('hard');
 

package/src/git/index.test.js

@@ -1,13 +1,17 @@
+import { spawn } from 'child_process';
 import fs from 'fs/promises';
 import os from 'os';
 import path from 'path';
 import { fileURLToPath } from 'url';
 
-import { expect } from 'chai';
+import { expect, use } from 'chai';
+import chaiAsPromised from 'chai-as-promised';
 import config from 'config';
 
 import Git, { GitObjectNotFoundError } from './index.js';
 
+use(chaiAsPromised);
+
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const RECORDER_PATH = path.resolve(__dirname, '../../', config.get('@opentermsarchive/engine.recorder.versions.storage.git.path'));
 
@@ -28,6 +32,51 @@ describe('Git', () => {
     return subject.initialize();
   });
 
+  describe('#initialize', () => {
+    const AUTHOR = { name: 'Writer lock tester', email: 'writer-lock@example.com' };
+    let repositoryPath;
+    let lockFilePath;
+
+    beforeEach(async () => {
+      repositoryPath = await fs.mkdtemp(path.join(os.tmpdir(), 'ota-writer-lock-'));
+      lockFilePath = path.join(repositoryPath, '.git', 'ota-writer.lock');
+    });
+
+    afterEach(() => fs.rm(repositoryPath, { recursive: true, force: true }));
+
+    it('locks the repository with the PID of the current process', async () => {
+      await new Git({ path: repositoryPath, author: AUTHOR }).initialize();
+
+      expect(await fs.readFile(lockFilePath, 'utf8')).to.equal(String(process.pid));
+    });
+
+    it('can be initialized again by the same process', async () => {
+      await new Git({ path: repositoryPath, author: AUTHOR }).initialize();
+
+      await expect(new Git({ path: repositoryPath, author: AUTHOR }).initialize()).to.be.fulfilled;
+    });
+
+    context('when another running process holds the lock', () => {
+      let otherProcess;
+
+      beforeEach(async () => {
+        otherProcess = spawn(process.execPath, [ '-e', 'setInterval(() => {}, 1000)' ]);
+        await fs.mkdir(path.dirname(lockFilePath), { recursive: true });
+        await fs.writeFile(lockFilePath, String(otherProcess.pid));
+      });
+
+      afterEach(() => otherProcess.kill());
+
+      it('rejects with an error naming the holder', async () => {
+        await expect(new Git({ path: repositoryPath, author: AUTHOR }).initialize()).to.be.rejectedWith(`already being written by the process ${otherProcess.pid}`);
+      });
+
+      it('names the lock file in the error', async () => {
+        await expect(new Git({ path: repositoryPath, author: AUTHOR }).initialize()).to.be.rejectedWith(lockFilePath);
+      });
+    });
+  });
+
   describe('#commit', () => {
     const expectedFilePath = `${RECORDER_PATH}/test.md`;
 
@@ -136,6 +185,36 @@ describe('Git', () => {
         expect(infoDirectoryContent).to.not.include('commit-graph.lock');
       });
     });
+    context('when an index lock has been left behind by an interrupted process', () => {
+      const indexLockFilePath = path.join(RECORDER_PATH, '.git', 'index.lock');
+
+      before(async () => {
+        const filePath = `${RECORDER_PATH}/file-to-clean.md`;
+
+        await fs.writeFile(filePath, DEFAULT_CONTENT);
+        await subject.add(filePath);
+        await subject.commit({ filePath, message: DEFAULT_COMMIT_MESSAGE });
+
+        await fs.writeFile(indexLockFilePath, '');
+
+        await subject.cleanUp();
+      });
+
+      after(() => subject.destroyHistory());
+
+      it('removes the stale index lock', async () => {
+        await expect(fs.access(indexLockFilePath)).to.be.rejected;
+      });
+
+      it('leaves the repository writable', async () => {
+        const filePath = `${RECORDER_PATH}/file-after-clean.md`;
+
+        await fs.writeFile(filePath, DEFAULT_CONTENT);
+        await subject.add(filePath);
+
+        expect(await subject.commit({ filePath, message: DEFAULT_COMMIT_MESSAGE })).to.be.a('string');
+      });
+    });
   });
 
   describe('.getHeadSha', () => {

package/src/git/writerLock.js

@@ -0,0 +1,101 @@
+import fsApi from 'fs';
+import os from 'os';
+
+const fs = fsApi.promises;
+
+const heldLocks = new Set();
+
+process.on('exit', () => heldLocks.forEach(release)); // A process killed without exiting leaves its lock behind, which the next taker detects as stale
+
+export class WriterLockHeldError extends Error {
+  constructor(lockFilePath, holderPid) {
+    super(`Lock ${lockFilePath} is held by the process ${holderPid}`);
+    this.name = 'WriterLockHeldError';
+    this.lockFilePath = lockFilePath;
+    this.holderPid = holderPid;
+  }
+}
+
+export async function acquireWriterLock(lockFilePath) { // Held until the process exits; a process already holding the lock acquires it again silently
+  try {
+    await fs.writeFile(lockFilePath, String(process.pid), { flag: 'wx' }); // Exclusive creation, so that only one of two processes starting at the same time gets the lock
+  } catch (error) {
+    if (error.code !== 'EEXIST') {
+      throw error;
+    }
+
+    const { holderPid, writtenAt } = await read(lockFilePath);
+
+    if (holderPid === process.pid) { // Left by a previous process that had the same PID, such as the same container restarted after a kill
+      heldLocks.add(lockFilePath);
+
+      return;
+    }
+
+    if (isAlive(holderPid) && writtenAt > Date.now() - os.uptime() * 1000) { // A lock written before the last boot is stale, even if its PID has since been reused by another process
+      throw new WriterLockHeldError(lockFilePath, holderPid);
+    }
+
+    await takeOver(lockFilePath);
+
+    return acquireWriterLock(lockFilePath);
+  }
+
+  heldLocks.add(lockFilePath);
+}
+
+async function read(lockFilePath) {
+  try {
+    const [ content, { mtimeMs }] = await Promise.all([ fs.readFile(lockFilePath, 'utf8'), fs.stat(lockFilePath) ]);
+
+    return { holderPid: Number(content), writtenAt: mtimeMs };
+  } catch (error) {
+    if (error.code !== 'ENOENT') {
+      throw error;
+    }
+
+    return { holderPid: null, writtenAt: 0 }; // The holder released the lock in the meantime, so it is taken over as a stale one
+  }
+}
+
+async function takeOver(lockFilePath) {
+  const staleLockFilePath = `${lockFilePath}.${process.pid}`;
+
+  try {
+    await fs.rename(lockFilePath, staleLockFilePath); // Atomic, so that only one of two processes taking over the same stale lock at the same time succeeds; the other finds the fresh lock of the winner when it retries
+  } catch (error) {
+    if (error.code !== 'ENOENT') {
+      throw error;
+    }
+  }
+
+  await fs.rm(staleLockFilePath, { force: true });
+}
+
+function release(lockFilePath) { // Synchronous, as asynchronous work is not run on exit
+  let holder;
+
+  try {
+    holder = fsApi.readFileSync(lockFilePath, 'utf8');
+  } catch {
+    return; // Already released, for example along with the directory that held it
+  }
+
+  if (holder === String(process.pid)) { // Leave the lock to a process that took it over, which happens when the clock was set forward after this process wrote it
+    fsApi.rmSync(lockFilePath, { force: true });
+  }
+}
+
+function isAlive(pid) {
+  if (!Number.isInteger(pid) || pid <= 0) { // An empty or corrupted lock file, left by a process killed while creating it
+    return false;
+  }
+
+  try {
+    process.kill(pid, 0); // Signal 0 only checks that the process exists
+
+    return true;
+  } catch (error) {
+    return error.code === 'EPERM'; // The process exists but belongs to another user
+  }
+}

package/src/git/writerLock.test.js

@@ -0,0 +1,148 @@
+import { execFile, spawn } from 'child_process';
+import fs from 'fs/promises';
+import os from 'os';
+import path from 'path';
+import { fileURLToPath, pathToFileURL } from 'url';
+import { promisify } from 'util';
+
+import { expect, use } from 'chai';
+import chaiAsPromised from 'chai-as-promised';
+
+import { acquireWriterLock, WriterLockHeldError } from './writerLock.js';
+
+use(chaiAsPromised);
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+describe('WriterLock', () => {
+  describe('#acquireWriterLock', () => {
+    let directory;
+    let lockFilePath;
+
+    beforeEach(async () => {
+      directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ota-writer-lock-'));
+      lockFilePath = path.join(directory, 'writer.lock');
+    });
+
+    afterEach(() => fs.rm(directory, { recursive: true, force: true }));
+
+    async function writeLock(content, { date } = {}) {
+      await fs.writeFile(lockFilePath, content);
+
+      if (date) {
+        await fs.utimes(lockFilePath, date, date);
+      }
+    }
+
+    it('writes the PID of the current process in the lock file', async () => {
+      await acquireWriterLock(lockFilePath);
+
+      expect(await fs.readFile(lockFilePath, 'utf8')).to.equal(String(process.pid));
+    });
+
+    it('can be acquired again by the same process', async () => {
+      await acquireWriterLock(lockFilePath);
+
+      await expect(acquireWriterLock(lockFilePath)).to.be.fulfilled;
+    });
+
+    context('when another running process holds the lock', () => {
+      let otherProcess;
+
+      beforeEach(async () => {
+        otherProcess = spawn(process.execPath, [ '-e', 'setInterval(() => {}, 1000)' ]);
+        await writeLock(String(otherProcess.pid));
+      });
+
+      afterEach(() => otherProcess.kill());
+
+      it('rejects with an error naming the holder and the lock file', async () => {
+        const error = await acquireWriterLock(lockFilePath).catch(error => error);
+
+        expect(error).to.be.an.instanceOf(WriterLockHeldError);
+        expect(error.holderPid).to.equal(otherProcess.pid);
+        expect(error.lockFilePath).to.equal(lockFilePath);
+      });
+
+      it('leaves the lock to its holder', async () => {
+        await acquireWriterLock(lockFilePath).catch(() => {});
+
+        expect(await fs.readFile(lockFilePath, 'utf8')).to.equal(String(otherProcess.pid));
+      });
+
+      context('when the lock was written before the last boot', () => {
+        beforeEach(() => writeLock(String(otherProcess.pid), { date: new Date(Date.now() - (os.uptime() + 60) * 1000) }));
+
+        it('takes over the lock', async () => {
+          await acquireWriterLock(lockFilePath);
+
+          expect(await fs.readFile(lockFilePath, 'utf8')).to.equal(String(process.pid));
+        });
+      });
+    });
+
+    context('when the lock was left by a process that no longer runs', () => {
+      beforeEach(async () => {
+        const { pid } = await new Promise(resolve => {
+          const exitedProcess = spawn(process.execPath, [ '-e', '' ]);
+
+          exitedProcess.on('exit', () => resolve(exitedProcess));
+        });
+
+        await writeLock(String(pid));
+      });
+
+      it('takes over the lock', async () => {
+        await acquireWriterLock(lockFilePath);
+
+        expect(await fs.readFile(lockFilePath, 'utf8')).to.equal(String(process.pid));
+      });
+
+      it('leaves no copy of the stale lock behind', async () => {
+        await acquireWriterLock(lockFilePath);
+
+        expect(await fs.readdir(directory)).to.deep.equal(['writer.lock']);
+      });
+    });
+
+    context('when the lock file is empty', () => {
+      beforeEach(() => writeLock(''));
+
+      it('takes over the lock', async () => {
+        await acquireWriterLock(lockFilePath);
+
+        expect(await fs.readFile(lockFilePath, 'utf8')).to.equal(String(process.pid));
+      });
+    });
+
+    context('when the process exits', function () {
+      this.timeout(10000); // Starts a separate Node.js process
+
+      function runInSeparateProcess(script) {
+        return promisify(execFile)(process.execPath, [ '--input-type=module', '-e', `import { acquireWriterLock } from ${JSON.stringify(pathToFileURL(path.join(__dirname, 'writerLock.js')).href)}; ${script}` ]); // A file URL rather than a path: on Windows, an absolute path is parsed by the ESM loader as a URL with the drive letter as scheme, which it refuses
+      }
+
+      it('releases the lock', async () => {
+        await runInSeparateProcess(`await acquireWriterLock(${JSON.stringify(lockFilePath)});`);
+
+        await expect(fs.access(lockFilePath)).to.be.rejected;
+      });
+
+      context('when the lock was left by a previous process with the same PID', () => {
+        it('releases the lock', async () => {
+          await runInSeparateProcess(`import fs from 'fs/promises'; await fs.writeFile(${JSON.stringify(lockFilePath)}, String(process.pid)); await acquireWriterLock(${JSON.stringify(lockFilePath)});`);
+
+          await expect(fs.access(lockFilePath)).to.be.rejected;
+        });
+      });
+
+      context('when another process took the lock over meanwhile', () => {
+        it('leaves the lock to that process', async () => {
+          await runInSeparateProcess(`import fs from 'fs/promises'; await acquireWriterLock(${JSON.stringify(lockFilePath)}); await fs.writeFile(${JSON.stringify(lockFilePath)}, '${process.pid}');`);
+
+          expect(await fs.readFile(lockFilePath, 'utf8')).to.equal(String(process.pid));
+        });
+      });
+    });
+  });
+});

package/src/index.js

@@ -85,7 +85,7 @@ export default async function track({ services, types, schedule }) {
 }
 
 export async function applyTechnicalUpgrades({ services, types }) {
-  const { archivist, services: filteredServices } = await initialize(services); // Without tracking-results: technical upgrades never record any, and initializing the module would finalize as crashed the run that a tracking process may have in progress
+  const { archivist, services: filteredServices } = await initialize(services); // Without tracking-results, as technical upgrades never record any
 
   await archivist.applyTechnicalUpgrades({ services: filteredServices, types });
 }