@opentermsarchive/engine 16.0.1 → 16.0.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "16.0.1",
+  "version": "16.0.2",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {

package/scripts/dataset/export/index.js

@@ -20,7 +20,7 @@ const fs = fsApi.promises;
 const ARCHIVE_FORMAT = 'zip'; // for supported formats, see https://www.archiverjs.com/docs/archive-formats
 
 export default async function generate({ archivePath, releaseDate }) {
-  const versionsRepository = await RepositoryFactory.create(config.get('@opentermsarchive/engine.recorder.versions.storage')).initialize();
+  const versionsRepository = await RepositoryFactory.create(config.get('@opentermsarchive/engine.recorder.versions.storage'), { readOnly: true }).initialize(); // The export only reads the repository the tracker writes to
 
   const temporaryArchivePath = `${archivePath}${TEMPORARY_SUFFIX}`;
   const archive = await initializeArchive(temporaryArchivePath, path.basename(archivePath, path.extname(archivePath)));

package/src/archivist/collection/index.test.js

@@ -13,6 +13,7 @@ describe('Collection', () => {
   let metadataBackup;
   let getCollection;
   let collection;
+  let instanceCount = 0;
 
   before(async () => {
     try {
@@ -29,7 +30,7 @@ describe('Collection', () => {
   });
 
   beforeEach(async () => {
-    const { getCollection: reloadedGetCollection } = await import(`./index.js?t=${Date.now()}`); // Ensure a new instance is loaded for each test
+    const { getCollection: reloadedGetCollection } = await import(`./index.js?instance=${++instanceCount}`); // Ensure a new instance is loaded for each test, even when two tests start within the same millisecond
 
     getCollection = reloadedGetCollection;
   });

package/src/archivist/recorder/repositories/factory.js

@@ -4,15 +4,16 @@ import GitRepository from './git/index.js';
 import MongoRepository from './mongo/index.js';
 
 export default class RepositoryFactory {
-  static create(params) {
+  static create(params, { readOnly = false } = {}) {
     switch (params.type) {
     case 'git':
       return new GitRepository({
         ...params.git,
         path: path.resolve(process.cwd(), params.git.path),
+        readOnly,
       });
     case 'mongo':
-      return new MongoRepository(params.mongo);
+      return new MongoRepository({ ...params.mongo, readOnly });
     default:
       throw new Error(`Unknown storage repository configuration for type '${params.type}'`);
     }

package/src/archivist/recorder/repositories/git/git.js

@@ -17,14 +17,10 @@ export default class Git {
 
   async initialize() {
     if (!fsApi.existsSync(this.path)) {
-      await fs.mkdir(this.path, { recursive: true });
+      await fs.mkdir(this.path, { recursive: true }); // simple-git cannot be instantiated on a missing directory
     }
 
-    this.git = simpleGit(this.path, {
-      trimmed: true,
-      maxConcurrentProcesses: 1, // Concurrent runs on the same repository race the index and the commit-graph and can corrupt them
-    });
-
+    this.#connect();
     await this.git.init();
 
     return this.git
@@ -37,6 +33,21 @@ export default class Git {
       .addConfig('gc.writeCommitGraph', false); // Prevent automatic `git gc` from also writing the commit-graph: the engine writes it explicitly (see `writeCommitGraph`/`updateCommitGraph`), and a concurrent gc write races those, which can leave a stale `commit-graph.lock` and make subsequent operations fail
   }
 
+  open() {
+    if (!fsApi.existsSync(path.join(this.path, '.git'))) {
+      throw new Error(`Repository ${this.path} does not exist, it has to be created by a writer such as the tracker first`); // Without this check, git would silently walk up to an enclosing repository, such as the collection one
+    }
+
+    this.#connect();
+  }
+
+  #connect() {
+    this.git = simpleGit(this.path, {
+      trimmed: true,
+      maxConcurrentProcesses: 1, // Concurrent runs on the same repository race the index and the commit-graph and can corrupt them
+    });
+  }
+
   add(filePath) {
     return this.git.add(this.relativePath(filePath));
   }
@@ -129,6 +140,10 @@ export default class Git {
     return this.git.show(options);
   }
 
+  showBuffer(options) {
+    return this.git.showBuffer(options);
+  }
+
   async cleanUp() {
     await fs.rm(path.join(this.path, '.git', 'objects', 'info', 'commit-graph.lock'), { force: true }); // Remove a leftover commit-graph lock from a previous `commit-graph write` that was killed mid-write (e.g. the process was terminated during a deploy or restart). The commit-graph is a disposable cache rebuilt by `writeCommitGraph`, so clearing a stale lock is safe and prevents every subsequent run from failing.
     await this.git.reset('hard');
@@ -144,15 +159,6 @@ export default class Git {
     ]);
   }
 
-  restore(path, commit) {
-    return this.git.raw([
-      'restore',
-      '-s', commit, // Take the file contents from this specific commit rather than from the index
-      '--', // Everything after is a pathspec, not a revision or option
-      path,
-    ]);
-  }
-
   async destroyHistory() {
     await fs.rm(this.path, { recursive: true });
 

package/src/archivist/recorder/repositories/git/index.js

@@ -35,15 +35,22 @@ function canMatchRecordFilePath(...pathSegments) {
 }
 
 export default class GitRepository extends RepositoryInterface {
-  constructor({ path, author, publish, snapshotIdentiferTemplate }) {
+  constructor({ path, author, publish, snapshotIdentiferTemplate, readOnly = false }) {
     super();
     this.path = path;
     this.needsPublication = publish;
+    this.readOnly = readOnly; // Readers share the repository with the tracker, so they must never touch the working tree nor the commit-graph: both race the tracker, and the commit-graph write then fails with `commit-graph.lock: File exists`
     this.git = new Git({ path: this.path, author });
     this.snapshotIdentiferTemplate = snapshotIdentiferTemplate;
   }
 
   async initialize() {
+    if (this.readOnly) {
+      this.git.open();
+
+      return this;
+    }
+
     await this.git.initialize();
     await this.git.cleanUp(); // Drop all uncommitted changes and remove all leftover files that may be present if the process was killed aggressively
     await this.git.writeCommitGraph(); // Create or replace the commit graph with a new one to ensure it's fully consistent
@@ -52,6 +59,8 @@ export default class GitRepository extends RepositoryInterface {
   }
 
   async save(record) {
+    this.#assertWritable('save records');
+
     const { serviceId, termsType, documentId, fetchDate } = record;
 
     if (record.isFirstRecord === undefined || record.isFirstRecord === null) {
@@ -75,6 +84,10 @@ export default class GitRepository extends RepositoryInterface {
   }
 
   async finalize() {
+    if (this.readOnly) {
+      return;
+    }
+
     if (this.needsPublication) {
       await this.git.pushChanges();
     }
@@ -214,31 +227,20 @@ export default class GitRepository extends RepositoryInterface {
     }
   }
 
-  removeAll() {
-    return this.git.destroyHistory();
+  async removeAll() {
+    this.#assertWritable('remove records');
+
+    await this.git.destroyHistory();
   }
 
   async loadRecordContent(record) {
     const relativeFilePath = DataMapper.generateFilePath(record.serviceId, record.termsType, record.documentId, record.mimeType);
 
-    if (record.mimeType != mime.getType('pdf')) {
-      record.content = await this.git.show(`${record.id}:${relativeFilePath}`);
-
-      return;
-    }
-
-    // In case of PDF files, `git show` cannot be used as it converts PDF binary into strings that do not retain the original binary representation
-    // It is impossible to restore the original binary data from the resulting string
-    let pdfBuffer;
+    const objectPath = `${record.id}:${relativeFilePath}`;
 
-    try {
-      await this.git.restore(relativeFilePath, record.id); // Temporarily restore the PDF file to a specific commit
-      pdfBuffer = await fs.readFile(`${this.path}/${relativeFilePath}`); // …read the content
-    } finally {
-      await this.git.restore(relativeFilePath, 'HEAD'); // …and finally restore the file to its most recent state
-    }
-
-    record.content = pdfBuffer;
+    record.content = record.mimeType == mime.getType('pdf')
+      ? await this.git.showBuffer(objectPath) // Binary content is read straight from the object database, so nothing is ever written to the working tree shared with the tracker
+      : await this.git.show(objectPath);
   }
 
   getDiffStats(recordId) {
@@ -308,6 +310,12 @@ export default class GitRepository extends RepositoryInterface {
     return this.git.isTracked(`${this.path}/${DataMapper.generateFilePath(serviceId, termsType, documentId)}`);
   }
 
+  #assertWritable(operation) {
+    if (this.readOnly) {
+      throw new Error(`Cannot ${operation} in read-only repository ${this.path}`);
+    }
+  }
+
   async #toDomain(commit, { deferContentLoading } = {}) {
     if (!commit) {
       return null;

package/src/archivist/recorder/repositories/git/index.test.js

@@ -2,7 +2,8 @@ import fs from 'fs';
 import path from 'path';
 import { fileURLToPath } from 'url';
 
-import { expect } from 'chai';
+import { expect, use } from 'chai';
+import chaiAsPromised from 'chai-as-promised';
 import config from 'config';
 import mime from 'mime';
 
@@ -14,6 +15,8 @@ import Git from './git.js';
 
 import GitRepository from './index.js';
 
+use(chaiAsPromised);
+
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 const RECORDER_PATH = path.resolve(__dirname, '../../../../..', config.get('@opentermsarchive/engine.recorder.versions.storage.git.path'));
@@ -2063,4 +2066,75 @@ describe('GitRepository', () => {
       });
     });
   });
+
+  context('when read-only', () => {
+    const UNTRACKED_FILE_PATH = `${RECORDER_PATH}/untracked-file.md`;
+    const COMMIT_GRAPH_PATH = `${RECORDER_PATH}/.git/objects/info/commit-graph`;
+
+    let writer;
+    let record;
+
+    before(async () => {
+      writer = new GitRepository({
+        ...config.get('@opentermsarchive/engine.recorder.versions.storage.git'),
+        path: RECORDER_PATH,
+      });
+
+      await writer.git.initialize(); // Bypass GitRepository#initialize, which writes the commit-graph, to check that the read-only repository does not write it either
+      fs.rmSync(COMMIT_GRAPH_PATH, { force: true });
+
+      record = await writer.save(new Version({
+        serviceId: SERVICE_PROVIDER_ID,
+        termsType: TERMS_TYPE,
+        content: CONTENT,
+        fetchDate: FETCH_DATE,
+        snapshotIds: [SNAPSHOT_ID],
+      }));
+
+      fs.writeFileSync(UNTRACKED_FILE_PATH, CONTENT);
+
+      subject = new GitRepository({
+        ...config.get('@opentermsarchive/engine.recorder.versions.storage.git'),
+        path: RECORDER_PATH,
+        readOnly: true,
+      });
+
+      await subject.initialize();
+      await subject.finalize();
+    });
+
+    after(() => writer.removeAll());
+
+    it('reads the records', async () => {
+      const records = await subject.findAll();
+
+      expect(records.map(({ id }) => id)).to.deep.equal([record.id]);
+    });
+
+    it('leaves uncommitted changes untouched', () => {
+      expect(fs.existsSync(UNTRACKED_FILE_PATH)).to.be.true;
+    });
+
+    it('does not write the commit-graph', () => {
+      expect(fs.existsSync(COMMIT_GRAPH_PATH)).to.be.false;
+    });
+
+    it('rejects saving records', async () => {
+      await expect(subject.save(record)).to.be.rejectedWith(Error, /read-only/);
+    });
+
+    it('rejects removing records', async () => {
+      await expect(subject.removeAll()).to.be.rejectedWith(Error, /read-only/);
+    });
+
+    it('rejects opening a missing repository', async () => {
+      const missingRepository = new GitRepository({
+        ...config.get('@opentermsarchive/engine.recorder.versions.storage.git'),
+        path: `${RECORDER_PATH}-missing`,
+        readOnly: true,
+      });
+
+      await expect(missingRepository.initialize()).to.be.rejectedWith(Error, /does not exist/);
+    });
+  });
 });

package/src/archivist/recorder/repositories/mongo/index.js

@@ -10,12 +10,13 @@ import RepositoryInterface from '../interface.js';
 import * as DataMapper from './dataMapper.js';
 
 export default class MongoRepository extends RepositoryInterface {
-  constructor({ database: databaseName, collection: collectionName, connectionURI }) {
+  constructor({ database: databaseName, collection: collectionName, connectionURI, readOnly = false }) {
     super();
 
     this.client = new MongoClient(connectionURI);
     this.databaseName = databaseName;
     this.collectionName = collectionName;
+    this.readOnly = readOnly;
   }
 
   async initialize() {
@@ -34,6 +35,8 @@ export default class MongoRepository extends RepositoryInterface {
   }
 
   async save(record) {
+    this.#assertWritable('save records');
+
     const { serviceId, termsType, documentId } = record;
 
     if (record.isFirstRecord === undefined || record.isFirstRecord === null) {
@@ -223,8 +226,16 @@ export default class MongoRepository extends RepositoryInterface {
     }
   }
 
-  removeAll() {
-    return this.collection.deleteMany();
+  async removeAll() {
+    this.#assertWritable('remove records');
+
+    await this.collection.deleteMany();
+  }
+
+  #assertWritable(operation) {
+    if (this.readOnly) {
+      throw new Error(`Cannot ${operation} in read-only repository ${this.databaseName}.${this.collectionName}`);
+    }
   }
 
   async loadRecordContent(record) {

package/src/archivist/recorder/repositories/mongo/index.test.js

@@ -2,7 +2,8 @@ import fs from 'fs';
 import path from 'path';
 import { fileURLToPath } from 'url';
 
-import { expect } from 'chai';
+import { expect, use } from 'chai';
+import chaiAsPromised from 'chai-as-promised';
 import config from 'config';
 import mime from 'mime';
 import { MongoClient, ObjectId } from 'mongodb';
@@ -12,6 +13,8 @@ import Version from '../../version.js';
 
 import MongoRepository from './index.js';
 
+use(chaiAsPromised);
+
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 const { connectionURI } = config.get('@opentermsarchive/engine.recorder.snapshots.storage.mongo');
@@ -1944,4 +1947,27 @@ describe('MongoRepository', () => {
       });
     });
   });
+
+  context('when read-only', () => {
+    before(() => {
+      subject = new MongoRepository({
+        ...config.get('@opentermsarchive/engine.recorder.versions.storage.mongo'),
+        readOnly: true,
+      });
+    });
+
+    it('rejects saving records', async () => {
+      await expect(subject.save(new Version({
+        serviceId: SERVICE_PROVIDER_ID,
+        termsType: TERMS_TYPE,
+        content: CONTENT,
+        fetchDate: FETCH_DATE,
+        snapshotIds: [SNAPSHOT_ID],
+      }))).to.be.rejectedWith(Error, /read-only/);
+    });
+
+    it('rejects removing records', async () => {
+      await expect(subject.removeAll()).to.be.rejectedWith(Error, /read-only/);
+    });
+  });
 });

package/src/collection-api/routes/index.js

@@ -39,8 +39,8 @@ export default async function apiRouter(basePath) {
   const services = await Services.load();
   const collection = await getCollection();
   const versionsStorageConfig = config.get('@opentermsarchive/engine.recorder.versions.storage');
-  const versionsRepository = await RepositoryFactory.create(versionsStorageConfig).initialize();
-  const snapshotsRepository = await RepositoryFactory.create(config.get('@opentermsarchive/engine.recorder.snapshots.storage')).initialize();
+  const versionsRepository = await RepositoryFactory.create(versionsStorageConfig, { readOnly: true }).initialize(); // The API only reads the repositories the tracker writes to
+  const snapshotsRepository = await RepositoryFactory.create(config.get('@opentermsarchive/engine.recorder.snapshots.storage'), { readOnly: true }).initialize();
   const feedConfig = config.get('@opentermsarchive/engine.collection-api.feed');
   const datasetStorage = new DatasetStorage(config.get('@opentermsarchive/engine.dataset.storagePath'));