@opentermsarchive/engine 0.15.0

package/src/archivist/recorder/repositories/git/git.js

@@ -0,0 +1,122 @@
+import fsApi from 'fs';
+import path from 'path';
+
+import simpleGit from 'simple-git';
+
+process.env.LC_ALL = 'en_GB'; // Ensure git messages will be in English as some errors are handled by analysing the message content
+
+const fs = fsApi.promises;
+
+export default class Git {
+  constructor({ path: repositoryPath, author }) {
+    this.path = repositoryPath;
+    this.author = author;
+  }
+
+  async initialize() {
+    if (!fsApi.existsSync(this.path)) {
+      await fs.mkdir(this.path, { recursive: true });
+    }
+
+    this.git = simpleGit(this.path, { maxConcurrentProcesses: 1 });
+
+    await this.git.init();
+
+    return this.git
+      .addConfig('core.autocrlf', false)
+      .addConfig('push.default', 'current')
+      .addConfig('user.name', this.author.name)
+      .addConfig('user.email', this.author.email);
+  }
+
+  async add(filePath) {
+    return this.git.add(this.relativePath(filePath));
+  }
+
+  async commit({ filePath, message, date = new Date() }) {
+    const commitDate = new Date(date).toISOString();
+    let summary;
+
+    try {
+      process.env.GIT_AUTHOR_DATE = commitDate;
+      process.env.GIT_COMMITTER_DATE = commitDate;
+
+      summary = await this.git.commit(message, filePath);
+    } finally {
+      process.env.GIT_AUTHOR_DATE = '';
+      process.env.GIT_COMMITTER_DATE = '';
+    }
+
+    if (!summary.commit) { // Nothing committed, no hash to return
+      return;
+    }
+
+    return summary.commit;
+  }
+
+  async pushChanges() {
+    return this.git.push();
+  }
+
+  async listCommits(options = []) {
+    return this.log([ '--reverse', '--no-merges', '--name-only', ...options ]);
+  }
+
+  async getCommit(options) {
+    const [commit] = await this.listCommits([ '-1', ...options ]);
+
+    return commit;
+  }
+
+  async log(options = []) {
+    try {
+      const logSummary = await this.git.log(options);
+
+      return logSummary.all;
+    } catch (error) {
+      if (/unknown revision or path not in the working tree|does not have any commits yet/.test(error.message)) {
+        return [];
+      }
+
+      throw error;
+    }
+  }
+
+  async isTracked(filePath) {
+    const result = await this.git.raw('ls-files', this.relativePath(filePath));
+
+    return Boolean(result);
+  }
+
+  async checkout(options) {
+    return this.git.checkout(options);
+  }
+
+  async show(options) {
+    return this.git.show(options);
+  }
+
+  async cleanUp() {
+    await this.git.reset('hard');
+
+    return this.git.clean('f', '-d');
+  }
+
+  async getFullHash(shortHash) {
+    return (await this.git.show([ shortHash, '--pretty=%H', '-s' ])).trim();
+  }
+
+  async restore(path, commit) {
+    return this.git.raw([ 'restore', '-s', commit, '--', path ]);
+  }
+
+  async destroyHistory() {
+    await fs.rm(this.path, { recursive: true });
+
+    return this.initialize();
+  }
+
+  relativePath(absolutePath) {
+    return path.relative(this.path, absolutePath); // Git needs a path relative to the .git directory, not an absolute one
+  }
+}

package/src/archivist/recorder/repositories/git/git.test.js

@@ -0,0 +1,86 @@
+import fs from 'fs/promises';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+import { expect } from 'chai';
+import config from 'config';
+
+import Git from './git.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const RECORDER_PATH = path.resolve(__dirname, '../../../../../', config.get('recorder.versions.storage.git.path'));
+
+describe('Git', () => {
+  const DEFAULT_CONTENT = 'default content';
+  const DEFAULT_COMMIT_MESSAGE = 'default commit message';
+  let subject;
+
+  before(() => {
+    subject = new Git({
+      path: RECORDER_PATH,
+      author: {
+        name: config.get('recorder.versions.storage.git.author.name'),
+        email: config.get('recorder.versions.storage.git.author.email'),
+      },
+    });
+
+    return subject.initialize();
+  });
+
+  describe('#commit', () => {
+    const expectedFilePath = `${RECORDER_PATH}/test.md`;
+
+    let commitId;
+
+    before(async () => {
+      await fs.writeFile(expectedFilePath, DEFAULT_CONTENT);
+
+      await subject.add(expectedFilePath);
+      commitId = await subject.commit({ filePath: expectedFilePath, message: DEFAULT_COMMIT_MESSAGE });
+    });
+
+    after(() => subject.destroyHistory());
+
+    it('returns a full length SHA1 commit ID', () => {
+      expect(commitId).to.match(/\b[0-9a-f]{40}\b/);
+    });
+
+    context('when stage area is dirty', () => {
+      const expectedFileName = 'file-to-commit.md';
+      const expectedFilePath = `${RECORDER_PATH}/${expectedFileName}`;
+      const unwantedFilePath = `${RECORDER_PATH}/unwanted-file.md`;
+      let commit;
+      let committedFiles;
+      let committedFileName;
+
+      before(async () => {
+        await fs.writeFile(expectedFilePath, DEFAULT_CONTENT);
+        await fs.writeFile(unwantedFilePath, DEFAULT_CONTENT);
+
+        await subject.add(expectedFilePath);
+        await subject.add(unwantedFilePath);
+
+        const commitId = await subject.commit({ filePath: expectedFilePath, message: DEFAULT_COMMIT_MESSAGE });
+
+        commit = await subject.getCommit([commitId]);
+
+        if (!commit) {
+          return;
+        }
+
+        ({ files: committedFiles } = commit.diff);
+        ([{ file: committedFileName }] = committedFiles);
+      });
+
+      after(() => subject.destroyHistory());
+
+      it('commits the specified file', () => {
+        expect(committedFileName).to.equal(expectedFileName);
+      });
+
+      it('commits only one file', () => {
+        expect(committedFiles).to.have.lengthOf(1);
+      });
+    });
+  });
+});

package/src/archivist/recorder/repositories/git/index.js

@@ -0,0 +1,182 @@
+/**
+ * This module is the boundary beyond which the usage of git is abstracted.
+ * Commit SHAs are used as opaque unique IDs.
+ */
+
+import fsApi from 'fs';
+import path from 'path';
+
+import mime from 'mime';
+
+import RepositoryInterface from '../interface.js';
+
+import * as DataMapper from './dataMapper.js';
+import Git from './git.js';
+
+const fs = fsApi.promises;
+
+mime.define({ 'text/markdown': ['md'] }, true); // ensure extension for markdown files is `.md` and not `.markdown`
+
+export default class GitRepository extends RepositoryInterface {
+  constructor({ path, author, publish, snapshotIdentiferTemplate }) {
+    super();
+    this.path = path;
+    this.needsPublication = publish;
+    this.git = new Git({ path: this.path, author });
+    this.snapshotIdentiferTemplate = snapshotIdentiferTemplate;
+  }
+
+  async initialize() {
+    await this.git.initialize();
+    await this.git.cleanUp(); // Drop all uncommitted changes and remove all leftover files that may be present if the process was killed aggressively
+
+    return this;
+  }
+
+  async save(record) {
+    const { serviceId, documentType, pageId, fetchDate } = record;
+
+    if (record.isFirstRecord === undefined || record.isFirstRecord === null) {
+      record.isFirstRecord = !await this.#isTracked(serviceId, documentType, pageId);
+    }
+
+    const { message, content, filePath: relativeFilePath } = await this.#toPersistence(record);
+
+    const filePath = path.join(this.path, relativeFilePath);
+
+    await GitRepository.#writeFile({ filePath, content });
+    const sha = await this.#commit({ filePath, message, date: fetchDate });
+
+    if (!sha) {
+      return Object(null);
+    }
+
+    record.id = sha;
+
+    return record;
+  }
+
+  finalize() {
+    if (!this.needsPublication) {
+      return;
+    }
+
+    return this.git.pushChanges();
+  }
+
+  async findLatest(serviceId, documentType, pageId) {
+    const filePath = DataMapper.generateFilePath(serviceId, documentType, pageId);
+    const commit = await this.git.getCommit([filePath]);
+
+    return this.#toDomain(commit);
+  }
+
+  async findById(recordId) {
+    const commit = await this.git.getCommit([recordId]);
+
+    return this.#toDomain(commit);
+  }
+
+  async findAll() {
+    return Promise.all((await this.#getCommits()).map(commit => this.#toDomain(commit, { deferContentLoading: true })));
+  }
+
+  async count() {
+    return (await this.git.log([
+      `--grep=${DataMapper.COMMIT_MESSAGE_PREFIX.startTracking}`,
+      `--grep=${DataMapper.COMMIT_MESSAGE_PREFIX.refilter}`,
+      `--grep=${DataMapper.COMMIT_MESSAGE_PREFIX.update}`,
+    ])).length;
+  }
+
+  async* iterate() {
+    const commits = await this.#getCommits();
+
+    for (const commit of commits) {
+      yield this.#toDomain(commit);
+    }
+  }
+
+  async removeAll() {
+    return this.git.destroyHistory();
+  }
+
+  async loadRecordContent(record) {
+    const relativeFilePath = DataMapper.generateFilePath(record.serviceId, record.documentType, record.pageId, record.mimeType);
+
+    if (record.mimeType != mime.getType('pdf')) {
+      record.content = await this.git.show(`${record.id}:${relativeFilePath}`);
+
+      return;
+    }
+
+    // In case of PDF files, `git show` cannot be used as it converts PDF binary into strings that do not retain the original binary representation
+    // It is impossible to restore the original binary data from the resulting string
+    let pdfBuffer;
+
+    try {
+      await this.git.restore(relativeFilePath, record.id); // Temporarily restore the PDF file to a specific commit
+      pdfBuffer = await fs.readFile(`${this.path}/${relativeFilePath}`); // …read the content
+    } finally {
+      await this.git.restore(relativeFilePath, 'HEAD'); // …and finally restore the file to its most recent state
+    }
+
+    record.content = pdfBuffer;
+  }
+
+  async #getCommits() {
+    return (await this.git.listCommits())
+      .filter(({ message }) => message.match(DataMapper.COMMIT_MESSAGE_PREFIXES_REGEXP)) // Skip commits which are not a document record (README, LICENSE…)
+      .sort((commitA, commitB) => new Date(commitA.date) - new Date(commitB.date)); // Make sure that the commits are sorted in ascending chronological order
+  }
+
+  static async #writeFile({ filePath, content }) {
+    const directory = path.dirname(filePath);
+
+    if (!fsApi.existsSync(directory)) {
+      await fs.mkdir(directory, { recursive: true });
+    }
+
+    await fs.writeFile(filePath, content);
+
+    return filePath;
+  }
+
+  async #commit({ filePath, message, date }) {
+    try {
+      await this.git.add(filePath);
+
+      return await this.git.commit({ filePath, message, date });
+    } catch (error) {
+      throw new Error(`Could not commit ${filePath} with message "${message}" due to error: "${error}"`);
+    }
+  }
+
+  #isTracked(serviceId, documentType, pageId) {
+    return this.git.isTracked(`${this.path}/${DataMapper.generateFilePath(serviceId, documentType, pageId)}`);
+  }
+
+  async #toDomain(commit, { deferContentLoading } = {}) {
+    if (!commit) {
+      return null;
+    }
+
+    const record = DataMapper.toDomain(commit);
+
+    if (deferContentLoading) {
+      return record;
+    }
+
+    await this.loadRecordContent(record);
+
+    return record;
+  }
+
+  async #toPersistence(record) {
+    if (record.content === undefined || record.content === null) {
+      await this.loadRecordContent(record);
+    }
+
+    return DataMapper.toPersistence(record, this.snapshotIdentiferTemplate);
+  }
+}