@opentermsarchive/engine 0.26.1 → 0.27.0

package/src/archivist/recorder/record.js

@@ -1,10 +1,17 @@
+/**
+ * Abstract Class Record.
+ *
+ * @class Record
+ */
 export default class Record {
   #content;
 
-  static REQUIRED_PARAMS = Object.freeze([ 'serviceId', 'documentType', 'mimeType', 'fetchDate' ]);
+  static REQUIRED_PARAMS = Object.freeze([ 'serviceId', 'termsType', 'fetchDate', 'content' ]);
 
   constructor(params) {
-    Record.validate(params);
+    if (this.constructor == Record) {
+      throw new Error("Abstract Record class can't be instantiated.");
+    }
 
     Object.assign(this, Object.fromEntries(Object.entries(params)));
 
@@ -15,7 +22,7 @@ export default class Record {
 
   get content() {
     if (this.#content === undefined) {
-      throw new Error('Record content not defined, set the content or use Repository#loadRecordContent');
+      throw new Error('Content not defined, set the content or use Repository#loadRecordContent');
     }
 
     return this.#content;
@@ -25,10 +32,14 @@ export default class Record {
     this.#content = content;
   }
 
-  static validate(givenParams) {
-    for (const param of Record.REQUIRED_PARAMS) {
-      if (!Object.prototype.hasOwnProperty.call(givenParams, param) || givenParams[param] == null) {
-        throw new Error(`"${param}" is required`);
+  validate() {
+    for (const requiredParam of this.constructor.REQUIRED_PARAMS) {
+      if (requiredParam == 'content') {
+        if (this[requiredParam] == '' || this[requiredParam] == null) {
+          throw new Error(`${this.constructor.name} is not valid; "${requiredParam}" is empty or null`);
+        }
+      } else if (!Object.prototype.hasOwnProperty.call(this, requiredParam) || this[requiredParam] == null) {
+        throw new Error(`${this.constructor.name} is not valid; "${requiredParam}" is missing`);
       }
     }
   }

package/src/archivist/recorder/repositories/git/dataMapper.js

@@ -2,44 +2,48 @@ import path from 'path';
 
 import mime from 'mime';
 
-import Record from '../../record.js';
+import Snapshot from '../../snapshot.js';
+import Version from '../../version.js';
 
 mime.define({ 'text/markdown': ['md'] }, true); // ensure extension for markdown files is `.md` and not `.markdown`
 
-export const COMMIT_MESSAGE_PREFIX = {
-  startTracking: 'Start tracking',
-  refilter: 'Refilter',
-  update: 'Update',
+export const COMMIT_MESSAGE_PREFIXES = {
+  startTracking: 'First record of',
+  extractOnly: 'Apply technical or declaration upgrade on',
+  update: 'Record new changes of',
+  deprecated_startTracking: 'Start tracking',
+  deprecated_refilter: 'Refilter',
+  deprecated_update: 'Update',
 };
 
-export const DOCUMENT_TYPE_AND_PAGE_ID_SEPARATOR = ' #';
+export const TERMS_TYPE_AND_DOCUMENT_ID_SEPARATOR = ' #';
 export const SNAPSHOT_ID_MARKER = '%SNAPSHOT_ID';
-const SINGLE_SNAPSHOT_PREFIX = 'This version was recorded after filtering snapshot';
-const MULTIPLE_SNAPSHOT_PREFIX = 'This version was recorded after filtering and assembling the following snapshots from %NUMBER pages:';
+const SINGLE_SOURCE_DOCUMENT_PREFIX = 'This version was recorded after extracting from snapshot';
+const MULTIPLE_SOURCE_DOCUMENTS_PREFIX = 'This version was recorded after extracting from and assembling the following snapshots from %NUMBER source documents:';
 
-export const COMMIT_MESSAGE_PREFIXES_REGEXP = new RegExp(`^(${COMMIT_MESSAGE_PREFIX.startTracking}|${COMMIT_MESSAGE_PREFIX.refilter}|${COMMIT_MESSAGE_PREFIX.update})`);
+export const COMMIT_MESSAGE_PREFIXES_REGEXP = new RegExp(`^(${Object.values(COMMIT_MESSAGE_PREFIXES).join('|')})`);
 
 export function toPersistence(record, snapshotIdentiferTemplate) {
-  const { serviceId, documentType, pageId, isRefilter, snapshotIds = [], mimeType, isFirstRecord } = record;
+  const { serviceId, termsType, documentId, isExtractOnly, snapshotIds = [], mimeType, isFirstRecord } = record;
 
-  let prefix = isRefilter ? COMMIT_MESSAGE_PREFIX.refilter : COMMIT_MESSAGE_PREFIX.update;
+  let prefix = isExtractOnly ? COMMIT_MESSAGE_PREFIXES.extractOnly : COMMIT_MESSAGE_PREFIXES.update;
 
-  prefix = isFirstRecord ? COMMIT_MESSAGE_PREFIX.startTracking : prefix;
+  prefix = isFirstRecord ? COMMIT_MESSAGE_PREFIXES.startTracking : prefix;
 
-  const subject = `${prefix} ${serviceId} ${documentType}`;
-  const pageIdMessage = `${pageId ? `Page ID ${pageId}\n\n` : ''}`;
+  const subject = `${prefix} ${serviceId} ${termsType}`;
+  const documentIdMessage = `${documentId ? `Document ID ${documentId}\n\n` : ''}`;
   let snapshotIdsMessage;
 
   if (snapshotIds.length == 1) {
-    snapshotIdsMessage = `${SINGLE_SNAPSHOT_PREFIX} ${snapshotIdentiferTemplate.replace(SNAPSHOT_ID_MARKER, snapshotIds[0])}`;
+    snapshotIdsMessage = `${SINGLE_SOURCE_DOCUMENT_PREFIX} ${snapshotIdentiferTemplate.replace(SNAPSHOT_ID_MARKER, snapshotIds[0])}`;
   } else if (snapshotIds.length > 1) {
-    snapshotIdsMessage = `${MULTIPLE_SNAPSHOT_PREFIX.replace('%NUMBER', snapshotIds.length)}\n${snapshotIds.map(snapshotId => `- ${snapshotIdentiferTemplate.replace(SNAPSHOT_ID_MARKER, snapshotId)}`).join('\n')}`;
+    snapshotIdsMessage = `${MULTIPLE_SOURCE_DOCUMENTS_PREFIX.replace('%NUMBER', snapshotIds.length)}\n${snapshotIds.map(snapshotId => `- ${snapshotIdentiferTemplate.replace(SNAPSHOT_ID_MARKER, snapshotId)}`).join('\n')}`;
   }
 
-  const filePath = generateFilePath(serviceId, documentType, pageId, mimeType);
+  const filePath = generateFilePath(serviceId, termsType, documentId, mimeType);
 
   return {
-    message: `${subject}\n\n${pageIdMessage || ''}\n\n${snapshotIdsMessage || ''}`,
+    message: `${subject}\n\n${documentIdMessage || ''}\n\n${snapshotIdsMessage || ''}`,
     content: record.content,
     filePath,
   };
@@ -51,33 +55,39 @@ export function toDomain(commit) {
   const modifiedFilesInCommit = diff.files.map(({ file }) => file);
 
   if (modifiedFilesInCommit.length > 1) {
-    throw new Error(`Only one document should have been recorded in ${hash}, but all these documents were recorded: ${modifiedFilesInCommit.join(', ')}`);
+    throw new Error(`Only one file should have been recorded in ${hash}, but all these files were recorded: ${modifiedFilesInCommit.join(', ')}`);
   }
 
   const [relativeFilePath] = modifiedFilesInCommit;
   const snapshotIdsMatch = body.match(/\b[0-9a-f]{5,40}\b/g);
 
-  const [ documentType, pageId ] = path.basename(relativeFilePath, path.extname(relativeFilePath)).split(DOCUMENT_TYPE_AND_PAGE_ID_SEPARATOR);
+  const [ termsType, documentId ] = path.basename(relativeFilePath, path.extname(relativeFilePath)).split(TERMS_TYPE_AND_DOCUMENT_ID_SEPARATOR);
 
-  return new Record({
+  const attributes = {
     id: hash,
     serviceId: path.dirname(relativeFilePath),
-    documentType,
-    pageId,
+    termsType,
+    documentId,
     mimeType: mime.getType(relativeFilePath),
     fetchDate: new Date(date),
-    isFirstRecord: message.startsWith(COMMIT_MESSAGE_PREFIX.startTracking),
-    isRefilter: message.startsWith(COMMIT_MESSAGE_PREFIX.refilter),
+    isFirstRecord: message.startsWith(COMMIT_MESSAGE_PREFIXES.startTracking) || message.startsWith(COMMIT_MESSAGE_PREFIXES.deprecated_startTracking),
+    isExtractOnly: message.startsWith(COMMIT_MESSAGE_PREFIXES.extractOnly) || message.startsWith(COMMIT_MESSAGE_PREFIXES.deprecated_refilter),
     snapshotIds: snapshotIdsMatch || [],
-  });
+  };
+
+  if (attributes.mimeType == mime.getType('markdown')) {
+    return new Version(attributes);
+  }
+
+  return new Snapshot(attributes);
 }
 
-function generateFileName(documentType, pageId, extension) {
-  return `${documentType}${pageId ? `${DOCUMENT_TYPE_AND_PAGE_ID_SEPARATOR}${pageId}` : ''}.${extension}`;
+function generateFileName(termsType, documentId, extension) {
+  return `${termsType}${documentId ? `${TERMS_TYPE_AND_DOCUMENT_ID_SEPARATOR}${documentId}` : ''}.${extension}`;
 }
 
-export function generateFilePath(serviceId, documentType, pageId, mimeType) {
-  const extension = mime.getExtension(mimeType) || '*'; // If mime type is undefined, an asterisk is set as an extension. Used to match all files for the given service ID, terms type and page ID when mime type is unknown.
+export function generateFilePath(serviceId, termsType, documentId, mimeType) {
+  const extension = mime.getExtension(mimeType) || '*'; // If mime type is undefined, an asterisk is set as an extension. Used to match all files for the given service ID, terms type and document ID when mime type is unknown
 
-  return `${serviceId}/${generateFileName(documentType, pageId, extension)}`; // Do not use `path.join` as even for Windows, the path should be with `/` and not `\`
+  return `${serviceId}/${generateFileName(termsType, documentId, extension)}`; // Do not use `path.join` as even for Windows, the path should be with `/` and not `\`
 }

package/src/archivist/recorder/repositories/git/index.js

@@ -34,17 +34,17 @@ export default class GitRepository extends RepositoryInterface {
   }
 
   async save(record) {
-    const { serviceId, documentType, pageId, fetchDate } = record;
+    const { serviceId, termsType, documentId, fetchDate } = record;
 
     if (record.isFirstRecord === undefined || record.isFirstRecord === null) {
-      record.isFirstRecord = !await this.#isTracked(serviceId, documentType, pageId);
+      record.isFirstRecord = !await this.#isTracked(serviceId, termsType, documentId);
     }
 
     const { message, content, filePath: relativeFilePath } = await this.#toPersistence(record);
 
     const filePath = path.join(this.path, relativeFilePath);
 
-    await GitRepository.#writeFile({ filePath, content });
+    await GitRepository.writeFile({ filePath, content });
     const sha = await this.#commit({ filePath, message, date: fetchDate });
 
     if (!sha) {
@@ -64,8 +64,8 @@ export default class GitRepository extends RepositoryInterface {
     return this.git.pushChanges();
   }
 
-  async findLatest(serviceId, documentType, pageId) {
-    const filePath = DataMapper.generateFilePath(serviceId, documentType, pageId);
+  async findLatest(serviceId, termsType, documentId) {
+    const filePath = DataMapper.generateFilePath(serviceId, termsType, documentId);
     const commit = await this.git.getCommit([filePath]);
 
     return this.#toDomain(commit);
@@ -82,11 +82,7 @@ export default class GitRepository extends RepositoryInterface {
   }
 
   async count() {
-    return (await this.git.log([
-      `--grep=${DataMapper.COMMIT_MESSAGE_PREFIX.startTracking}`,
-      `--grep=${DataMapper.COMMIT_MESSAGE_PREFIX.refilter}`,
-      `--grep=${DataMapper.COMMIT_MESSAGE_PREFIX.update}`,
-    ])).length;
+    return (await this.git.log(Object.values(DataMapper.COMMIT_MESSAGE_PREFIXES).map(prefix => `--grep=${prefix}`))).length;
   }
 
   async* iterate() {
@@ -102,7 +98,7 @@ export default class GitRepository extends RepositoryInterface {
   }
 
   async loadRecordContent(record) {
-    const relativeFilePath = DataMapper.generateFilePath(record.serviceId, record.documentType, record.pageId, record.mimeType);
+    const relativeFilePath = DataMapper.generateFilePath(record.serviceId, record.termsType, record.documentId, record.mimeType);
 
     if (record.mimeType != mime.getType('pdf')) {
       record.content = await this.git.show(`${record.id}:${relativeFilePath}`);
@@ -126,11 +122,11 @@ export default class GitRepository extends RepositoryInterface {
 
   async #getCommits() {
     return (await this.git.listCommits())
-      .filter(({ message }) => message.match(DataMapper.COMMIT_MESSAGE_PREFIXES_REGEXP)) // Skip commits which are not a document record (README, LICENSE…)
+      .filter(({ message }) => message.match(DataMapper.COMMIT_MESSAGE_PREFIXES_REGEXP)) // Skip commits which are not a record (README, LICENSE…)
       .sort((commitA, commitB) => new Date(commitA.date) - new Date(commitB.date)); // Make sure that the commits are sorted in ascending chronological order
   }
 
-  static async #writeFile({ filePath, content }) {
+  static async writeFile({ filePath, content }) {
     const directory = path.dirname(filePath);
 
     if (!fsApi.existsSync(directory)) {
@@ -152,8 +148,8 @@ export default class GitRepository extends RepositoryInterface {
     }
   }
 
-  #isTracked(serviceId, documentType, pageId) {
-    return this.git.isTracked(`${this.path}/${DataMapper.generateFilePath(serviceId, documentType, pageId)}`);
+  #isTracked(serviceId, termsType, documentId) {
+    return this.git.isTracked(`${this.path}/${DataMapper.generateFilePath(serviceId, termsType, documentId)}`);
   }
 
   async #toDomain(commit, { deferContentLoading } = {}) {