@opentermsarchive/engine 2.7.0 → 2.7.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "2.7.0",
+  "version": "2.7.2",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {
@@ -96,7 +96,7 @@
     "puppeteer-extra": "^3.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2",
     "sib-api-v3-sdk": "^8.2.1",
-    "simple-git": "^3.8.0",
+    "simple-git": "^3.27.0",
     "swagger-jsdoc": "^6.2.8",
     "swagger-ui-express": "^5.0.0",
     "winston": "^3.9.0",

package/scripts/dataset/logger/index.js

@@ -7,7 +7,7 @@ const { combine, timestamp, printf, colorize } = winston.format;
 
 logger.format = combine(
   colorize(),
-  timestamp({ format: 'YYYY-MM-DDTHH:MM:SSZ' }),
+  timestamp({ format: 'YYYY-MM-DDTHH:mm:ssZ' }),
   printf(({ level, message, counter, hash, timestamp }) => {
     const prefix = counter && hash ? `${counter.toString().padEnd(6)} ${hash.padEnd(40)}` : '';
 

package/src/archivist/index.js

@@ -28,6 +28,7 @@ export const EVENTS = [
   'trackingStarted',
   'trackingCompleted',
   'inaccessibleContent',
+  'info',
   'error',
   'pluginError',
 ];
@@ -45,6 +46,7 @@ export default class Archivist extends events.EventEmitter {
   }
 
   async initialize() {
+    this.emit('info', 'Initializing engine…');
     if (this.services) {
       return;
     }
@@ -67,6 +69,8 @@ export default class Archivist extends events.EventEmitter {
       process.exit(1);
     });
 
+    this.emit('info', 'Initialization completed');
+
     return this;
   }
 
@@ -140,7 +144,13 @@ export default class Archivist extends events.EventEmitter {
       return;
     }
 
-    return this.recordVersion(terms, extractOnly);
+    await this.recordVersion(terms, extractOnly);
+
+    terms.sourceDocuments.forEach(sourceDocument => {
+      sourceDocument.content = null; // Reduce memory usage by clearing no longer needed large content strings
+      sourceDocument.mimeType = null; // …and associated MIME type
+      sourceDocument.snapshotId = null; // …and associated snapshot ID for consistency
+    });
   }
 
   async fetchSourceDocuments(terms) {

package/src/archivist/recorder/repositories/git/git.js

@@ -18,7 +18,10 @@ export default class Git {
       await fs.mkdir(this.path, { recursive: true });
     }
 
-    this.git = simpleGit(this.path, { maxConcurrentProcesses: 1 });
+    this.git = simpleGit(this.path, {
+      trimmed: true,
+      maxConcurrentProcesses: 1,
+    });
 
     await this.git.init();
 
@@ -27,7 +30,8 @@ export default class Git {
       .addConfig('push.default', 'current')
       .addConfig('user.name', this.author.name)
       .addConfig('user.email', this.author.email)
-      .addConfig('core.quotePath', false); // disable Git's encoding of special characters in pathnames. For example, `service·A` will be encoded as `service\302\267A` without this setting, leading to issues. See https://git-scm.com/docs/git-config#Documentation/git-config.txt-corequotePath
+      .addConfig('core.quotePath', false) // Disable Git's encoding of special characters in pathnames. For example, `service·A` will be encoded as `service\302\267A` without this setting, leading to issues. See https://git-scm.com/docs/git-config#Documentation/git-config.txt-corequotePath
+      .addConfig('core.commitGraph', true); // Enable `commit-graph` feature for efficient commit data storage, improving performance of operations like `git log`
   }
 
   add(filePath) {
@@ -42,7 +46,7 @@ export default class Git {
       process.env.GIT_AUTHOR_DATE = commitDate;
       process.env.GIT_COMMITTER_DATE = commitDate;
 
-      summary = await this.git.commit(message, filePath);
+      summary = await this.git.commit(message, filePath, ['--no-verify']); // Skip pre-commit and commit-msg hooks, as commits are programmatically managed, to optimize performance
     } finally {
       process.env.GIT_AUTHOR_DATE = '';
       process.env.GIT_COMMITTER_DATE = '';
@@ -60,11 +64,11 @@ export default class Git {
   }
 
   listCommits(options = []) {
-    return this.log([ '--reverse', '--no-merges', '--name-only', ...options ]);
+    return this.log([ '--reverse', '--no-merges', '--name-only', ...options ]); // Returns all commits in chronological order (`--reverse`), excluding merge commits (`--no-merges`), with modified files names (`--name-only`)
   }
 
   async getCommit(options) {
-    const [commit] = await this.listCommits([ '-1', ...options ]);
+    const [commit] = await this.listCommits([ '-1', ...options ]); // Returns only the most recent commit matching the given options
 
     return commit;
   }
@@ -103,8 +107,8 @@ export default class Git {
     return this.git.clean('f', '-d');
   }
 
-  async getFullHash(shortHash) {
-    return (await this.git.show([ shortHash, '--pretty=%H', '-s' ])).trim();
+  getFullHash(shortHash) {
+    return this.git.show([ shortHash, '--pretty=%H', '-s' ]);
   }
 
   restore(path, commit) {
@@ -120,4 +124,16 @@ export default class Git {
   relativePath(absolutePath) {
     return path.relative(this.path, absolutePath); // Git needs a path relative to the .git directory, not an absolute one
   }
+
+  async listFiles(path) {
+    return (await this.git.raw([ 'ls-files', path ])).split('\n');
+  }
+
+  async writeCommitGraph() {
+    await this.git.raw([ 'commit-graph', 'write', '--reachable', '--changed-paths' ]);
+  }
+
+  async updateCommitGraph() {
+    await this.git.raw([ 'commit-graph', 'write', '--reachable', '--changed-paths', '--append' ]);
+  }
 }

package/src/archivist/recorder/repositories/git/index.js

@@ -29,6 +29,7 @@ export default class GitRepository extends RepositoryInterface {
   async initialize() {
     await this.git.initialize();
     await this.git.cleanUp(); // Drop all uncommitted changes and remove all leftover files that may be present if the process was killed aggressively
+    await this.git.writeCommitGraph(); // Create or replace the commit graph with a new one to ensure it's fully consistent
 
     return this;
   }
@@ -56,17 +57,22 @@ export default class GitRepository extends RepositoryInterface {
     return record;
   }
 
-  finalize() {
-    if (!this.needsPublication) {
-      return;
+  async finalize() {
+    if (this.needsPublication) {
+      await this.git.pushChanges();
     }
 
-    return this.git.pushChanges();
+    return this.git.updateCommitGraph();
   }
 
   async findLatest(serviceId, termsType, documentId) {
-    const filePath = DataMapper.generateFilePath(serviceId, termsType, documentId);
-    const commit = await this.git.getCommit([filePath]);
+    const matchingFilesPaths = await this.git.listFiles(DataMapper.generateFilePath(serviceId, termsType, documentId));
+
+    if (!matchingFilesPaths.length) {
+      return null;
+    }
+
+    const commit = await this.git.getCommit([...matchingFilesPaths]); // Returns the most recent commit that modified any of the matching files. If multiple files match the path pattern (e.g. both HTML and PDF versions exist), returns the commit that last modified any of them
 
     return this.#toDomain(commit);
   }

package/src/collection-api/logger.js

@@ -30,7 +30,7 @@ if (config.get('@opentermsarchive/engine.logger.sendMailOnError')) {
 const logger = winston.createLogger({
   format: combine(
     colorize(),
-    timestamp({ format: 'YYYY-MM-DDTHH:MM:SSZ' }),
+    timestamp({ format: 'YYYY-MM-DDTHH:mm:ssZ' }),
     printf(({ level, message, timestamp }) => {
       const timestampPrefix = config.get('@opentermsarchive/engine.logger.timestampPrefix') ? `${timestamp} ` : '';
 

package/src/logger/index.js

@@ -4,11 +4,13 @@ import config from 'config';
 import winston from 'winston';
 import 'winston-mail';
 
+import { formatDuration } from './utils.js';
+
 const { combine, timestamp, printf, colorize } = winston.format;
 
 const alignedWithColorsAndTime = combine(
   colorize(),
-  timestamp({ format: 'YYYY-MM-DDTHH:MM:SSZ' }),
+  timestamp({ format: 'YYYY-MM-DDTHH:mm:ssZ' }),
   printf(({ level, message, timestamp, serviceId, termsType, documentId }) => {
     const servicePrefix = serviceId && termsType
       ? `${serviceId} — ${termsType}${documentId ? `:${documentId}` : ''}`
@@ -82,6 +84,7 @@ logger.configure({
 
 let recordedSnapshotsCount;
 let recordedVersionsCount;
+let trackingStartTime;
 
 logger.onFirstSnapshotRecorded = ({ serviceId, termsType, documentId, id }) => {
   logger.info({ message: `Recorded first snapshot with id ${id}`, serviceId, termsType, documentId });
@@ -119,14 +122,17 @@ logger.onTrackingStarted = (numberOfServices, numberOfTerms, extractOnly) => {
   }
   recordedSnapshotsCount = 0;
   recordedVersionsCount = 0;
+  trackingStartTime = Date.now();
 };
 
 logger.onTrackingCompleted = (numberOfServices, numberOfTerms, extractOnly) => {
+  const duration = formatDuration(Date.now() - trackingStartTime);
+
   if (extractOnly) {
-    logger.info(`Examined ${numberOfTerms} terms from ${numberOfServices} services for extraction`);
+    logger.info(`Examined ${numberOfTerms} terms from ${numberOfServices} services for extraction in ${duration}`);
     logger.info(`Recorded ${recordedVersionsCount} new versions\n`);
   } else {
-    logger.info(`Tracked changes of ${numberOfTerms} terms from ${numberOfServices} services`);
+    logger.info(`Tracked changes of ${numberOfTerms} terms from ${numberOfServices} services in ${duration}`);
     logger.info(`Recorded ${recordedSnapshotsCount} new snapshots and ${recordedVersionsCount} new versions\n`);
   }
 };
@@ -139,6 +145,10 @@ logger.onError = (error, terms) => {
   logger.error({ message: error.stack, serviceId: terms.service.id, termsType: terms.type });
 };
 
+logger.onInfo = message => {
+  logger.info({ message });
+};
+
 logger.onPluginError = (error, pluginName) => {
   logger.error({ message: `Error in "${pluginName}" plugin: ${error.stack}` });
 };

package/src/logger/utils.js

@@ -0,0 +1,22 @@
+export const formatDuration = milliseconds => {
+  const seconds = Math.floor(milliseconds / 1000);
+  const hours = Math.floor(seconds / 3600);
+  const minutes = Math.floor((seconds % 3600) / 60);
+  const remainingSeconds = seconds % 60;
+
+  const parts = [];
+
+  if (hours > 0) {
+    parts.push(`${hours} hour${hours > 1 ? 's' : ''}`);
+  }
+
+  if (minutes > 0) {
+    parts.push(`${minutes} minute${minutes > 1 ? 's' : ''}`);
+  }
+
+  if (remainingSeconds > 0 || parts.length === 0) {
+    parts.push(`${remainingSeconds} second${remainingSeconds !== 1 ? 's' : ''}`);
+  }
+
+  return parts.join(' and ');
+};