@opentermsarchive/engine 5.3.1 → 5.4.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "5.3.1",
+  "version": "5.4.1",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {

package/src/archivist/collection/index.js

@@ -0,0 +1,75 @@
+import fs from 'fs/promises';
+import path from 'path';
+
+import config from 'config';
+import yaml from 'js-yaml';
+
+export const METADATA_FILE = 'metadata.yml';
+export const INVENTORY_FILE = 'deployment/inventory.yml';
+
+class Collection {
+  metadata;
+  id;
+  name;
+  inventory;
+  host;
+  hostConfig;
+  collectionPath;
+
+  constructor() {
+    this.collectionPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.collectionPath'));
+  }
+
+  async loadYamlFile(filename) {
+    const content = await fs.readFile(path.join(this.collectionPath, filename), 'utf8');
+
+    return yaml.load(content);
+  }
+
+  async initialize() {
+    const fileLoadingPromises = await Promise.allSettled([
+      this.loadYamlFile(METADATA_FILE),
+      this.loadYamlFile(INVENTORY_FILE),
+    ]);
+
+    const [ metadata, inventory ] = fileLoadingPromises.map(({ status, value, reason }) => {
+      if (status === 'fulfilled') {
+        return value;
+      }
+
+      if (reason && reason.code !== 'ENOENT') { // Ignore missing files errors (inventory and metadata are optional), but throw other errors like invalid YAML
+        throw reason;
+      }
+
+      return null;
+    });
+
+    if (metadata) {
+      this.metadata = metadata;
+      this.id = metadata.id;
+      this.name = metadata.name;
+    }
+
+    if (inventory) {
+      const [host] = Object.keys(inventory.all.hosts);
+
+      this.inventory = inventory;
+      this.host = host;
+      this.hostConfig = inventory.all.hosts[host];
+    }
+
+    return this;
+  }
+}
+
+let instancePromise = null;
+
+export async function getCollection() {
+  if (!instancePromise) {
+    const collection = new Collection();
+
+    instancePromise = await collection.initialize();
+  }
+
+  return instancePromise;
+}

package/src/archivist/collection/index.test.js

@@ -0,0 +1,155 @@
+import fs from 'fs/promises';
+import path from 'path';
+
+import { expect } from 'chai';
+import config from 'config';
+
+import { METADATA_FILE, INVENTORY_FILE } from './index.js';
+
+describe('Collection', () => {
+  const testCollectionPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.collectionPath'));
+  const metadataPath = path.join(testCollectionPath, METADATA_FILE);
+  const inventoryPath = path.join(testCollectionPath, INVENTORY_FILE);
+  let metadataBackup;
+  let getCollection;
+  let collection;
+
+  before(async () => {
+    try {
+      metadataBackup = await fs.readFile(metadataPath, 'utf8');
+    } catch (error) {
+      if (error.code !== 'ENOENT') throw error;
+    }
+  });
+
+  after(async () => {
+    if (metadataBackup) {
+      await fs.writeFile(metadataPath, metadataBackup);
+    }
+  });
+
+  beforeEach(async () => {
+    const { getCollection: reloadedGetCollection } = await import(`./index.js?t=${Date.now()}`); // Ensure a new instance is loaded for each test
+
+    getCollection = reloadedGetCollection;
+  });
+
+  describe('singleton behavior', () => {
+    beforeEach(async () => {
+      collection = await getCollection();
+    });
+
+    it('returns same instance on multiple calls', async () => {
+      const collection2 = await getCollection();
+
+      expect(collection).to.equal(collection2);
+    });
+  });
+
+  describe('metadata handling', () => {
+    const metadata = {
+      id: 'test-collection',
+      name: 'Test Collection',
+      tagline: 'Test collection for testing',
+    };
+
+    describe('with existing metadata', () => {
+      beforeEach(async () => {
+        await fs.mkdir(testCollectionPath, { recursive: true });
+        await fs.writeFile(metadataPath, JSON.stringify(metadata));
+        collection = await getCollection();
+      });
+
+      afterEach(async () => {
+        await fs.rm(metadataPath, { recursive: true, force: true });
+      });
+
+      it('loads collection id from metadata', () => {
+        expect(collection.id).to.equal(metadata.id);
+      });
+
+      it('loads collection name from metadata', () => {
+        expect(collection.name).to.equal(metadata.name);
+      });
+
+      it('loads full metadata object', () => {
+        expect(collection.metadata).to.deep.equal(metadata);
+      });
+    });
+
+    describe('without metadata', () => {
+      beforeEach(async () => {
+        await fs.rm(metadataPath, { force: true });
+        collection = await getCollection();
+      });
+
+      it('has undefined metadata', () => {
+        expect(collection.metadata).to.be.undefined;
+      });
+
+      it('has undefined id', () => {
+        expect(collection.id).to.be.undefined;
+      });
+
+      it('has undefined name', () => {
+        expect(collection.name).to.be.undefined;
+      });
+    });
+  });
+
+  describe('inventory handling', () => {
+    const inventory = {
+      all: {
+        hosts: {
+          'test-host': {
+            ansible_host: 'localhost',
+            ansible_port: 22,
+          },
+        },
+      },
+    };
+
+    describe('with existing inventory', () => {
+      beforeEach(async () => {
+        await fs.mkdir(path.dirname(inventoryPath), { recursive: true });
+        await fs.writeFile(inventoryPath, JSON.stringify(inventory));
+        collection = await getCollection();
+      });
+
+      afterEach(async () => {
+        await fs.rm(inventoryPath, { recursive: true, force: true });
+      });
+
+      it('loads host from inventory', () => {
+        expect(collection.host).to.equal('test-host');
+      });
+
+      it('loads host config from inventory', () => {
+        expect(collection.hostConfig).to.deep.equal(inventory.all.hosts['test-host']);
+      });
+
+      it('loads full inventory object', () => {
+        expect(collection.inventory).to.deep.equal(inventory);
+      });
+    });
+
+    describe('without inventory', () => {
+      beforeEach(async () => {
+        await fs.rm(inventoryPath, { force: true });
+        collection = await getCollection();
+      });
+
+      it('has undefined inventory', () => {
+        expect(collection.inventory).to.be.undefined;
+      });
+
+      it('has undefined host', () => {
+        expect(collection.host).to.be.undefined;
+      });
+
+      it('has undefined host config', () => {
+        expect(collection.hostConfig).to.be.undefined;
+      });
+    });
+  });
+});

package/src/archivist/fetcher/htmlOnlyFetcher.js

@@ -49,6 +49,10 @@ export default async function fetch(url, config) {
       content,
     };
   } catch (error) {
+    if (error.type == 'system') { // Node-fetch wraps system-level errors (ENOTFOUND, ECONNREFUSED, ECONNRESET, etc.) with type 'system' and includes the original error code
+      throw new Error(`Network system error ${error.code} occurred when trying to fetch '${url}'`);
+    }
+
     if (error instanceof AbortError) {
       throw new Error(`Timed out after ${config.navigationTimeout / 1000} seconds when trying to fetch '${url}'`);
     }

package/src/collection-api/routes/index.js

@@ -1,9 +1,7 @@
-import path from 'path';
-
-import config from 'config';
 import express from 'express';
 import helmet from 'helmet';
 
+import { getCollection } from '../../archivist/collection/index.js';
 import * as Services from '../../archivist/services/index.js';
 
 import docsRouter from './docs.js';
@@ -33,10 +31,10 @@ export default async function apiRouter(basePath) {
     res.json({ message: 'Welcome to an instance of the Open Terms Archive API. Documentation is available at /docs. Learn more on Open Terms Archive on https://opentermsarchive.org.' });
   });
 
-  const collectionPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.collectionPath'));
   const services = await Services.load();
+  const collection = await getCollection();
 
-  router.use(await metadataRouter(collectionPath, services));
+  router.use(await metadataRouter(collection, services));
   router.use(servicesRouter(services));
   router.use(versionsRouter);
 

package/src/collection-api/routes/metadata.js

@@ -1,18 +1,11 @@
-import fs from 'fs/promises';
-import path from 'path';
-
 import express from 'express';
-import yaml from 'js-yaml';
 
 import Service from '../../archivist/services/service.js';
 
-const METADATA_FILENAME = 'metadata.yml';
-const PACKAGE_JSON_PATH = '../../../package.json';
-
 /**
- * @param   {string}         collectionPath The path to the collection
- * @param   {object}         services       The services of the collection
- * @returns {express.Router}                The router instance
+ * @param   {object}         collection The collection
+ * @param   {object}         services   The services of the collection
+ * @returns {express.Router}            The router instance
  * @swagger
  * tags:
  *   name: Metadata
@@ -175,11 +168,9 @@ const PACKAGE_JSON_PATH = '../../../package.json';
  *           additionalProperties:
  *             type: object
  */
-export default async function metadataRouter(collectionPath, services) {
+export default function metadataRouter(collection, services) {
   const router = express.Router();
-
-  const STATIC_METADATA = yaml.load(await fs.readFile(path.join(collectionPath, METADATA_FILENAME), 'utf8'));
-  const { version: engineVersion } = JSON.parse(await fs.readFile(new URL(PACKAGE_JSON_PATH, import.meta.url)));
+  const engineVersion = process.env.npm_package_version;
 
   /**
    * @swagger
@@ -201,7 +192,7 @@ export default async function metadataRouter(collectionPath, services) {
     };
 
     res.json({
-      ...STATIC_METADATA,
+      ...collection.metadata,
       ...dynamicMetadata,
     });
   });

package/src/collection-api/routes/metadata.test.js

@@ -1,5 +1,3 @@
-import fs from 'fs/promises';
-
 import { expect } from 'chai';
 import config from 'config';
 import request from 'supertest';
@@ -7,7 +5,7 @@ import request from 'supertest';
 import app from '../server.js';
 
 const basePath = config.get('@opentermsarchive/engine.collection-api.basePath');
-const { version: engineVersion } = JSON.parse(await fs.readFile(new URL('../../../package.json', import.meta.url)));
+const engineVersion = process.env.npm_package_version;
 
 const EXPECTED_RESPONSE = {
   totalServices: 7,

package/src/index.js

@@ -1,16 +1,13 @@
-import { createRequire } from 'module';
-
 import config from 'config';
 import cron from 'croner';
 import cronstrue from 'cronstrue';
 
+import { getCollection } from './archivist/collection/index.js';
 import Archivist from './archivist/index.js';
 import logger from './logger/index.js';
 import Notifier from './notifier/index.js';
 import Reporter from './reporter/index.js';
 
-const require = createRequire(import.meta.url);
-
 export default async function track({ services, types, extractOnly, schedule }) {
   const archivist = new Archivist({
     recorderConfig: config.get('@opentermsarchive/engine.recorder'),
@@ -21,9 +18,10 @@ export default async function track({ services, types, extractOnly, schedule })
 
   await archivist.initialize();
 
-  const { version } = require('../package.json');
+  const collection = await getCollection();
+  const collectionName = collection?.name ? ` with ${collection.name} collection` : '';
 
-  logger.info(`Start Open Terms Archive engine v${version}\n`);
+  logger.info(`Start engine v${process.env.npm_package_version}${collectionName}\n`);
 
   if (services?.length) {
     services = services.filter(serviceId => {

package/src/logger/index.js

@@ -4,10 +4,14 @@ import config from 'config';
 import winston from 'winston';
 import 'winston-mail';
 
+import { getCollection } from '../archivist/collection/index.js';
+
 import { formatDuration } from './utils.js';
 
 const { combine, timestamp, printf, colorize } = winston.format;
 
+const collection = await getCollection();
+
 const alignedWithColorsAndTime = combine(
   colorize(),
   timestamp({ format: 'YYYY-MM-DDTHH:mm:ssZ' }),
@@ -57,20 +61,97 @@ if (config.get('@opentermsarchive/engine.logger.sendMailOnError')) {
       password: process.env.OTA_ENGINE_SMTP_PASSWORD,
       ssl: true,
       timeout: 30 * 1000,
-      formatter: args => args[Object.getOwnPropertySymbols(args)[1]], // Returns the full error message, the same visible in the console. It is referenced in the argument object with a Symbol of which we do not have the reference but we know it is the second one.
+      html: false,
+      formatter({ message, level }) {
+        const isError = level.includes('error');
+        const titleColor = isError ? '#dc3545' : '#ffc107';
+        const titleText = isError ? 'Error details' : 'Warning details';
+
+        return `
+          <!DOCTYPE html>
+          <html lang="en">
+            <head>
+              <meta charset="utf-8">
+              <meta name="viewport" content="width=device-width, initial-scale=1.0">
+              <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+              <title>OTA Error Report</title>
+            </head>
+            <body style="font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Helvetica, Arial, sans-serif; line-height: 1.6; color: #333333; max-width: 800px; margin: 0 auto; padding: 0px 20px 20px 20px;">
+              <h1 style="color: #212529; font-size: 24px; margin: 10px 0; text-align: center; padding-bottom: 10px;">Open Terms Archive engine error report — ${collection.name} Collection</h1>
+              
+              <div style="background-color: #f8f9fa; border-radius: 8px; padding: 15px; margin-bottom: 0;">
+                <h2 style="color: ${titleColor}; margin: 0 0 0 0; font-size: 20px; border-bottom: 2px solid ${titleColor}; padding-bottom: 8px;">${titleText}</h2>
+                <div style="background-color: #ffffff; border: 1px solid #dee2e6; border-radius: 4px; padding: 12px; margin: 8px 0;">
+                  <code style="maring: 0; font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace; font-size: 14px; color: #212529; white-space: pre-wrap; display: block;">${message}</code>
+                </div>
+              </div>
+
+              <div style="background-color: #f8f9fa; border-radius: 8px; padding: 15px; margin-bottom: 0;">
+                <h2 style="color: #212529; margin: 0 0 0 0; font-size: 20px; border-bottom: 2px solid #212529; padding-bottom: 8px;">System information</h2>
+                <div style="color: #6c757d; font-size: 14px; margin: 0;">
+                  Hostname: ${os.hostname()}<br>
+                  Platform: ${os.platform()} ${os.release()}<br>
+                  Architecture: ${os.arch()}<br>
+                  CPU Cores: ${os.cpus().length}<br>
+                  CPU Load (1/5/15 min): ${os.loadavg().map(load =>
+    `${Math.min(100, (load / os.cpus().length) * 100).toFixed(1)}%`).join(' / ')}<br>
+                  Total Memory: ${(os.totalmem() / (1024 * 1024 * 1024)).toFixed(2)} GB<br>
+                  Free Memory: ${(os.freemem() / (1024 * 1024 * 1024)).toFixed(2)} GB${collection.host ? `<br>
+                  Server IP: ${collection.host}` : ''}
+                </div>
+              </div>
+
+              <div style="background-color: #f8f9fa; border-radius: 8px; padding: 15px; margin-bottom: 0;">
+                <h2 style="color: #198754; margin: 0 0 0 0; font-size: 20px; border-bottom: 2px solid #198754; padding-bottom: 8px;">Helpful commands</h2>
+                <ul style="list-style-type: none; padding-left: 0; margin: 0;">
+                  ${collection.host && collection.hostConfig?.ansible_user ? `
+                  <li style="margin: 0; padding: 0">
+                    <strong style="display: block; margin-bottom: 0;">Connect to the server:</strong>
+                    <code style="maring: 0; font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace; background-color: #ffffff; border: 1px solid #dee2e6; border-radius: 4px; padding: 6px 10px; display: inline-block; color: #212529; cursor: text; user-select: all; -webkit-user-select: all; -moz-user-select: all; -ms-user-select: all;">ssh ${collection.hostConfig.ansible_user}@${collection.host}</code>
+                  </li>` : ''}
+
+                  <li style="margin: 0; padding: 0">
+                    <strong style="display: block; margin-bottom: 0;">List processes on the server:</strong>
+                    <code style="maring: 0; font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace; background-color: #ffffff; border: 1px solid #dee2e6; border-radius: 4px; padding: 6px 10px; display: inline-block; color: #212529; cursor: text; user-select: all; -webkit-user-select: all; -moz-user-select: all; -ms-user-select: all;">pm2 list</code>
+                  </li>
+
+                  <li style="margin: 0; padding: 0">
+                    <strong style="display: block; margin-bottom: 0;">View the logs on the server:</strong>
+                    <code style="maring: 0; font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace; background-color: #ffffff; border: 1px solid #dee2e6; border-radius: 4px; padding: 6px 10px; display: inline-block; color: #212529; cursor: text; user-select: all; -webkit-user-select: all; -moz-user-select: all; -ms-user-select: all;">pm2 logs &lt;process-name&gt;</code>
+                  </li>
+
+                  <li style="margin: 0; padding: 0">
+                    <strong style="display: block; margin-bottom: 0;">View additional logging options:</strong>
+                    <code style="maring: 0; font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace; background-color: #ffffff; border: 1px solid #dee2e6; border-radius: 4px; padding: 6px 10px; display: inline-block; color: #212529; cursor: text; user-select: all; -webkit-user-select: all; -moz-user-select: all; -ms-user-select: all;">pm2 logs &lt;process-name&gt; --help</code>
+                  </li>
+
+                  <li style="margin: 0; padding: 0">
+                    <strong style="display: block; margin-bottom: 0;">View deployment documentation to see how to start, stop, and restart the server:</strong>
+                    <a href="https://github.com/OpenTermsArchive/deployment" style="color: #198754; text-decoration: none; border-bottom: 1px solid #198754;">github.com/OpenTermsArchive/deployment</a>
+                  </li>
+                </ul>
+              </div>
+
+              <div style="margin-top: 15px; padding-top: 15px; border-top: 1px solid #dee2e6; font-size: 12px; color: #6c757d; text-align: center;">
+                This is an automated message from the Open Terms Archive engine. Please do not reply to this email.
+              </div>
+            </body>
+          </html>
+        `;
+      },
     };
 
     transports.push(new winston.transports.Mail({
       ...mailerOptions,
       level: 'error',
-      subject: `[OTA] Error Report — ${os.hostname()}`,
+      subject: `Server error on ${collection.id} collection`,
     }));
 
     if (config.get('@opentermsarchive/engine.logger.sendMailOnError.sendWarnings')) {
       transports.push(new winston.transports.Mail({
         ...mailerOptions,
         level: 'warn',
-        subject: `[OTA] Inaccessible content — ${os.hostname()}`,
+        subject: `Inaccessible content on ${collection.id} collection`,
       }));
     }
   }