@opentermsarchive/engine 15.2.0 → 15.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "15.2.0",
+  "version": "15.3.0",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {

package/src/collection-api/logger.js

@@ -1,34 +1,13 @@
-import os from 'os';
-
 import config from 'config';
 import dotenv from 'dotenv';
 import winston from 'winston';
 
-import MailTransportWithRetry from '../logger/mail-transport-with-retry.js';
+import { addErrorMail } from '../logger/error-mail.js';
 
 dotenv.config({ quiet: true });
 
 const { combine, timestamp, printf, colorize } = winston.format;
 
-const transports = [new winston.transports.Console()];
-
-if (config.get('@opentermsarchive/engine.logger.sendMailOnError')) {
-  transports.push(new MailTransportWithRetry({
-    to: config.get('@opentermsarchive/engine.logger.sendMailOnError.to'),
-    from: config.get('@opentermsarchive/engine.logger.sendMailOnError.from'),
-    host: config.get('@opentermsarchive/engine.logger.smtp.host'),
-    port: config.get('@opentermsarchive/engine.logger.smtp.port'),
-    username: config.get('@opentermsarchive/engine.logger.smtp.username'),
-    password: process.env.OTA_ENGINE_SMTP_PASSWORD,
-    tls: true,
-    timeout: 60 * 1000,
-    formatter: args => args[Object.getOwnPropertySymbols(args)[1]], // Returns the full error message, the same visible in the console. It is referenced in the argument object with a Symbol of which we do not have the reference but we know it is the second one.
-    exitOnError: true,
-    level: 'error',
-    subject: `[OTA API] Error Report — ${os.hostname()}`,
-  }));
-}
-
 const logger = winston.createLogger({
   format: combine(
     colorize(),
@@ -39,8 +18,10 @@ const logger = winston.createLogger({
       return `${timestampPrefix}${level.padEnd(15)} ${message}`;
     }),
   ),
-  transports,
-  rejectionHandlers: transports,
+  transports: [new winston.transports.Console({ handleRejections: true })],
+  exitOnError: false,
 });
 
+await addErrorMail(logger, { component: 'Collection API', subject: 'API error' });
+
 export default logger;

package/src/logger/error-mail.js

@@ -0,0 +1,149 @@
+import os from 'os';
+
+import config from 'config';
+import winston from 'winston';
+
+import { getCollection } from '../archivist/collection/index.js';
+
+import MailTransportWithRetry from './mail-transport-with-retry.js';
+
+const SMTP_TIMEOUT = 60 * 1000;
+
+export const SENDING_BUDGET = SMTP_TIMEOUT; // The duration of one attempt: retries would only delay the restart when the relay is down
+
+const escapeHtml = text => String(text).replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+
+const CODE_STYLE = "font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace; background-color: #ffffff; border: 1px solid #dee2e6; border-radius: 4px; padding: 6px 10px; display: inline-block; color: #212529; cursor: text; user-select: all; -webkit-user-select: all; -moz-user-select: all; -ms-user-select: all;";
+
+const section = (title, color, content) => `
+              <div style="background-color: #f8f9fa; border-radius: 8px; padding: 15px; margin-bottom: 0;">
+                <h2 style="color: ${color}; margin: 0 0 0 0; font-size: 20px; border-bottom: 2px solid ${color}; padding-bottom: 8px;">${title}</h2>${content}
+              </div>`;
+
+const command = (label, code) => `
+                  <li style="margin: 0; padding: 0">
+                    <strong style="display: block; margin-bottom: 0;">${label}</strong>
+                    <code style="${CODE_STYLE}">${escapeHtml(code)}</code>
+                  </li>`;
+
+function formatBody({ collection, component, environmentPrefix }, { message, level }) {
+  const isError = level.includes('error');
+  const titleColor = isError ? '#dc3545' : '#ffc107';
+  const titleText = isError ? 'Error details' : 'Warning details';
+  const cpuCount = os.cpus().length;
+
+  const commands = [
+    ...(collection.host && collection.hostConfig?.ansible_user ? [[ 'Connect to the server:', `ssh ${collection.hostConfig.ansible_user}@${collection.host}` ]] : []),
+    [ 'List processes on the server:', 'pm2 list' ],
+    [ 'View the logs on the server:', 'pm2 logs <process-name>' ],
+    [ 'View additional logging options:', 'pm2 logs <process-name> --help' ],
+  ];
+
+  return `
+          <!DOCTYPE html>
+          <html lang="en">
+            <head>
+              <meta charset="utf-8">
+              <meta name="viewport" content="width=device-width, initial-scale=1.0">
+              <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+              <title>OTA Error Report</title>
+            </head>
+            <body style="font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Helvetica, Arial, sans-serif; line-height: 1.6; color: #333333; max-width: 800px; margin: 0 auto; padding: 0px 20px 20px 20px;">
+              <h1 style="color: #212529; font-size: 24px; margin: 10px 0; text-align: center; padding-bottom: 10px;">${escapeHtml(`${environmentPrefix}Open Terms Archive ${component} error report — ${collection.name} Collection`)}</h1>
+              ${section(titleText, titleColor, `
+                <div style="background-color: #ffffff; border: 1px solid #dee2e6; border-radius: 4px; padding: 12px; margin: 8px 0;">
+                  <code style="margin: 0; font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace; font-size: 14px; color: #212529; white-space: pre-wrap; display: block;">${escapeHtml(message)}</code>
+                </div>`)}
+              ${section('System information', '#212529', `
+                <div style="color: #6c757d; font-size: 14px; margin: 0;">
+                  Hostname: ${escapeHtml(os.hostname())}<br>
+                  Platform: ${escapeHtml(`${os.platform()} ${os.release()}`)}<br>
+                  Architecture: ${os.arch()}<br>
+                  CPU Cores: ${cpuCount}<br>
+                  CPU Load (1/5/15 min): ${os.loadavg().map(load => `${Math.min(100, (load / cpuCount) * 100).toFixed(1)}%`).join(' / ')}<br>
+                  Total Memory: ${(os.totalmem() / (1024 * 1024 * 1024)).toFixed(2)} GB<br>
+                  Free Memory: ${(os.freemem() / (1024 * 1024 * 1024)).toFixed(2)} GB${collection.host ? `<br>
+                  Server IP: ${escapeHtml(collection.host)}` : ''}
+                </div>`)}
+              ${section('Helpful commands', '#198754', `
+                <ul style="list-style-type: none; padding-left: 0; margin: 0;">${commands.map(([ label, code ]) => command(label, code)).join('')}
+                  <li style="margin: 0; padding: 0">
+                    <strong style="display: block; margin-bottom: 0;">View deployment documentation to see how to start, stop, and restart the server:</strong>
+                    <a href="https://github.com/OpenTermsArchive/deployment" style="color: #198754; text-decoration: none; border-bottom: 1px solid #198754;">github.com/OpenTermsArchive/deployment</a>
+                  </li>
+                </ul>`)}
+              <div style="margin-top: 15px; padding-top: 15px; border-top: 1px solid #dee2e6; font-size: 12px; color: #6c757d; text-align: center;">
+                This is an automated message from the Open Terms Archive engine. Please do not reply to this email.
+              </div>
+            </body>
+          </html>
+        `;
+}
+
+export function createErrorMailTransports(logger, { collection, component, subject, warningSubject }) {
+  if (!config.get('@opentermsarchive/engine.logger.sendMailOnError')) {
+    return [];
+  }
+
+  if (process.env.OTA_ENGINE_SMTP_PASSWORD === undefined) {
+    logger.warn('Environment variable "OTA_ENGINE_SMTP_PASSWORD" was not found; log emails cannot be sent');
+
+    return [];
+  }
+
+  const environment = process.env.NODE_ENV || 'development'; // Same default as node-config, so the prefix matches the loaded configuration
+  const environmentPrefix = environment == 'production' ? '' : `[${environment}] `; // Make emails sent from a developer machine recognisable at a glance
+
+  const mailerOptions = {
+    to: config.get('@opentermsarchive/engine.logger.sendMailOnError.to'),
+    from: config.get('@opentermsarchive/engine.logger.sendMailOnError.from'),
+    host: config.get('@opentermsarchive/engine.logger.smtp.host'),
+    port: config.get('@opentermsarchive/engine.logger.smtp.port'),
+    username: config.get('@opentermsarchive/engine.logger.smtp.username'),
+    password: process.env.OTA_ENGINE_SMTP_PASSWORD,
+    tls: true,
+    timeout: SMTP_TIMEOUT,
+    html: true,
+    formatter: info => formatBody({ collection, component, environmentPrefix }, info),
+    handleRejections: true,
+  };
+
+  const transports = [new MailTransportWithRetry({ ...mailerOptions, level: 'error', subject: `${environmentPrefix}${subject}` })];
+
+  if (warningSubject && config.has('@opentermsarchive/engine.logger.sendMailOnError.sendWarnings') && config.get('@opentermsarchive/engine.logger.sendMailOnError.sendWarnings')) { // Only callers providing a subject for warnings can send them
+    transports.push(new MailTransportWithRetry({
+      ...mailerOptions,
+      level: 'warn',
+      subject: `${environmentPrefix}${warningSubject}`,
+      format: winston.format(info => (info[Symbol.for('level')] === 'warn' ? info : false))(), // Winston transports receive every level at or above theirs, so errors would otherwise be emailed a second time as warnings
+    }));
+  }
+
+  return transports;
+}
+
+export function exitOnUnhandledRejection(transports, { emitter = process } = {}) {
+  const mailTransports = transports.filter(transport => transport instanceof MailTransportWithRetry);
+
+  emitter.on('unhandledRejection', async () => {
+    await new Promise(resolve => { setImmediate(resolve); }); // Winston's own listener, registered before this one, logs the rejection on the current tick; give the transports a chance to receive it before waiting for them
+    await Promise.race([
+      Promise.all(mailTransports.map(transport => transport.flush())),
+      new Promise(resolve => { setTimeout(resolve, SENDING_BUDGET).unref(); }),
+    ]);
+    process.exit(1);
+  });
+}
+
+export async function addErrorMail(logger, { component, subject, warningSubject }, { emitter = process } = {}) {
+  const collection = await getCollection();
+  const transports = createErrorMailTransports(logger, {
+    collection,
+    component,
+    subject: `${subject} on ${collection.id} collection`,
+    warningSubject: warningSubject && `${warningSubject} on ${collection.id} collection`,
+  });
+
+  transports.forEach(transport => logger.add(transport));
+  exitOnUnhandledRejection(logger.transports, { emitter });
+}

package/src/logger/error-mail.test.js

@@ -0,0 +1,458 @@
+import { EventEmitter } from 'node:events';
+import os from 'node:os';
+
+import { expect, use } from 'chai';
+import config from 'config';
+import sinon from 'sinon';
+import sinonChai from 'sinon-chai';
+import winston from 'winston';
+
+import { SENDING_BUDGET, addErrorMail, createErrorMailTransports, exitOnUnhandledRejection } from './error-mail.js';
+import MailTransportWithRetry from './mail-transport-with-retry.js';
+
+use(sinonChai);
+
+const SEND_MAIL_ON_ERROR = {
+  to: 'admin@example.com',
+  from: 'noreply@example.com',
+  sendWarnings: false,
+};
+
+describe('Error mail', () => {
+  let configValues;
+  let originalPassword;
+  let originalEnvironment;
+
+  before(() => {
+    originalPassword = process.env.OTA_ENGINE_SMTP_PASSWORD;
+    originalEnvironment = process.env.NODE_ENV;
+  });
+
+  after(() => {
+    if (originalPassword === undefined) {
+      delete process.env.OTA_ENGINE_SMTP_PASSWORD;
+    } else {
+      process.env.OTA_ENGINE_SMTP_PASSWORD = originalPassword;
+    }
+    process.env.NODE_ENV = originalEnvironment;
+  });
+
+  beforeEach(() => {
+    configValues = {
+      '@opentermsarchive/engine.logger.sendMailOnError': SEND_MAIL_ON_ERROR,
+      '@opentermsarchive/engine.logger.sendMailOnError.to': SEND_MAIL_ON_ERROR.to,
+      '@opentermsarchive/engine.logger.sendMailOnError.from': SEND_MAIL_ON_ERROR.from,
+      '@opentermsarchive/engine.logger.sendMailOnError.sendWarnings': SEND_MAIL_ON_ERROR.sendWarnings,
+      '@opentermsarchive/engine.logger.smtp.host': 'smtp.example.com',
+      '@opentermsarchive/engine.logger.smtp.port': 587,
+      '@opentermsarchive/engine.logger.smtp.username': 'user',
+      '@opentermsarchive/engine.collectionPath': './test/test-declarations',
+    };
+    sinon.stub(config, 'get').callsFake(key => configValues[key]);
+    sinon.stub(config, 'has').callsFake(key => configValues[key] !== undefined);
+    process.env.NODE_ENV = 'production';
+    process.env.OTA_ENGINE_SMTP_PASSWORD = 'secret';
+  });
+
+  afterEach(() => {
+    sinon.restore();
+  });
+
+  describe('#createErrorMailTransports', () => {
+    const logger = { warn: () => {} };
+    const collection = {
+      id: 'test',
+      name: 'Test',
+      host: '203.0.113.1',
+      hostConfig: { ansible_user: 'ota' },
+    };
+    const component = 'test component';
+    const subject = 'Error on test collection';
+    const warningSubject = 'Warning on test collection';
+
+    context('when sending mail on error is disabled', () => {
+      beforeEach(() => {
+        configValues['@opentermsarchive/engine.logger.sendMailOnError'] = false;
+      });
+
+      it('returns no transport', () => {
+        expect(createErrorMailTransports(logger, { collection, component, subject, warningSubject })).to.be.empty;
+      });
+
+      it('does not warn', () => {
+        const warnSpy = sinon.spy(logger, 'warn');
+
+        createErrorMailTransports(logger, { collection, component, subject, warningSubject });
+
+        expect(warnSpy).to.not.have.been.called;
+      });
+    });
+
+    context('when the SMTP password is not defined', () => {
+      beforeEach(() => {
+        delete process.env.OTA_ENGINE_SMTP_PASSWORD;
+      });
+
+      it('returns no transport', () => {
+        expect(createErrorMailTransports(logger, { collection, component, subject, warningSubject })).to.be.empty;
+      });
+
+      it('warns through the logger that emails cannot be sent', () => {
+        const warnSpy = sinon.spy(logger, 'warn');
+
+        createErrorMailTransports(logger, { collection, component, subject, warningSubject });
+
+        expect(warnSpy).to.have.been.calledOnce;
+        expect(warnSpy.firstCall.args[0]).to.include('OTA_ENGINE_SMTP_PASSWORD');
+      });
+    });
+
+    context('when sending mail on error is enabled', () => {
+      let transports;
+
+      context('without warnings', () => {
+        beforeEach(() => {
+          transports = createErrorMailTransports(logger, { collection, component, subject, warningSubject });
+        });
+
+        it('returns a single transport', () => {
+          expect(transports).to.have.lengthOf(1);
+        });
+
+        it('returns a mail transport with retry', () => {
+          expect(transports[0]).to.be.an.instanceOf(MailTransportWithRetry);
+        });
+
+        it('sends errors only', () => {
+          expect(transports[0].level).to.equal('error');
+        });
+
+        it('uses the given subject', () => {
+          expect(transports[0].mailTransport.subject).to.equal(subject);
+        });
+
+        it('uses the configured recipient and sender', () => {
+          expect(transports[0].mailTransport.to).to.equal(SEND_MAIL_ON_ERROR.to);
+          expect(transports[0].mailTransport.from).to.equal(SEND_MAIL_ON_ERROR.from);
+        });
+
+        it('sends HTML emails', () => {
+          expect(transports[0].mailTransport.html).to.be.true;
+        });
+
+        it('handles unhandled rejections', () => {
+          expect(transports[0].handleRejections).to.be.true;
+        });
+      });
+
+      context('with warnings enabled and a subject for them', () => {
+        beforeEach(() => {
+          configValues['@opentermsarchive/engine.logger.sendMailOnError.sendWarnings'] = true;
+          transports = createErrorMailTransports(logger, { collection, component, subject, warningSubject });
+        });
+
+        it('returns two transports', () => {
+          expect(transports).to.have.lengthOf(2);
+        });
+
+        it('sends warnings with the second transport', () => {
+          expect(transports[1].level).to.equal('warn');
+        });
+
+        it('uses the warning subject for the second transport', () => {
+          expect(transports[1].mailTransport.subject).to.equal(warningSubject);
+        });
+
+        context('when logging through both transports', () => {
+          let sent;
+
+          beforeEach(() => {
+            sent = [];
+            transports.forEach((transport, index) => {
+              sinon.stub(transport.mailTransport, 'log').callsFake((info, callback) => {
+                sent.push(index);
+                setImmediate(() => {
+                  transport.mailTransport.emit('logged');
+                  callback();
+                });
+              });
+            });
+          });
+
+          it('sends errors through the error transport only', async () => {
+            const logger = winston.createLogger({ format: winston.format.colorize(), transports });
+
+            logger.error('boom');
+            await new Promise(resolve => { setTimeout(resolve, 10); });
+
+            expect(sent).to.deep.equal([0]);
+          });
+
+          it('sends warnings through the warning transport only', async () => {
+            const logger = winston.createLogger({ format: winston.format.colorize(), transports });
+
+            logger.warn('inaccessible');
+            await new Promise(resolve => { setTimeout(resolve, 10); });
+
+            expect(sent).to.deep.equal([1]);
+          });
+        });
+      });
+
+      context('with warnings enabled but no subject for them', () => {
+        beforeEach(() => {
+          configValues['@opentermsarchive/engine.logger.sendMailOnError.sendWarnings'] = true;
+          transports = createErrorMailTransports(logger, { collection, component, subject });
+        });
+
+        it('returns the error transport only', () => {
+          expect(transports).to.have.lengthOf(1);
+          expect(transports[0].level).to.equal('error');
+        });
+      });
+
+      context('with a subject for warnings but warnings not configured', () => {
+        beforeEach(() => {
+          delete configValues['@opentermsarchive/engine.logger.sendMailOnError.sendWarnings'];
+          transports = createErrorMailTransports(logger, { collection, component, subject, warningSubject });
+        });
+
+        it('returns the error transport only', () => {
+          expect(transports).to.have.lengthOf(1);
+          expect(transports[0].level).to.equal('error');
+        });
+      });
+
+      context('with a subject for warnings but warnings disabled', () => {
+        beforeEach(() => {
+          transports = createErrorMailTransports(logger, { collection, component, subject, warningSubject });
+        });
+
+        it('returns the error transport only', () => {
+          expect(transports).to.have.lengthOf(1);
+          expect(transports[0].level).to.equal('error');
+        });
+      });
+
+      context('email body', () => {
+        let formatter;
+        let body;
+
+        beforeEach(() => {
+          [{ mailTransport: { formatter } }] = createErrorMailTransports(logger, { collection, component, subject, warningSubject });
+        });
+
+        context('for an error', () => {
+          beforeEach(() => {
+            body = formatter({ message: 'Error: <boom> & co', level: 'error' });
+          });
+
+          it('is titled as an error', () => {
+            expect(body).to.include('Error details');
+          });
+
+          it('includes the escaped message', () => {
+            expect(body).to.include('Error: &lt;boom&gt; &amp; co');
+            expect(body).to.not.include('<boom>');
+          });
+
+          it('names the collection', () => {
+            expect(body).to.include(`${collection.name} Collection`);
+          });
+
+          it('names the component', () => {
+            expect(body).to.include(`Open Terms Archive ${component} error report`);
+          });
+
+          it('includes the hostname', () => {
+            expect(body).to.include(os.hostname());
+          });
+
+          it('includes the command to connect to the server', () => {
+            expect(body).to.include(`ssh ${collection.hostConfig.ansible_user}@${collection.host}`);
+          });
+        });
+
+        context('for a warning', () => {
+          beforeEach(() => {
+            body = formatter({ message: 'Inaccessible content', level: 'warn' });
+          });
+
+          it('is titled as a warning', () => {
+            expect(body).to.include('Warning details');
+          });
+        });
+
+        context('when interpolated values contain HTML characters', () => {
+          beforeEach(() => {
+            [{ mailTransport: { formatter } }] = createErrorMailTransports(logger, { collection: { id: 'test', name: 'R&D <beta>', host: '203.0.113.1', hostConfig: { ansible_user: 'ota<x>' } }, component: 'API <v2>', subject, warningSubject });
+            body = formatter({ message: 'Error', level: 'error' });
+          });
+
+          it('escapes the collection name', () => {
+            expect(body).to.include('R&amp;D &lt;beta&gt; Collection');
+            expect(body).to.not.include('<beta>');
+          });
+
+          it('escapes the component', () => {
+            expect(body).to.include('Open Terms Archive API &lt;v2&gt; error report');
+          });
+
+          it('escapes the command to connect to the server', () => {
+            expect(body).to.include('ssh ota&lt;x&gt;@203.0.113.1');
+            expect(body).to.not.include('ota<x>');
+          });
+
+          it('escapes the commands placeholders', () => {
+            expect(body).to.include('pm2 logs &lt;process-name&gt;');
+          });
+        });
+
+        context('when the collection has no deployment inventory', () => {
+          beforeEach(() => {
+            [{ mailTransport: { formatter } }] = createErrorMailTransports(logger, { collection: { id: 'test', name: 'Test' }, component, subject, warningSubject });
+            body = formatter({ message: 'Error', level: 'error' });
+          });
+
+          it('omits the command to connect to the server', () => {
+            expect(body).to.not.include('ssh ');
+          });
+        });
+      });
+
+      context('outside production', () => {
+        beforeEach(() => {
+          process.env.NODE_ENV = 'staging';
+          configValues['@opentermsarchive/engine.logger.sendMailOnError.sendWarnings'] = true;
+          transports = createErrorMailTransports(logger, { collection, component, subject, warningSubject });
+        });
+
+        it('prefixes the error subject with the environment', () => {
+          expect(transports[0].mailTransport.subject).to.equal(`[staging] ${subject}`);
+        });
+
+        it('prefixes the warning subject with the environment', () => {
+          expect(transports[1].mailTransport.subject).to.equal(`[staging] ${warningSubject}`);
+        });
+
+        it('prefixes the title of the body with the environment', () => {
+          expect(transports[0].mailTransport.formatter({ message: 'Error', level: 'error' })).to.include(`[staging] Open Terms Archive ${component} error report`);
+        });
+      });
+
+      context('when the environment is not defined', () => {
+        beforeEach(() => {
+          delete process.env.NODE_ENV;
+          transports = createErrorMailTransports(logger, { collection, component, subject, warningSubject });
+        });
+
+        it('prefixes the subject with the development environment', () => {
+          expect(transports[0].mailTransport.subject).to.equal(`[development] ${subject}`);
+        });
+      });
+    });
+  });
+
+  describe('#addErrorMail', () => {
+    let logger;
+    let warnSpy;
+
+    beforeEach(() => {
+      logger = winston.createLogger({ transports: [new winston.transports.Console({ silent: true })], exitOnError: false });
+      warnSpy = sinon.spy(logger, 'warn');
+    });
+
+    context('when sending mail on error is enabled with warnings', () => {
+      beforeEach(async () => {
+        configValues['@opentermsarchive/engine.logger.sendMailOnError.sendWarnings'] = true;
+        await addErrorMail(logger, { component: 'engine', subject: 'Server error', warningSubject: 'Inaccessible content' }, { emitter: new EventEmitter() });
+      });
+
+      it('adds the mail transports to the logger', () => {
+        expect(logger.transports).to.have.lengthOf(3);
+        expect(logger.transports[1]).to.be.an.instanceOf(MailTransportWithRetry);
+        expect(logger.transports[2]).to.be.an.instanceOf(MailTransportWithRetry);
+      });
+
+      it('names the collection in the subjects', () => {
+        expect(logger.transports[1].mailTransport.subject).to.equal('Server error on test collection');
+        expect(logger.transports[2].mailTransport.subject).to.equal('Inaccessible content on test collection');
+      });
+    });
+
+    context('when the SMTP password is not defined', () => {
+      beforeEach(async () => {
+        delete process.env.OTA_ENGINE_SMTP_PASSWORD;
+        await addErrorMail(logger, { component: 'engine', subject: 'Server error' }, { emitter: new EventEmitter() });
+      });
+
+      it('warns through the logger', () => {
+        expect(warnSpy).to.have.been.calledOnce;
+        expect(warnSpy.firstCall.args[0]).to.include('OTA_ENGINE_SMTP_PASSWORD');
+      });
+
+      it('adds no transport', () => {
+        expect(logger.transports).to.have.lengthOf(1);
+      });
+    });
+  });
+
+  describe('#exitOnUnhandledRejection', () => {
+    let emitter;
+    let processExitStub;
+    let mailTransport;
+    let sent;
+
+    const tick = () => new Promise(resolve => { setImmediate(resolve); });
+
+    beforeEach(() => {
+      emitter = new EventEmitter();
+      processExitStub = sinon.stub(process, 'exit');
+      mailTransport = Object.create(MailTransportWithRetry.prototype);
+      mailTransport.flush = () => new Promise(resolve => { sent = resolve; });
+      exitOnUnhandledRejection([ new winston.transports.Console({ silent: true }), mailTransport ], { emitter });
+    });
+
+    context('when emails are sent', () => {
+      beforeEach(() => {
+        emitter.emit('unhandledRejection', new Error('boom'));
+      });
+
+      it('waits for pending emails before exiting', async () => {
+        await tick();
+        await tick();
+
+        expect(processExitStub).to.not.have.been.called;
+      });
+
+      it('exits with a failure code once emails are sent', async () => {
+        await tick();
+        sent();
+        await tick();
+
+        expect(processExitStub).to.have.been.calledOnceWithExactly(1);
+      });
+    });
+
+    context('when emails cannot be sent', () => {
+      let clock;
+
+      beforeEach(() => {
+        clock = sinon.useFakeTimers();
+        emitter.emit('unhandledRejection', new Error('boom'));
+      });
+
+      afterEach(() => {
+        clock.restore();
+      });
+
+      it('exits with a failure code after one sending attempt', async () => {
+        await clock.tickAsync(SENDING_BUDGET - 1);
+        expect(processExitStub).to.not.have.been.called;
+
+        await clock.tickAsync(1);
+        expect(processExitStub).to.have.been.calledOnceWithExactly(1);
+      });
+    });
+  });
+});

package/src/logger/index.js

@@ -1,17 +1,11 @@
-import os from 'os';
-
 import config from 'config';
 import winston from 'winston';
 
-import { getCollection } from '../archivist/collection/index.js';
-
-import MailTransportWithRetry from './mail-transport-with-retry.js';
+import { addErrorMail } from './error-mail.js';
 import { formatDuration } from './utils.js';
 
 const { combine, timestamp, printf, colorize } = winston.format;
 
-const collection = await getCollection();
-
 const alignedWithColorsAndTime = combine(
   colorize(),
   timestamp({ format: 'YYYY-MM-DDTHH:mm:ssZ' }),
@@ -28,141 +22,15 @@ const alignedWithColorsAndTime = combine(
   }),
 );
 
-const consoleTransport = new winston.transports.Console({ silent: process.env.NODE_ENV === 'test' });
-
-const transports = [consoleTransport];
+const consoleTransport = new winston.transports.Console({ silent: process.env.NODE_ENV === 'test', handleRejections: true });
 
 const logger = winston.createLogger({
   format: alignedWithColorsAndTime,
-  transports,
-  rejectionHandlers: transports,
-  exitOnError: true,
+  transports: [consoleTransport],
+  exitOnError: false,
 });
 
-logger.on('error', err => {
-  if ('smtp' in err) { // Check if err has an `smtp` property, even if it's undefined
-    logger.warn({ message: `Uncaught exception from SMTP mailer detected and treated as an operational error; process will continue running:\n${err.stack}` });
-
-    return; // Prevent process exit
-  }
-
-  return process.exit(1); // Exit process for other errors
-});
-
-if (config.get('@opentermsarchive/engine.logger.sendMailOnError')) {
-  if (process.env.OTA_ENGINE_SMTP_PASSWORD === undefined) {
-    logger.warn('Environment variable "OTA_ENGINE_SMTP_PASSWORD" was not found; log emails cannot be sent');
-  } else {
-    const mailerOptions = {
-      to: config.get('@opentermsarchive/engine.logger.sendMailOnError.to'),
-      from: config.get('@opentermsarchive/engine.logger.sendMailOnError.from'),
-      host: config.get('@opentermsarchive/engine.logger.smtp.host'),
-      port: config.get('@opentermsarchive/engine.logger.smtp.port'),
-      username: config.get('@opentermsarchive/engine.logger.smtp.username'),
-      password: process.env.OTA_ENGINE_SMTP_PASSWORD,
-      tls: true,
-      timeout: 60 * 1000,
-      html: false,
-      formatter({ message, level }) {
-        const isError = level.includes('error');
-        const titleColor = isError ? '#dc3545' : '#ffc107';
-        const titleText = isError ? 'Error details' : 'Warning details';
-
-        return `
-          <!DOCTYPE html>
-          <html lang="en">
-            <head>
-              <meta charset="utf-8">
-              <meta name="viewport" content="width=device-width, initial-scale=1.0">
-              <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-              <title>OTA Error Report</title>
-            </head>
-            <body style="font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Helvetica, Arial, sans-serif; line-height: 1.6; color: #333333; max-width: 800px; margin: 0 auto; padding: 0px 20px 20px 20px;">
-              <h1 style="color: #212529; font-size: 24px; margin: 10px 0; text-align: center; padding-bottom: 10px;">Open Terms Archive engine error report — ${collection.name} Collection</h1>
-              
-              <div style="background-color: #f8f9fa; border-radius: 8px; padding: 15px; margin-bottom: 0;">
-                <h2 style="color: ${titleColor}; margin: 0 0 0 0; font-size: 20px; border-bottom: 2px solid ${titleColor}; padding-bottom: 8px;">${titleText}</h2>
-                <div style="background-color: #ffffff; border: 1px solid #dee2e6; border-radius: 4px; padding: 12px; margin: 8px 0;">
-                  <code style="maring: 0; font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace; font-size: 14px; color: #212529; white-space: pre-wrap; display: block;">${message}</code>
-                </div>
-              </div>
-
-              <div style="background-color: #f8f9fa; border-radius: 8px; padding: 15px; margin-bottom: 0;">
-                <h2 style="color: #212529; margin: 0 0 0 0; font-size: 20px; border-bottom: 2px solid #212529; padding-bottom: 8px;">System information</h2>
-                <div style="color: #6c757d; font-size: 14px; margin: 0;">
-                  Hostname: ${os.hostname()}<br>
-                  Platform: ${os.platform()} ${os.release()}<br>
-                  Architecture: ${os.arch()}<br>
-                  CPU Cores: ${os.cpus().length}<br>
-                  CPU Load (1/5/15 min): ${os.loadavg().map(load =>
-    `${Math.min(100, (load / os.cpus().length) * 100).toFixed(1)}%`).join(' / ')}<br>
-                  Total Memory: ${(os.totalmem() / (1024 * 1024 * 1024)).toFixed(2)} GB<br>
-                  Free Memory: ${(os.freemem() / (1024 * 1024 * 1024)).toFixed(2)} GB${collection.host ? `<br>
-                  Server IP: ${collection.host}` : ''}
-                </div>
-              </div>
-
-              <div style="background-color: #f8f9fa; border-radius: 8px; padding: 15px; margin-bottom: 0;">
-                <h2 style="color: #198754; margin: 0 0 0 0; font-size: 20px; border-bottom: 2px solid #198754; padding-bottom: 8px;">Helpful commands</h2>
-                <ul style="list-style-type: none; padding-left: 0; margin: 0;">
-                  ${collection.host && collection.hostConfig?.ansible_user ? `
-                  <li style="margin: 0; padding: 0">
-                    <strong style="display: block; margin-bottom: 0;">Connect to the server:</strong>
-                    <code style="maring: 0; font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace; background-color: #ffffff; border: 1px solid #dee2e6; border-radius: 4px; padding: 6px 10px; display: inline-block; color: #212529; cursor: text; user-select: all; -webkit-user-select: all; -moz-user-select: all; -ms-user-select: all;">ssh ${collection.hostConfig.ansible_user}@${collection.host}</code>
-                  </li>` : ''}
-
-                  <li style="margin: 0; padding: 0">
-                    <strong style="display: block; margin-bottom: 0;">List processes on the server:</strong>
-                    <code style="maring: 0; font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace; background-color: #ffffff; border: 1px solid #dee2e6; border-radius: 4px; padding: 6px 10px; display: inline-block; color: #212529; cursor: text; user-select: all; -webkit-user-select: all; -moz-user-select: all; -ms-user-select: all;">pm2 list</code>
-                  </li>
-
-                  <li style="margin: 0; padding: 0">
-                    <strong style="display: block; margin-bottom: 0;">View the logs on the server:</strong>
-                    <code style="maring: 0; font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace; background-color: #ffffff; border: 1px solid #dee2e6; border-radius: 4px; padding: 6px 10px; display: inline-block; color: #212529; cursor: text; user-select: all; -webkit-user-select: all; -moz-user-select: all; -ms-user-select: all;">pm2 logs &lt;process-name&gt;</code>
-                  </li>
-
-                  <li style="margin: 0; padding: 0">
-                    <strong style="display: block; margin-bottom: 0;">View additional logging options:</strong>
-                    <code style="maring: 0; font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace; background-color: #ffffff; border: 1px solid #dee2e6; border-radius: 4px; padding: 6px 10px; display: inline-block; color: #212529; cursor: text; user-select: all; -webkit-user-select: all; -moz-user-select: all; -ms-user-select: all;">pm2 logs &lt;process-name&gt; --help</code>
-                  </li>
-
-                  <li style="margin: 0; padding: 0">
-                    <strong style="display: block; margin-bottom: 0;">View deployment documentation to see how to start, stop, and restart the server:</strong>
-                    <a href="https://github.com/OpenTermsArchive/deployment" style="color: #198754; text-decoration: none; border-bottom: 1px solid #198754;">github.com/OpenTermsArchive/deployment</a>
-                  </li>
-                </ul>
-              </div>
-
-              <div style="margin-top: 15px; padding-top: 15px; border-top: 1px solid #dee2e6; font-size: 12px; color: #6c757d; text-align: center;">
-                This is an automated message from the Open Terms Archive engine. Please do not reply to this email.
-              </div>
-            </body>
-          </html>
-        `;
-      },
-    };
-
-    transports.push(new MailTransportWithRetry({
-      ...mailerOptions,
-      level: 'error',
-      subject: `Server error on ${collection.id} collection`,
-    }));
-
-    if (config.get('@opentermsarchive/engine.logger.sendMailOnError.sendWarnings')) {
-      transports.push(new MailTransportWithRetry({
-        ...mailerOptions,
-        level: 'warn',
-        subject: `Inaccessible content on ${collection.id} collection`,
-      }));
-    }
-  }
-}
-
-logger.configure({
-  transports,
-  rejectionHandlers: transports,
-  exitOnError: true,
-});
+await addErrorMail(logger, { component: 'engine', subject: 'Server error', warningSubject: 'Inaccessible content' });
 
 let recordedSnapshotsCount;
 let recordedVersionsCount;

package/src/logger/mail-transport-with-retry.js

@@ -1,5 +1,3 @@
-import { once } from 'node:events';
-
 import async from 'async';
 import winston from 'winston';
 
@@ -21,25 +19,52 @@ class MailTransportWithRetry extends winston.Transport {
   constructor(options) {
     super(options);
     this.mailTransport = new winston.transports.Mail(options);
+    this.pending = new Set();
   }
 
-  async log(info, callback) {
-    try {
-      await async.retry(RETRY_OPTIONS, async () => {
-        const result = Promise.race([
-          once(this.mailTransport, 'logged'),
-          once(this.mailTransport, 'error').then(([err]) => { throw err; }),
-        ]);
+  log(info, callback) {
+    callback(); // Winston delivers each log to every transport in sequence, so waiting for the SMTP retries here would hold back the other transports too
 
-        this.mailTransport.log(info, () => {});
+    const sending = this.send(info).finally(() => this.pending.delete(sending));
 
-        return result;
-      });
+    this.pending.add(sending);
+
+    return sending;
+  }
+
+  async send(info) {
+    try {
+      await async.retry(RETRY_OPTIONS, async () => { await this.attempt(info); }); // The task must be an async function for async.retry to await it rather than wait for a callback
     } catch (error) {
-      console.warn(`SMTP mail sending failed after ${RETRY_OPTIONS.times} attempts: ${error.message}`);
-      this.emit('error', error);
+      console.warn(`SMTP mail sending failed after ${RETRY_OPTIONS.times} attempts; giving up on this email:\n${error.stack}`);
     }
-    callback();
+  }
+
+  attempt(info) {
+    return new Promise((resolve, reject) => {
+      const removeListeners = () => {
+        this.mailTransport.off('logged', onLogged);
+        this.mailTransport.off('error', onError);
+      };
+
+      function onLogged() {
+        removeListeners();
+        resolve();
+      }
+
+      function onError(error) {
+        removeListeners();
+        reject(error);
+      }
+
+      this.mailTransport.once('logged', onLogged);
+      this.mailTransport.once('error', onError);
+      this.mailTransport.log(info, () => {});
+    });
+  }
+
+  async flush() {
+    await Promise.allSettled(this.pending);
   }
 }
 

package/src/logger/mail-transport-with-retry.test.js

@@ -29,6 +29,25 @@ describe('MailTransportWithRetry', () => {
     clock.restore();
   });
 
+  describe('#flush', () => {
+    it('resolves once pending emails are sent', async () => {
+      let flushed = false;
+
+      transport.log({ message: 'test' }, () => {});
+
+      const flushPromise = transport.flush().then(() => { flushed = true; });
+
+      await clock.tickAsync(0);
+      expect(flushed).to.be.false;
+
+      mockMailTransport.emit('logged');
+      await flushPromise;
+
+      expect(flushed).to.be.true;
+      expect(transport.pending).to.be.empty;
+    });
+  });
+
   describe('#log', () => {
     context('when email is sent successfully on first attempt', () => {
       it('calls callback without error', async () => {
@@ -62,6 +81,24 @@ describe('MailTransportWithRetry', () => {
 
         expect(consoleWarnStub).not.to.have.been.called;
       });
+
+      it('calls callback before the email is sent', () => {
+        const callback = sinon.spy();
+
+        transport.log({ message: 'test' }, callback);
+
+        expect(callback).to.have.been.calledOnce;
+      });
+
+      it('removes its listeners from the mail transport', async () => {
+        const logPromise = transport.log({ message: 'test' }, () => {});
+
+        mockMailTransport.emit('logged');
+        await logPromise;
+
+        expect(mockMailTransport.listenerCount('logged')).to.equal(0);
+        expect(mockMailTransport.listenerCount('error')).to.equal(0);
+      });
     });
 
     context('when email fails then succeeds on retry', () => {
@@ -98,7 +135,7 @@ describe('MailTransportWithRetry', () => {
     });
 
     context('when email fails after all retry attempts', () => {
-      it('emits error event after all retries are exhausted', async () => {
+      it('does not emit an error event after all retries are exhausted', async () => {
         const errorHandler = sinon.spy();
 
         transport.on('error', errorHandler);
@@ -117,15 +154,12 @@ describe('MailTransportWithRetry', () => {
         mockMailTransport.emit('error', new Error('SMTP timeout'));
         await logPromise;
 
-        expect(errorHandler).to.have.been.calledOnce;
-        expect(errorHandler.firstCall.args[0].message).to.equal('SMTP timeout');
+        expect(errorHandler).not.to.have.been.called;
       });
 
       it('calls callback even after failure', async () => {
         const callback = sinon.spy();
 
-        transport.on('error', () => {}); // Prevent unhandled error
-
         const logPromise = transport.log({ message: 'test' }, callback);
 
         mockMailTransport.emit('error', new Error('SMTP timeout'));
@@ -144,8 +178,6 @@ describe('MailTransportWithRetry', () => {
       });
 
       it('logs final failure warning', async () => {
-        transport.on('error', () => {}); // Prevent unhandled error
-
         const logPromise = transport.log({ message: 'test' }, () => {});
 
         mockMailTransport.emit('error', new Error('SMTP timeout'));
@@ -161,6 +193,26 @@ describe('MailTransportWithRetry', () => {
         await logPromise;
 
         expect(consoleWarnStub.lastCall.args[0]).to.include(`failed after ${RETRY_DELAYS.length + 1} attempts`);
+        expect(consoleWarnStub.lastCall.args[0]).to.include('Error: SMTP timeout');
+      });
+
+      it('removes its listeners from the mail transport', async () => {
+        const logPromise = transport.log({ message: 'test' }, () => {});
+
+        mockMailTransport.emit('error', new Error('SMTP timeout'));
+        await clock.tickAsync(RETRY_DELAYS[0]);
+
+        mockMailTransport.emit('error', new Error('SMTP timeout'));
+        await clock.tickAsync(RETRY_DELAYS[1]);
+
+        mockMailTransport.emit('error', new Error('SMTP timeout'));
+        await clock.tickAsync(RETRY_DELAYS[2]);
+
+        mockMailTransport.emit('error', new Error('SMTP timeout'));
+        await logPromise;
+
+        expect(mockMailTransport.listenerCount('logged')).to.equal(0);
+        expect(mockMailTransport.listenerCount('error')).to.equal(0);
       });
     });