npm - @opentermsarchive/engine - Versions diffs - 15.1.0 → 15.3.0 - Mend

@opentermsarchive/engine 15.1.0 → 15.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (11) hide show

package/package.json +1 -1
package/src/archivist/extract/exposedFilters.js +32 -1
package/src/archivist/extract/exposedFilters.test.js +20 -0
package/src/archivist/services/index.js +1 -1
package/src/archivist/services/index.test.js +14 -2
package/src/collection-api/logger.js +5 -24
package/src/logger/error-mail.js +149 -0
package/src/logger/error-mail.test.js +458 -0
package/src/logger/index.js +5 -137
package/src/logger/mail-transport-with-retry.js +40 -15
package/src/logger/mail-transport-with-retry.test.js +59 -7

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "15.1.0",
+  "version": "15.3.0",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {

package/src/archivist/extract/exposedFilters.js CHANGED Viewed

@@ -1,4 +1,35 @@
-export function removeQueryParams(webPageDOM, paramsToRemove = []) {
+const DEFAULT_TRACKING_QUERY_PARAMS = Object.freeze([
+  'utm_source', // Campaign parameters, see https://en.wikipedia.org/wiki/UTM_parameters
+  'utm_medium',
+  'utm_campaign',
+  'utm_term',
+  'utm_content',
+  'fbclid', // Click and user identifiers, see https://github.com/privacytests/privacytests/blob/dda473a462e5c37f9f2c8b2367fbc5f834796be4/live/results.js#L31-L65
+  'gclid',
+  'msclkid',
+  'mc_eid',
+  'dclid',
+  'oly_anon_id',
+  'oly_enc_id',
+  '_openstat',
+  'vero_conv',
+  'vero_id',
+  'wickedid',
+  'yclid',
+  '__s',
+  'rb_clickid',
+  's_cid',
+  'ml_subscriber',
+  'ml_subscriber_hash',
+  '_hsenc',
+  '__hssc',
+  '__hstc',
+  '__hsfp',
+  'hsCtaTracking',
+  'mkt_tok',
+]);
+export function removeQueryParams(webPageDOM, paramsToRemove = DEFAULT_TRACKING_QUERY_PARAMS) {
   const normalizedParams = Array.isArray(paramsToRemove) ? paramsToRemove : [paramsToRemove];
   if (!normalizedParams.length) {

package/src/archivist/extract/exposedFilters.test.js CHANGED Viewed

@@ -51,6 +51,26 @@ describe('exposedFilters', () => {
       });
     });
+    describe('without parameters', () => {
+      let link;
+      before(() => {
+        link = webPageDOM.createElement('a');
+        link.setAttribute('href', 'https://example.com/page?utm_source=newsletter&utm_campaign=spring&fbclid=abc&gclid=def&mkt_tok=ghi&keep=value');
+        webPageDOM.body.appendChild(link);
+      });
+      after(() => {
+        link.remove();
+      });
+      it('removes well-known tracking parameters', () => {
+        removeQueryParams(webPageDOM);
+        expect(link.getAttribute('href')).to.equal('https://example.com/page?keep=value');
+      });
+    });
     describe('with string parameter', () => {
       let link;

package/src/archivist/services/index.js CHANGED Viewed

@@ -106,7 +106,7 @@ function createWrappedFilter(baseFunction, filterName, filterParams) {
     return;
   }
-  if (filterParams) {
+  if (filterParams || exposedFilters[filterName]) { // Built-in filters always receive their parameters before the context, even when none are declared
     const wrappedFilter = (webPageDOM, context) => baseFunction(webPageDOM, filterParams, context);
     Object.defineProperty(wrappedFilter, 'name', { value: filterName });

package/src/archivist/services/index.test.js CHANGED Viewed

@@ -6,6 +6,7 @@ import sinon from 'sinon';
 import sinonChai from 'sinon-chai';
 import expectedServices from '../../../test/fixtures/services.js';
+import createWebPageDOM from '../extract/dom.js';
 import * as exposedFilters from '../extract/exposedFilters.js';
 import Service from './service.js';
@@ -272,7 +273,18 @@ describe('Services', () => {
       const filterName = filterNames[0];
       const result = getServiceFilters({}, [filterName]);
-      expect(result).to.deep.equal([exposedFilters[filterName]]);
+      expect(result).to.have.length(1);
+      expect(result[0]).to.be.a('function');
+      expect(result[0].name).to.equal(filterName);
+    });
+    it('calls exposedFilters with their default parameters when declared by string name', () => {
+      const webPageDOM = createWebPageDOM('<!DOCTYPE html><html><body><a href="https://example.com/page?fbclid=abc&keep=value"></a></body></html>', 'https://example.com');
+      const [removeQueryParams] = getServiceFilters({}, ['removeQueryParams']);
+      removeQueryParams(webPageDOM, { fetch: 'https://example.com', select: [], remove: [], filter: ['removeQueryParams'] });
+      expect(webPageDOM.querySelector('a').getAttribute('href')).to.equal('https://example.com/page?keep=value');
     });
     it('returns filters from serviceFilters by string name', () => {
@@ -371,7 +383,7 @@ describe('Services', () => {
         expect(sourceDocument.filters).to.be.an('array');
         expect(sourceDocument.filters).to.have.length(2);
         expect(sourceDocument.filters[0]).to.be.a('function');
-        expect(sourceDocument.filters[0]).to.equal(exposedFilters[realFilterNames[0]]);
+        expect(sourceDocument.filters[0].name).to.equal(realFilterNames[0]);
         expect(sourceDocument.filters[1]).to.be.a('function');
         expect(sourceDocument.filters[1].name).to.equal('removePrintButton');
       });

package/src/collection-api/logger.js CHANGED Viewed

@@ -1,34 +1,13 @@
-import os from 'os';
 import config from 'config';
 import dotenv from 'dotenv';
 import winston from 'winston';
-import MailTransportWithRetry from '../logger/mail-transport-with-retry.js';
+import { addErrorMail } from '../logger/error-mail.js';
 dotenv.config({ quiet: true });
 const { combine, timestamp, printf, colorize } = winston.format;
-const transports = [new winston.transports.Console()];
-if (config.get('@opentermsarchive/engine.logger.sendMailOnError')) {
-  transports.push(new MailTransportWithRetry({
-    to: config.get('@opentermsarchive/engine.logger.sendMailOnError.to'),
-    from: config.get('@opentermsarchive/engine.logger.sendMailOnError.from'),
-    host: config.get('@opentermsarchive/engine.logger.smtp.host'),
-    port: config.get('@opentermsarchive/engine.logger.smtp.port'),
-    username: config.get('@opentermsarchive/engine.logger.smtp.username'),
-    password: process.env.OTA_ENGINE_SMTP_PASSWORD,
-    tls: true,
-    timeout: 60 * 1000,
-    formatter: args => args[Object.getOwnPropertySymbols(args)[1]], // Returns the full error message, the same visible in the console. It is referenced in the argument object with a Symbol of which we do not have the reference but we know it is the second one.
-    exitOnError: true,
-    level: 'error',
-    subject: `[OTA API] Error Report — ${os.hostname()}`,
-  }));
-}
 const logger = winston.createLogger({
   format: combine(
     colorize(),
@@ -39,8 +18,10 @@ const logger = winston.createLogger({
       return `${timestampPrefix}${level.padEnd(15)} ${message}`;
     }),
   ),
-  transports,
-  rejectionHandlers: transports,
+  transports: [new winston.transports.Console({ handleRejections: true })],
+  exitOnError: false,
 });
+await addErrorMail(logger, { component: 'Collection API', subject: 'API error' });
 export default logger;

package/src/logger/error-mail.js ADDED Viewed

@@ -0,0 +1,149 @@
+import os from 'os';
+import config from 'config';
+import winston from 'winston';
+import { getCollection } from '../archivist/collection/index.js';
+import MailTransportWithRetry from './mail-transport-with-retry.js';
+const SMTP_TIMEOUT = 60 * 1000;
+export const SENDING_BUDGET = SMTP_TIMEOUT; // The duration of one attempt: retries would only delay the restart when the relay is down
+const escapeHtml = text => String(text).replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+const CODE_STYLE = "font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace; background-color: #ffffff; border: 1px solid #dee2e6; border-radius: 4px; padding: 6px 10px; display: inline-block; color: #212529; cursor: text; user-select: all; -webkit-user-select: all; -moz-user-select: all; -ms-user-select: all;";
+const section = (title, color, content) => `
+              <div style="background-color: #f8f9fa; border-radius: 8px; padding: 15px; margin-bottom: 0;">
+                <h2 style="color: ${color}; margin: 0 0 0 0; font-size: 20px; border-bottom: 2px solid ${color}; padding-bottom: 8px;">${title}</h2>${content}
+              </div>`;
+const command = (label, code) => `
+                  <li style="margin: 0; padding: 0">
+                    <strong style="display: block; margin-bottom: 0;">${label}</strong>
+                    <code style="${CODE_STYLE}">${escapeHtml(code)}</code>
+                  </li>`;
+function formatBody({ collection, component, environmentPrefix }, { message, level }) {
+  const isError = level.includes('error');
+  const titleColor = isError ? '#dc3545' : '#ffc107';
+  const titleText = isError ? 'Error details' : 'Warning details';
+  const cpuCount = os.cpus().length;
+  const commands = [
+    ...(collection.host && collection.hostConfig?.ansible_user ? [[ 'Connect to the server:', `ssh ${collection.hostConfig.ansible_user}@${collection.host}` ]] : []),
+    [ 'List processes on the server:', 'pm2 list' ],
+    [ 'View the logs on the server:', 'pm2 logs <process-name>' ],
+    [ 'View additional logging options:', 'pm2 logs <process-name> --help' ],
+  ];
+  return `
+          <!DOCTYPE html>
+          <html lang="en">
+            <head>
+              <meta charset="utf-8">
+              <meta name="viewport" content="width=device-width, initial-scale=1.0">
+              <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+              <title>OTA Error Report</title>
+            </head>
+            <body style="font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Helvetica, Arial, sans-serif; line-height: 1.6; color: #333333; max-width: 800px; margin: 0 auto; padding: 0px 20px 20px 20px;">
+              <h1 style="color: #212529; font-size: 24px; margin: 10px 0; text-align: center; padding-bottom: 10px;">${escapeHtml(`${environmentPrefix}Open Terms Archive ${component} error report — ${collection.name} Collection`)}</h1>
+              ${section(titleText, titleColor, `
+                <div style="background-color: #ffffff; border: 1px solid #dee2e6; border-radius: 4px; padding: 12px; margin: 8px 0;">
+                  <code style="margin: 0; font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace; font-size: 14px; color: #212529; white-space: pre-wrap; display: block;">${escapeHtml(message)}</code>
+                </div>`)}
+              ${section('System information', '#212529', `
+                <div style="color: #6c757d; font-size: 14px; margin: 0;">
+                  Hostname: ${escapeHtml(os.hostname())}<br>
+                  Platform: ${escapeHtml(`${os.platform()} ${os.release()}`)}<br>
+                  Architecture: ${os.arch()}<br>
+                  CPU Cores: ${cpuCount}<br>
+                  CPU Load (1/5/15 min): ${os.loadavg().map(load => `${Math.min(100, (load / cpuCount) * 100).toFixed(1)}%`).join(' / ')}<br>
+                  Total Memory: ${(os.totalmem() / (1024 * 1024 * 1024)).toFixed(2)} GB<br>
+                  Free Memory: ${(os.freemem() / (1024 * 1024 * 1024)).toFixed(2)} GB${collection.host ? `<br>
+                  Server IP: ${escapeHtml(collection.host)}` : ''}
+                </div>`)}
+              ${section('Helpful commands', '#198754', `
+                <ul style="list-style-type: none; padding-left: 0; margin: 0;">${commands.map(([ label, code ]) => command(label, code)).join('')}
+                  <li style="margin: 0; padding: 0">
+                    <strong style="display: block; margin-bottom: 0;">View deployment documentation to see how to start, stop, and restart the server:</strong>
+                    <a href="https://github.com/OpenTermsArchive/deployment" style="color: #198754; text-decoration: none; border-bottom: 1px solid #198754;">github.com/OpenTermsArchive/deployment</a>
+                  </li>
+                </ul>`)}
+              <div style="margin-top: 15px; padding-top: 15px; border-top: 1px solid #dee2e6; font-size: 12px; color: #6c757d; text-align: center;">
+                This is an automated message from the Open Terms Archive engine. Please do not reply to this email.
+              </div>
+            </body>
+          </html>
+        `;
+}
+export function createErrorMailTransports(logger, { collection, component, subject, warningSubject }) {
+  if (!config.get('@opentermsarchive/engine.logger.sendMailOnError')) {
+    return [];
+  }
+  if (process.env.OTA_ENGINE_SMTP_PASSWORD === undefined) {
+    logger.warn('Environment variable "OTA_ENGINE_SMTP_PASSWORD" was not found; log emails cannot be sent');
+    return [];
+  }
+  const environment = process.env.NODE_ENV || 'development'; // Same default as node-config, so the prefix matches the loaded configuration
+  const environmentPrefix = environment == 'production' ? '' : `[${environment}] `; // Make emails sent from a developer machine recognisable at a glance
+  const mailerOptions = {
+    to: config.get('@opentermsarchive/engine.logger.sendMailOnError.to'),
+    from: config.get('@opentermsarchive/engine.logger.sendMailOnError.from'),
+    host: config.get('@opentermsarchive/engine.logger.smtp.host'),
+    port: config.get('@opentermsarchive/engine.logger.smtp.port'),
+    username: config.get('@opentermsarchive/engine.logger.smtp.username'),
+    password: process.env.OTA_ENGINE_SMTP_PASSWORD,
+    tls: true,
+    timeout: SMTP_TIMEOUT,
+    html: true,
+    formatter: info => formatBody({ collection, component, environmentPrefix }, info),
+    handleRejections: true,
+  };
+  const transports = [new MailTransportWithRetry({ ...mailerOptions, level: 'error', subject: `${environmentPrefix}${subject}` })];
+  if (warningSubject && config.has('@opentermsarchive/engine.logger.sendMailOnError.sendWarnings') && config.get('@opentermsarchive/engine.logger.sendMailOnError.sendWarnings')) { // Only callers providing a subject for warnings can send them
+    transports.push(new MailTransportWithRetry({
+      ...mailerOptions,
+      level: 'warn',
+      subject: `${environmentPrefix}${warningSubject}`,
+      format: winston.format(info => (info[Symbol.for('level')] === 'warn' ? info : false))(), // Winston transports receive every level at or above theirs, so errors would otherwise be emailed a second time as warnings
+    }));
+  }
+  return transports;
+}
+export function exitOnUnhandledRejection(transports, { emitter = process } = {}) {
+  const mailTransports = transports.filter(transport => transport instanceof MailTransportWithRetry);
+  emitter.on('unhandledRejection', async () => {
+    await new Promise(resolve => { setImmediate(resolve); }); // Winston's own listener, registered before this one, logs the rejection on the current tick; give the transports a chance to receive it before waiting for them
+    await Promise.race([
+      Promise.all(mailTransports.map(transport => transport.flush())),
+      new Promise(resolve => { setTimeout(resolve, SENDING_BUDGET).unref(); }),
+    ]);
+    process.exit(1);
+  });
+}
+export async function addErrorMail(logger, { component, subject, warningSubject }, { emitter = process } = {}) {
+  const collection = await getCollection();
+  const transports = createErrorMailTransports(logger, {
+    collection,
+    component,
+    subject: `${subject} on ${collection.id} collection`,
+    warningSubject: warningSubject && `${warningSubject} on ${collection.id} collection`,
+  });
+  transports.forEach(transport => logger.add(transport));
+  exitOnUnhandledRejection(logger.transports, { emitter });
+}

package/src/logger/error-mail.test.js ADDED Viewed

@@ -0,0 +1,458 @@
+import { EventEmitter } from 'node:events';
+import os from 'node:os';
+import { expect, use } from 'chai';
+import config from 'config';
+import sinon from 'sinon';
+import sinonChai from 'sinon-chai';
+import winston from 'winston';
+import { SENDING_BUDGET, addErrorMail, createErrorMailTransports, exitOnUnhandledRejection } from './error-mail.js';
+import MailTransportWithRetry from './mail-transport-with-retry.js';
+use(sinonChai);
+const SEND_MAIL_ON_ERROR = {
+  to: 'admin@example.com',
+  from: 'noreply@example.com',
+  sendWarnings: false,
+};
+describe('Error mail', () => {
+  let configValues;
+  let originalPassword;
+  let originalEnvironment;
+  before(() => {
+    originalPassword = process.env.OTA_ENGINE_SMTP_PASSWORD;
+    originalEnvironment = process.env.NODE_ENV;
+  });
+  after(() => {
+    if (originalPassword === undefined) {
+      delete process.env.OTA_ENGINE_SMTP_PASSWORD;
+    } else {
+      process.env.OTA_ENGINE_SMTP_PASSWORD = originalPassword;
+    }
+    process.env.NODE_ENV = originalEnvironment;
+  });
+  beforeEach(() => {
+    configValues = {
+      '@opentermsarchive/engine.logger.sendMailOnError': SEND_MAIL_ON_ERROR,
+      '@opentermsarchive/engine.logger.sendMailOnError.to': SEND_MAIL_ON_ERROR.to,
+      '@opentermsarchive/engine.logger.sendMailOnError.from': SEND_MAIL_ON_ERROR.from,
+      '@opentermsarchive/engine.logger.sendMailOnError.sendWarnings': SEND_MAIL_ON_ERROR.sendWarnings,
+      '@opentermsarchive/engine.logger.smtp.host': 'smtp.example.com',
+      '@opentermsarchive/engine.logger.smtp.port': 587,
+      '@opentermsarchive/engine.logger.smtp.username': 'user',
+      '@opentermsarchive/engine.collectionPath': './test/test-declarations',
+    };
+    sinon.stub(config, 'get').callsFake(key => configValues[key]);
+    sinon.stub(config, 'has').callsFake(key => configValues[key] !== undefined);
+    process.env.NODE_ENV = 'production';
+    process.env.OTA_ENGINE_SMTP_PASSWORD = 'secret';
+  });
+  afterEach(() => {
+    sinon.restore();
+  });
+  describe('#createErrorMailTransports', () => {
+    const logger = { warn: () => {} };
+    const collection = {
+      id: 'test',
+      name: 'Test',
+      host: '203.0.113.1',
+      hostConfig: { ansible_user: 'ota' },
+    };
+    const component = 'test component';
+    const subject = 'Error on test collection';
+    const warningSubject = 'Warning on test collection';
+    context('when sending mail on error is disabled', () => {
+      beforeEach(() => {
+        configValues['@opentermsarchive/engine.logger.sendMailOnError'] = false;
+      });
+      it('returns no transport', () => {
+        expect(createErrorMailTransports(logger, { collection, component, subject, warningSubject })).to.be.empty;
+      });
+      it('does not warn', () => {
+        const warnSpy = sinon.spy(logger, 'warn');
+        createErrorMailTransports(logger, { collection, component, subject, warningSubject });
+        expect(warnSpy).to.not.have.been.called;
+      });
+    });
+    context('when the SMTP password is not defined', () => {
+      beforeEach(() => {
+        delete process.env.OTA_ENGINE_SMTP_PASSWORD;
+      });
+      it('returns no transport', () => {
+        expect(createErrorMailTransports(logger, { collection, component, subject, warningSubject })).to.be.empty;
+      });
+      it('warns through the logger that emails cannot be sent', () => {
+        const warnSpy = sinon.spy(logger, 'warn');
+        createErrorMailTransports(logger, { collection, component, subject, warningSubject });
+        expect(warnSpy).to.have.been.calledOnce;
+        expect(warnSpy.firstCall.args[0]).to.include('OTA_ENGINE_SMTP_PASSWORD');
+      });
+    });
+    context('when sending mail on error is enabled', () => {
+      let transports;
+      context('without warnings', () => {
+        beforeEach(() => {
+          transports = createErrorMailTransports(logger, { collection, component, subject, warningSubject });
+        });
+        it('returns a single transport', () => {
+          expect(transports).to.have.lengthOf(1);
+        });
+        it('returns a mail transport with retry', () => {
+          expect(transports[0]).to.be.an.instanceOf(MailTransportWithRetry);
+        });
+        it('sends errors only', () => {
+          expect(transports[0].level).to.equal('error');
+        });
+        it('uses the given subject', () => {
+          expect(transports[0].mailTransport.subject).to.equal(subject);
+        });
+        it('uses the configured recipient and sender', () => {
+          expect(transports[0].mailTransport.to).to.equal(SEND_MAIL_ON_ERROR.to);
+          expect(transports[0].mailTransport.from).to.equal(SEND_MAIL_ON_ERROR.from);
+        });
+        it('sends HTML emails', () => {
+          expect(transports[0].mailTransport.html).to.be.true;
+        });
+        it('handles unhandled rejections', () => {
+          expect(transports[0].handleRejections).to.be.true;
+        });
+      });
+      context('with warnings enabled and a subject for them', () => {
+        beforeEach(() => {
+          configValues['@opentermsarchive/engine.logger.sendMailOnError.sendWarnings'] = true;
+          transports = createErrorMailTransports(logger, { collection, component, subject, warningSubject });
+        });
+        it('returns two transports', () => {
+          expect(transports).to.have.lengthOf(2);
+        });
+        it('sends warnings with the second transport', () => {
+          expect(transports[1].level).to.equal('warn');
+        });
+        it('uses the warning subject for the second transport', () => {
+          expect(transports[1].mailTransport.subject).to.equal(warningSubject);
+        });
+        context('when logging through both transports', () => {
+          let sent;
+          beforeEach(() => {
+            sent = [];
+            transports.forEach((transport, index) => {
+              sinon.stub(transport.mailTransport, 'log').callsFake((info, callback) => {
+                sent.push(index);
+                setImmediate(() => {
+                  transport.mailTransport.emit('logged');
+                  callback();
+                });
+              });
+            });
+          });
+          it('sends errors through the error transport only', async () => {
+            const logger = winston.createLogger({ format: winston.format.colorize(), transports });
+            logger.error('boom');
+            await new Promise(resolve => { setTimeout(resolve, 10); });
+            expect(sent).to.deep.equal([0]);
+          });
+          it('sends warnings through the warning transport only', async () => {
+            const logger = winston.createLogger({ format: winston.format.colorize(), transports });
+            logger.warn('inaccessible');
+            await new Promise(resolve => { setTimeout(resolve, 10); });
+            expect(sent).to.deep.equal([1]);
+          });
+        });
+      });
+      context('with warnings enabled but no subject for them', () => {
+        beforeEach(() => {
+          configValues['@opentermsarchive/engine.logger.sendMailOnError.sendWarnings'] = true;
+          transports = createErrorMailTransports(logger, { collection, component, subject });
+        });
+        it('returns the error transport only', () => {
+          expect(transports).to.have.lengthOf(1);
+          expect(transports[0].level).to.equal('error');
+        });
+      });
+      context('with a subject for warnings but warnings not configured', () => {
+        beforeEach(() => {
+          delete configValues['@opentermsarchive/engine.logger.sendMailOnError.sendWarnings'];
+          transports = createErrorMailTransports(logger, { collection, component, subject, warningSubject });
+        });
+        it('returns the error transport only', () => {
+          expect(transports).to.have.lengthOf(1);
+          expect(transports[0].level).to.equal('error');
+        });
+      });
+      context('with a subject for warnings but warnings disabled', () => {
+        beforeEach(() => {
+          transports = createErrorMailTransports(logger, { collection, component, subject, warningSubject });
+        });
+        it('returns the error transport only', () => {
+          expect(transports).to.have.lengthOf(1);
+          expect(transports[0].level).to.equal('error');
+        });
+      });
+      context('email body', () => {
+        let formatter;
+        let body;
+        beforeEach(() => {
+          [{ mailTransport: { formatter } }] = createErrorMailTransports(logger, { collection, component, subject, warningSubject });
+        });
+        context('for an error', () => {
+          beforeEach(() => {
+            body = formatter({ message: 'Error: <boom> & co', level: 'error' });
+          });
+          it('is titled as an error', () => {
+            expect(body).to.include('Error details');
+          });
+          it('includes the escaped message', () => {
+            expect(body).to.include('Error: &lt;boom&gt; &amp; co');
+            expect(body).to.not.include('<boom>');
+          });
+          it('names the collection', () => {
+            expect(body).to.include(`${collection.name} Collection`);
+          });
+          it('names the component', () => {
+            expect(body).to.include(`Open Terms Archive ${component} error report`);
+          });
+          it('includes the hostname', () => {
+            expect(body).to.include(os.hostname());
+          });
+          it('includes the command to connect to the server', () => {
+            expect(body).to.include(`ssh ${collection.hostConfig.ansible_user}@${collection.host}`);
+          });
+        });
+        context('for a warning', () => {
+          beforeEach(() => {
+            body = formatter({ message: 'Inaccessible content', level: 'warn' });
+          });
+          it('is titled as a warning', () => {
+            expect(body).to.include('Warning details');
+          });
+        });
+        context('when interpolated values contain HTML characters', () => {
+          beforeEach(() => {
+            [{ mailTransport: { formatter } }] = createErrorMailTransports(logger, { collection: { id: 'test', name: 'R&D <beta>', host: '203.0.113.1', hostConfig: { ansible_user: 'ota<x>' } }, component: 'API <v2>', subject, warningSubject });
+            body = formatter({ message: 'Error', level: 'error' });
+          });
+          it('escapes the collection name', () => {
+            expect(body).to.include('R&amp;D &lt;beta&gt; Collection');
+            expect(body).to.not.include('<beta>');
+          });
+          it('escapes the component', () => {
+            expect(body).to.include('Open Terms Archive API &lt;v2&gt; error report');
+          });
+          it('escapes the command to connect to the server', () => {
+            expect(body).to.include('ssh ota&lt;x&gt;@203.0.113.1');
+            expect(body).to.not.include('ota<x>');
+          });
+          it('escapes the commands placeholders', () => {
+            expect(body).to.include('pm2 logs &lt;process-name&gt;');
+          });
+        });
+        context('when the collection has no deployment inventory', () => {
+          beforeEach(() => {
+            [{ mailTransport: { formatter } }] = createErrorMailTransports(logger, { collection: { id: 'test', name: 'Test' }, component, subject, warningSubject });
+            body = formatter({ message: 'Error', level: 'error' });
+          });
+          it('omits the command to connect to the server', () => {
+            expect(body).to.not.include('ssh ');
+          });
+        });
+      });
+      context('outside production', () => {
+        beforeEach(() => {
+          process.env.NODE_ENV = 'staging';
+          configValues['@opentermsarchive/engine.logger.sendMailOnError.sendWarnings'] = true;
+          transports = createErrorMailTransports(logger, { collection, component, subject, warningSubject });
+        });
+        it('prefixes the error subject with the environment', () => {
+          expect(transports[0].mailTransport.subject).to.equal(`[staging] ${subject}`);
+        });
+        it('prefixes the warning subject with the environment', () => {
+          expect(transports[1].mailTransport.subject).to.equal(`[staging] ${warningSubject}`);
+        });
+        it('prefixes the title of the body with the environment', () => {
+          expect(transports[0].mailTransport.formatter({ message: 'Error', level: 'error' })).to.include(`[staging] Open Terms Archive ${component} error report`);
+        });
+      });
+      context('when the environment is not defined', () => {
+        beforeEach(() => {
+          delete process.env.NODE_ENV;
+          transports = createErrorMailTransports(logger, { collection, component, subject, warningSubject });
+        });
+        it('prefixes the subject with the development environment', () => {
+          expect(transports[0].mailTransport.subject).to.equal(`[development] ${subject}`);
+        });
+      });
+    });
+  });
+  describe('#addErrorMail', () => {
+    let logger;
+    let warnSpy;
+    beforeEach(() => {
+      logger = winston.createLogger({ transports: [new winston.transports.Console({ silent: true })], exitOnError: false });
+      warnSpy = sinon.spy(logger, 'warn');
+    });
+    context('when sending mail on error is enabled with warnings', () => {
+      beforeEach(async () => {
+        configValues['@opentermsarchive/engine.logger.sendMailOnError.sendWarnings'] = true;
+        await addErrorMail(logger, { component: 'engine', subject: 'Server error', warningSubject: 'Inaccessible content' }, { emitter: new EventEmitter() });
+      });
+      it('adds the mail transports to the logger', () => {
+        expect(logger.transports).to.have.lengthOf(3);
+        expect(logger.transports[1]).to.be.an.instanceOf(MailTransportWithRetry);
+        expect(logger.transports[2]).to.be.an.instanceOf(MailTransportWithRetry);
+      });
+      it('names the collection in the subjects', () => {
+        expect(logger.transports[1].mailTransport.subject).to.equal('Server error on test collection');
+        expect(logger.transports[2].mailTransport.subject).to.equal('Inaccessible content on test collection');
+      });
+    });
+    context('when the SMTP password is not defined', () => {
+      beforeEach(async () => {
+        delete process.env.OTA_ENGINE_SMTP_PASSWORD;
+        await addErrorMail(logger, { component: 'engine', subject: 'Server error' }, { emitter: new EventEmitter() });
+      });
+      it('warns through the logger', () => {
+        expect(warnSpy).to.have.been.calledOnce;
+        expect(warnSpy.firstCall.args[0]).to.include('OTA_ENGINE_SMTP_PASSWORD');
+      });
+      it('adds no transport', () => {
+        expect(logger.transports).to.have.lengthOf(1);
+      });
+    });
+  });
+  describe('#exitOnUnhandledRejection', () => {
+    let emitter;
+    let processExitStub;
+    let mailTransport;
+    let sent;
+    const tick = () => new Promise(resolve => { setImmediate(resolve); });
+    beforeEach(() => {
+      emitter = new EventEmitter();
+      processExitStub = sinon.stub(process, 'exit');
+      mailTransport = Object.create(MailTransportWithRetry.prototype);
+      mailTransport.flush = () => new Promise(resolve => { sent = resolve; });
+      exitOnUnhandledRejection([ new winston.transports.Console({ silent: true }), mailTransport ], { emitter });
+    });
+    context('when emails are sent', () => {
+      beforeEach(() => {
+        emitter.emit('unhandledRejection', new Error('boom'));
+      });
+      it('waits for pending emails before exiting', async () => {
+        await tick();
+        await tick();
+        expect(processExitStub).to.not.have.been.called;
+      });
+      it('exits with a failure code once emails are sent', async () => {
+        await tick();
+        sent();
+        await tick();
+        expect(processExitStub).to.have.been.calledOnceWithExactly(1);
+      });
+    });
+    context('when emails cannot be sent', () => {
+      let clock;
+      beforeEach(() => {
+        clock = sinon.useFakeTimers();
+        emitter.emit('unhandledRejection', new Error('boom'));
+      });
+      afterEach(() => {
+        clock.restore();
+      });
+      it('exits with a failure code after one sending attempt', async () => {
+        await clock.tickAsync(SENDING_BUDGET - 1);
+        expect(processExitStub).to.not.have.been.called;
+        await clock.tickAsync(1);
+        expect(processExitStub).to.have.been.calledOnceWithExactly(1);
+      });
+    });
+  });
+});

package/src/logger/index.js CHANGED Viewed

@@ -1,17 +1,11 @@
-import os from 'os';
 import config from 'config';
 import winston from 'winston';
-import { getCollection } from '../archivist/collection/index.js';
-import MailTransportWithRetry from './mail-transport-with-retry.js';
+import { addErrorMail } from './error-mail.js';
 import { formatDuration } from './utils.js';
 const { combine, timestamp, printf, colorize } = winston.format;
-const collection = await getCollection();
 const alignedWithColorsAndTime = combine(
   colorize(),
   timestamp({ format: 'YYYY-MM-DDTHH:mm:ssZ' }),
@@ -28,141 +22,15 @@ const alignedWithColorsAndTime = combine(
   }),
 );
-const consoleTransport = new winston.transports.Console({ silent: process.env.NODE_ENV === 'test' });
-const transports = [consoleTransport];
+const consoleTransport = new winston.transports.Console({ silent: process.env.NODE_ENV === 'test', handleRejections: true });
 const logger = winston.createLogger({
   format: alignedWithColorsAndTime,
-  transports,
-  rejectionHandlers: transports,
-  exitOnError: true,
+  transports: [consoleTransport],
+  exitOnError: false,
 });
-logger.on('error', err => {
-  if ('smtp' in err) { // Check if err has an `smtp` property, even if it's undefined
-    logger.warn({ message: `Uncaught exception from SMTP mailer detected and treated as an operational error; process will continue running:\n${err.stack}` });
-    return; // Prevent process exit
-  }
-  return process.exit(1); // Exit process for other errors
-});
-if (config.get('@opentermsarchive/engine.logger.sendMailOnError')) {
-  if (process.env.OTA_ENGINE_SMTP_PASSWORD === undefined) {
-    logger.warn('Environment variable "OTA_ENGINE_SMTP_PASSWORD" was not found; log emails cannot be sent');
-  } else {
-    const mailerOptions = {
-      to: config.get('@opentermsarchive/engine.logger.sendMailOnError.to'),
-      from: config.get('@opentermsarchive/engine.logger.sendMailOnError.from'),
-      host: config.get('@opentermsarchive/engine.logger.smtp.host'),
-      port: config.get('@opentermsarchive/engine.logger.smtp.port'),
-      username: config.get('@opentermsarchive/engine.logger.smtp.username'),
-      password: process.env.OTA_ENGINE_SMTP_PASSWORD,
-      tls: true,
-      timeout: 60 * 1000,
-      html: false,
-      formatter({ message, level }) {
-        const isError = level.includes('error');
-        const titleColor = isError ? '#dc3545' : '#ffc107';
-        const titleText = isError ? 'Error details' : 'Warning details';
-        return `
-          <!DOCTYPE html>
-          <html lang="en">
-            <head>
-              <meta charset="utf-8">
-              <meta name="viewport" content="width=device-width, initial-scale=1.0">
-              <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-              <title>OTA Error Report</title>
-            </head>
-            <body style="font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Helvetica, Arial, sans-serif; line-height: 1.6; color: #333333; max-width: 800px; margin: 0 auto; padding: 0px 20px 20px 20px;">
-              <h1 style="color: #212529; font-size: 24px; margin: 10px 0; text-align: center; padding-bottom: 10px;">Open Terms Archive engine error report — ${collection.name} Collection</h1>
-              <div style="background-color: #f8f9fa; border-radius: 8px; padding: 15px; margin-bottom: 0;">
-                <h2 style="color: ${titleColor}; margin: 0 0 0 0; font-size: 20px; border-bottom: 2px solid ${titleColor}; padding-bottom: 8px;">${titleText}</h2>
-                <div style="background-color: #ffffff; border: 1px solid #dee2e6; border-radius: 4px; padding: 12px; margin: 8px 0;">
-                  <code style="maring: 0; font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace; font-size: 14px; color: #212529; white-space: pre-wrap; display: block;">${message}</code>
-                </div>
-              </div>
-              <div style="background-color: #f8f9fa; border-radius: 8px; padding: 15px; margin-bottom: 0;">
-                <h2 style="color: #212529; margin: 0 0 0 0; font-size: 20px; border-bottom: 2px solid #212529; padding-bottom: 8px;">System information</h2>
-                <div style="color: #6c757d; font-size: 14px; margin: 0;">
-                  Hostname: ${os.hostname()}<br>
-                  Platform: ${os.platform()} ${os.release()}<br>
-                  Architecture: ${os.arch()}<br>
-                  CPU Cores: ${os.cpus().length}<br>
-                  CPU Load (1/5/15 min): ${os.loadavg().map(load =>
-    `${Math.min(100, (load / os.cpus().length) * 100).toFixed(1)}%`).join(' / ')}<br>
-                  Total Memory: ${(os.totalmem() / (1024 * 1024 * 1024)).toFixed(2)} GB<br>
-                  Free Memory: ${(os.freemem() / (1024 * 1024 * 1024)).toFixed(2)} GB${collection.host ? `<br>
-                  Server IP: ${collection.host}` : ''}
-                </div>
-              </div>
-              <div style="background-color: #f8f9fa; border-radius: 8px; padding: 15px; margin-bottom: 0;">
-                <h2 style="color: #198754; margin: 0 0 0 0; font-size: 20px; border-bottom: 2px solid #198754; padding-bottom: 8px;">Helpful commands</h2>
-                <ul style="list-style-type: none; padding-left: 0; margin: 0;">
-                  ${collection.host && collection.hostConfig?.ansible_user ? `
-                  <li style="margin: 0; padding: 0">
-                    <strong style="display: block; margin-bottom: 0;">Connect to the server:</strong>
-                    <code style="maring: 0; font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace; background-color: #ffffff; border: 1px solid #dee2e6; border-radius: 4px; padding: 6px 10px; display: inline-block; color: #212529; cursor: text; user-select: all; -webkit-user-select: all; -moz-user-select: all; -ms-user-select: all;">ssh ${collection.hostConfig.ansible_user}@${collection.host}</code>
-                  </li>` : ''}
-                  <li style="margin: 0; padding: 0">
-                    <strong style="display: block; margin-bottom: 0;">List processes on the server:</strong>
-                    <code style="maring: 0; font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace; background-color: #ffffff; border: 1px solid #dee2e6; border-radius: 4px; padding: 6px 10px; display: inline-block; color: #212529; cursor: text; user-select: all; -webkit-user-select: all; -moz-user-select: all; -ms-user-select: all;">pm2 list</code>
-                  </li>
-                  <li style="margin: 0; padding: 0">
-                    <strong style="display: block; margin-bottom: 0;">View the logs on the server:</strong>
-                    <code style="maring: 0; font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace; background-color: #ffffff; border: 1px solid #dee2e6; border-radius: 4px; padding: 6px 10px; display: inline-block; color: #212529; cursor: text; user-select: all; -webkit-user-select: all; -moz-user-select: all; -ms-user-select: all;">pm2 logs &lt;process-name&gt;</code>
-                  </li>
-                  <li style="margin: 0; padding: 0">
-                    <strong style="display: block; margin-bottom: 0;">View additional logging options:</strong>
-                    <code style="maring: 0; font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace; background-color: #ffffff; border: 1px solid #dee2e6; border-radius: 4px; padding: 6px 10px; display: inline-block; color: #212529; cursor: text; user-select: all; -webkit-user-select: all; -moz-user-select: all; -ms-user-select: all;">pm2 logs &lt;process-name&gt; --help</code>
-                  </li>
-                  <li style="margin: 0; padding: 0">
-                    <strong style="display: block; margin-bottom: 0;">View deployment documentation to see how to start, stop, and restart the server:</strong>
-                    <a href="https://github.com/OpenTermsArchive/deployment" style="color: #198754; text-decoration: none; border-bottom: 1px solid #198754;">github.com/OpenTermsArchive/deployment</a>
-                  </li>
-                </ul>
-              </div>
-              <div style="margin-top: 15px; padding-top: 15px; border-top: 1px solid #dee2e6; font-size: 12px; color: #6c757d; text-align: center;">
-                This is an automated message from the Open Terms Archive engine. Please do not reply to this email.
-              </div>
-            </body>
-          </html>
-        `;
-      },
-    };
-    transports.push(new MailTransportWithRetry({
-      ...mailerOptions,
-      level: 'error',
-      subject: `Server error on ${collection.id} collection`,
-    }));
-    if (config.get('@opentermsarchive/engine.logger.sendMailOnError.sendWarnings')) {
-      transports.push(new MailTransportWithRetry({
-        ...mailerOptions,
-        level: 'warn',
-        subject: `Inaccessible content on ${collection.id} collection`,
-      }));
-    }
-  }
-}
-logger.configure({
-  transports,
-  rejectionHandlers: transports,
-  exitOnError: true,
-});
+await addErrorMail(logger, { component: 'engine', subject: 'Server error', warningSubject: 'Inaccessible content' });
 let recordedSnapshotsCount;
 let recordedVersionsCount;

package/src/logger/mail-transport-with-retry.js CHANGED Viewed

@@ -1,5 +1,3 @@
-import { once } from 'node:events';
 import async from 'async';
 import winston from 'winston';
@@ -21,25 +19,52 @@ class MailTransportWithRetry extends winston.Transport {
   constructor(options) {
     super(options);
     this.mailTransport = new winston.transports.Mail(options);
+    this.pending = new Set();
   }
-  async log(info, callback) {
-    try {
-      await async.retry(RETRY_OPTIONS, async () => {
-        const result = Promise.race([
-          once(this.mailTransport, 'logged'),
-          once(this.mailTransport, 'error').then(([err]) => { throw err; }),
-        ]);
+  log(info, callback) {
+    callback(); // Winston delivers each log to every transport in sequence, so waiting for the SMTP retries here would hold back the other transports too
-        this.mailTransport.log(info, () => {});
+    const sending = this.send(info).finally(() => this.pending.delete(sending));
-        return result;
-      });
+    this.pending.add(sending);
+    return sending;
+  }
+  async send(info) {
+    try {
+      await async.retry(RETRY_OPTIONS, async () => { await this.attempt(info); }); // The task must be an async function for async.retry to await it rather than wait for a callback
     } catch (error) {
-      console.warn(`SMTP mail sending failed after ${RETRY_OPTIONS.times} attempts: ${error.message}`);
-      this.emit('error', error);
+      console.warn(`SMTP mail sending failed after ${RETRY_OPTIONS.times} attempts; giving up on this email:\n${error.stack}`);
     }
-    callback();
+  }
+  attempt(info) {
+    return new Promise((resolve, reject) => {
+      const removeListeners = () => {
+        this.mailTransport.off('logged', onLogged);
+        this.mailTransport.off('error', onError);
+      };
+      function onLogged() {
+        removeListeners();
+        resolve();
+      }
+      function onError(error) {
+        removeListeners();
+        reject(error);
+      }
+      this.mailTransport.once('logged', onLogged);
+      this.mailTransport.once('error', onError);
+      this.mailTransport.log(info, () => {});
+    });
+  }
+  async flush() {
+    await Promise.allSettled(this.pending);
   }
 }

package/src/logger/mail-transport-with-retry.test.js CHANGED Viewed

@@ -29,6 +29,25 @@ describe('MailTransportWithRetry', () => {
     clock.restore();
   });
+  describe('#flush', () => {
+    it('resolves once pending emails are sent', async () => {
+      let flushed = false;
+      transport.log({ message: 'test' }, () => {});
+      const flushPromise = transport.flush().then(() => { flushed = true; });
+      await clock.tickAsync(0);
+      expect(flushed).to.be.false;
+      mockMailTransport.emit('logged');
+      await flushPromise;
+      expect(flushed).to.be.true;
+      expect(transport.pending).to.be.empty;
+    });
+  });
   describe('#log', () => {
     context('when email is sent successfully on first attempt', () => {
       it('calls callback without error', async () => {
@@ -62,6 +81,24 @@ describe('MailTransportWithRetry', () => {
         expect(consoleWarnStub).not.to.have.been.called;
       });
+      it('calls callback before the email is sent', () => {
+        const callback = sinon.spy();
+        transport.log({ message: 'test' }, callback);
+        expect(callback).to.have.been.calledOnce;
+      });
+      it('removes its listeners from the mail transport', async () => {
+        const logPromise = transport.log({ message: 'test' }, () => {});
+        mockMailTransport.emit('logged');
+        await logPromise;
+        expect(mockMailTransport.listenerCount('logged')).to.equal(0);
+        expect(mockMailTransport.listenerCount('error')).to.equal(0);
+      });
     });
     context('when email fails then succeeds on retry', () => {
@@ -98,7 +135,7 @@ describe('MailTransportWithRetry', () => {
     });
     context('when email fails after all retry attempts', () => {
-      it('emits error event after all retries are exhausted', async () => {
+      it('does not emit an error event after all retries are exhausted', async () => {
         const errorHandler = sinon.spy();
         transport.on('error', errorHandler);
@@ -117,15 +154,12 @@ describe('MailTransportWithRetry', () => {
         mockMailTransport.emit('error', new Error('SMTP timeout'));
         await logPromise;
-        expect(errorHandler).to.have.been.calledOnce;
-        expect(errorHandler.firstCall.args[0].message).to.equal('SMTP timeout');
+        expect(errorHandler).not.to.have.been.called;
       });
       it('calls callback even after failure', async () => {
         const callback = sinon.spy();
-        transport.on('error', () => {}); // Prevent unhandled error
         const logPromise = transport.log({ message: 'test' }, callback);
         mockMailTransport.emit('error', new Error('SMTP timeout'));
@@ -144,8 +178,6 @@ describe('MailTransportWithRetry', () => {
       });
       it('logs final failure warning', async () => {
-        transport.on('error', () => {}); // Prevent unhandled error
         const logPromise = transport.log({ message: 'test' }, () => {});
         mockMailTransport.emit('error', new Error('SMTP timeout'));
@@ -161,6 +193,26 @@ describe('MailTransportWithRetry', () => {
         await logPromise;
         expect(consoleWarnStub.lastCall.args[0]).to.include(`failed after ${RETRY_DELAYS.length + 1} attempts`);
+        expect(consoleWarnStub.lastCall.args[0]).to.include('Error: SMTP timeout');
+      });
+      it('removes its listeners from the mail transport', async () => {
+        const logPromise = transport.log({ message: 'test' }, () => {});
+        mockMailTransport.emit('error', new Error('SMTP timeout'));
+        await clock.tickAsync(RETRY_DELAYS[0]);
+        mockMailTransport.emit('error', new Error('SMTP timeout'));
+        await clock.tickAsync(RETRY_DELAYS[1]);
+        mockMailTransport.emit('error', new Error('SMTP timeout'));
+        await clock.tickAsync(RETRY_DELAYS[2]);
+        mockMailTransport.emit('error', new Error('SMTP timeout'));
+        await logPromise;
+        expect(mockMailTransport.listenerCount('logged')).to.equal(0);
+        expect(mockMailTransport.listenerCount('error')).to.equal(0);
       });
     });