@opentermsarchive/engine 10.3.3 → 10.5.0

package/config/default.json

@@ -43,7 +43,8 @@
     },
     "logger": {
       "smtp": {
-        "host": "smtp-relay.sendinblue.com",
+        "host": "smtp-relay.brevo.com",
+        "port": 587,
         "username": "admin@opentermsarchive.org"
       },
       "sendMailOnError": false,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "10.3.3",
+  "version": "10.5.0",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {

package/src/archivist/fetcher/fullDomFetcher.js

@@ -8,37 +8,64 @@ let browser;
 export default async function fetch(url, cssSelectors, config) {
   puppeteer.use(stealthPlugin({ locale: config.language }));
 
-  let context;
-  let page;
-  let client;
-  let response;
-  const selectors = [].concat(cssSelectors);
-
   if (!browser) {
     throw new Error('The headless browser should be controlled manually with "launchHeadlessBrowser" and "stopHeadlessBrowser".');
   }
 
+  let context;
+  let page;
+  let client;
+
   try {
     context = await browser.createBrowserContext(); // Create an isolated browser context to ensure complete isolation between fetches (cookies, localStorage, sessionStorage, IndexedDB, cache)
     page = await context.newPage();
+    client = await page.createCDPSession();
 
-    await page.setViewport({ width: 1920, height: 1080 }); // Set a realistic viewport size to avoid detection based on default Puppeteer dimensions (800x600)
-    await page.setDefaultNavigationTimeout(config.navigationTimeout);
-    await page.setExtraHTTPHeaders({ 'Accept-Language': config.language });
+    await configurePage(page, client, config);
 
-    // Use CDP to ensure the browser language is set correctly (most reliable method, see https://zirkelc.dev/posts/puppeteer-language-experiment)
-    client = await page.createCDPSession();
+    const selectors = [].concat(cssSelectors).filter(Boolean);
 
-    await client.send('Network.setUserAgentOverride', {
-      userAgent: await browser.userAgent(),
-      acceptLanguage: config.language,
-    });
+    let pdf = {};
+    let handled = null;
+
+    if (!selectors.length) { // CSS selectors are specified only for HTML content and omitted when fetching a PDF
+      ({ pdf, handled } = setupPdfInterception(client));
+    }
 
-    if (browser.proxyCredentials?.username && browser.proxyCredentials?.password) {
-      await page.authenticate(browser.proxyCredentials);
+    let response;
+    let navigationAborted = false;
+
+    try {
+      response = await page.goto(url, { waitUntil: 'load' }); // Using `load` instead of `networkidle0` as it's more reliable and faster. The 'load' event fires when the page and all its resources (stylesheets, scripts, images) have finished loading. `networkidle0` can be problematic as it waits for 500ms of network inactivity, which may never occur on dynamic pages and then triggers a navigation timeout.
+    } catch (error) {
+      if (error.message.includes('net::ERR_ABORTED')) {
+        // Chrome may sometimes abort navigation for files such as PDFs.
+        // Do not throw for now; wait for the PDF interception handler to finish processing the response.
+        navigationAborted = true;
+      } else {
+        throw error;
+      }
     }
 
-    response = await page.goto(url, { waitUntil: 'load' }); // Using `load` instead of `networkidle0` as it's more reliable and faster. The 'load' event fires when the page and all its resources (stylesheets, scripts, images) have finished loading. `networkidle0` can be problematic as it waits for 500ms of network inactivity, which may never occur on dynamic pages and then triggers a navigation timeout.
+    // PDF interception handling
+    if (handled) {
+      await handled; // Wait for the interception callback to finish processing the response
+
+      if (pdf.content) {
+        return {
+          mimeType: 'application/pdf',
+          content: pdf.content,
+        };
+      }
+
+      if (pdf.status) { // Status captured by CDP interception
+        throw new Error(`Received HTTP code ${pdf.status} when trying to fetch '${url}'`);
+      }
+    }
+
+    if (navigationAborted) {
+      throw new Error(`Navigation aborted when trying to fetch '${url}'`);
+    }
 
     if (!response) {
       throw new Error(`Response is empty when trying to fetch '${url}'`);
@@ -46,31 +73,11 @@ export default async function fetch(url, cssSelectors, config) {
 
     const statusCode = response.status();
 
-    if (statusCode < 200 || (statusCode >= 300 && statusCode !== 304)) {
+    if (!isValidHttpStatus(statusCode)) {
       throw new Error(`Received HTTP code ${statusCode} when trying to fetch '${url}'`);
     }
 
-    const waitForSelectorsPromises = selectors.filter(Boolean).map(selector =>
-      page.waitForFunction(
-        cssSelector => {
-          const element = document.querySelector(cssSelector); // eslint-disable-line no-undef
-
-          return element?.textContent.trim().length; // Ensures element exists and contains non-empty text, as an empty element may indicate content is still loading
-        },
-        { timeout: config.waitForElementsTimeout },
-        selector,
-      ));
-
-    // We expect all elements to be present on the page…
-    await Promise.all(waitForSelectorsPromises).catch(error => {
-      if (error.name == 'TimeoutError') {
-        // however, if they are not, this is not considered as an error since selectors may be out of date
-        // and the whole content of the page should still be returned.
-        return;
-      }
-
-      throw error;
-    });
+    await waitForSelectors(page, selectors, config.waitForElementsTimeout);
 
     return {
       mimeType: 'text/html',
@@ -80,17 +87,10 @@ export default async function fetch(url, cssSelectors, config) {
     if (error.name === 'TimeoutError') {
       throw new Error(`Timed out after ${config.navigationTimeout / 1000} seconds when trying to fetch '${url}'`);
     }
+
     throw new Error(error.message);
   } finally {
-    if (client) {
-      await client.detach();
-    }
-    if (page) {
-      await page.close();
-    }
-    if (context) {
-      await context.close(); // Close the isolated context to free resources and ensure complete cleanup
-    }
+    await cleanupPage(client, page, context);
   }
 }
 
@@ -151,3 +151,103 @@ export async function stopHeadlessBrowser() {
   await browser.close();
   browser = null;
 }
+
+function isValidHttpStatus(status) {
+  return (status >= 200 && status < 300) || status === 304;
+}
+
+async function configurePage(page, client, config) {
+  await page.setViewport({ width: 1920, height: 1080 }); // Realistic viewport to avoid detection based on default Puppeteer dimensions (800x600)
+  await page.setDefaultNavigationTimeout(config.navigationTimeout);
+  await page.setExtraHTTPHeaders({ 'Accept-Language': config.language });
+
+  // Use CDP to ensure browser language is set correctly (see https://zirkelc.dev/posts/puppeteer-language-experiment)
+  await client.send('Network.setUserAgentOverride', {
+    userAgent: await browser.userAgent(),
+    acceptLanguage: config.language,
+  });
+
+  if (browser.proxyCredentials?.username && browser.proxyCredentials?.password) {
+    await page.authenticate(browser.proxyCredentials);
+  }
+}
+
+function setupPdfInterception(client) {
+  const pdf = { content: null, status: null };
+  let onHandled;
+  const handled = new Promise(resolve => { onHandled = resolve; });
+
+  client.send('Fetch.enable', { patterns: [{ urlPattern: '*', requestStage: 'Response' }] }); // Intercept all responses before Chrome processes them, allowing to capture PDF content before it's handled by the PDF viewer
+
+  client.on('Fetch.requestPaused', async ({ requestId, resourceType, responseHeaders, responseStatusCode }) => {
+    try {
+      const contentType = responseHeaders?.find(header => header.name.toLowerCase() === 'content-type')?.value;
+
+      if (!contentType?.includes('application/pdf')) {
+        return;
+      }
+
+      pdf.status = responseStatusCode;
+
+      if (!isValidHttpStatus(responseStatusCode)) {
+        return;
+      }
+
+      try {
+        const { body, base64Encoded } = await client.send('Fetch.getResponseBody', { requestId });
+
+        pdf.content = Buffer.from(body, base64Encoded ? 'base64' : 'utf8');
+      } catch {
+        // Response body may be unavailable due to network error or connection interruption
+      }
+    } finally {
+      try {
+        await client.send('Fetch.continueResponse', { requestId });
+      } catch {
+        // Client may have been closed by cleanupPage() in fetch() while this async callback was still running
+      }
+
+      if (resourceType === 'Document') { // Signal that the main navigation request has been processed
+        onHandled();
+      }
+    }
+  });
+
+  return { pdf, handled };
+}
+
+async function waitForSelectors(page, selectors, timeout) {
+  const waitForSelectorsPromises = selectors.filter(Boolean).map(selector =>
+    page.waitForFunction(
+      cssSelector => {
+        const element = document.querySelector(cssSelector); // eslint-disable-line no-undef
+
+        return element?.textContent.trim().length; // Ensures element exists and has non-empty text
+      },
+      { timeout },
+      selector,
+    ));
+
+  // We expect all elements to be present on the page…
+  await Promise.all(waitForSelectorsPromises).catch(error => {
+    if (error.name == 'TimeoutError') {
+      // however, if they are not, this is not considered as an error since selectors may be out of date
+      // and the whole content of the page should still be returned.
+      return;
+    }
+
+    throw error;
+  });
+}
+
+async function cleanupPage(client, page, context) {
+  if (client) {
+    await client.detach().catch(() => {});
+  }
+  if (page) {
+    await page.close().catch(() => {});
+  }
+  if (context) {
+    await context.close().catch(() => {}); // Close the isolated context to free resources and ensure complete cleanup
+  }
+}

package/src/archivist/fetcher/fullDomFetcher.test.js

@@ -1,10 +1,15 @@
+import fs from 'fs';
 import http from 'http';
+import path from 'path';
+import { fileURLToPath } from 'url';
 
 import { expect, use } from 'chai';
 import chaiAsPromised from 'chai-as-promised';
 
 import fetch, { launchHeadlessBrowser, stopHeadlessBrowser } from './fullDomFetcher.js';
 
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
 const SERVER_PORT = 8977;
 
 use(chaiAsPromised);
@@ -16,6 +21,7 @@ describe('Full DOM Fetcher', function () {
   this.timeout(60000);
 
   let temporaryServer;
+  let expectedPDFContent;
 
   before(async () => {
     await launchHeadlessBrowser();
@@ -27,6 +33,10 @@ describe('Full DOM Fetcher', function () {
       if (request.url === '/delayed-content') {
         response.writeHead(200, { 'Content-Type': 'text/html' }).write(delayedContentHTML);
       }
+      if (request.url === '/terms.pdf') {
+        expectedPDFContent = fs.readFileSync(path.resolve(__dirname, '../../../test/fixtures/terms.pdf'));
+        response.writeHead(200, { 'Content-Type': 'application/pdf' }).write(expectedPDFContent);
+      }
 
       return response.end();
     }).listen(SERVER_PORT);
@@ -85,5 +95,27 @@ describe('Full DOM Fetcher', function () {
         await expect(fetch(url, ['.content'], { ...config, navigationTimeout: timeout })).to.be.rejectedWith(`Timed out after ${timeout / 1000} seconds when trying to fetch '${url}'`);
       });
     });
+
+    context('when URL targets a PDF file', () => {
+      let content;
+      let mimeType;
+      const pdfUrl = `http://127.0.0.1:${SERVER_PORT}/terms.pdf`;
+
+      before(async () => {
+        ({ content, mimeType } = await fetch(pdfUrl, [], config));
+      });
+
+      it('returns a buffer for PDF content', () => {
+        expect(content).to.be.an.instanceOf(Buffer);
+      });
+
+      it('returns the correct MIME type', () => {
+        expect(mimeType).to.equal('application/pdf');
+      });
+
+      it('returns the PDF file content', () => {
+        expect(content.equals(expectedPDFContent)).to.be.true;
+      });
+    });
   });
 });

package/src/collection-api/logger.js

@@ -3,7 +3,8 @@ import os from 'os';
 import config from 'config';
 import dotenv from 'dotenv';
 import winston from 'winston';
-import 'winston-mail';
+
+import MailTransportWithRetry from '../logger/mail-transport-with-retry.js';
 
 dotenv.config({ quiet: true });
 
@@ -12,14 +13,15 @@ const { combine, timestamp, printf, colorize } = winston.format;
 const transports = [new winston.transports.Console()];
 
 if (config.get('@opentermsarchive/engine.logger.sendMailOnError')) {
-  transports.push(new winston.transports.Mail({
+  transports.push(new MailTransportWithRetry({
     to: config.get('@opentermsarchive/engine.logger.sendMailOnError.to'),
     from: config.get('@opentermsarchive/engine.logger.sendMailOnError.from'),
     host: config.get('@opentermsarchive/engine.logger.smtp.host'),
+    port: config.get('@opentermsarchive/engine.logger.smtp.port'),
     username: config.get('@opentermsarchive/engine.logger.smtp.username'),
     password: process.env.OTA_ENGINE_SMTP_PASSWORD,
-    ssl: true,
-    timeout: 30 * 1000,
+    tls: true,
+    timeout: 60 * 1000,
     formatter: args => args[Object.getOwnPropertySymbols(args)[1]], // Returns the full error message, the same visible in the console. It is referenced in the argument object with a Symbol of which we do not have the reference but we know it is the second one.
     exitOnError: true,
     level: 'error',

package/src/logger/index.js

@@ -2,10 +2,10 @@ import os from 'os';
 
 import config from 'config';
 import winston from 'winston';
-import 'winston-mail';
 
 import { getCollection } from '../archivist/collection/index.js';
 
+import MailTransportWithRetry from './mail-transport-with-retry.js';
 import { formatDuration } from './utils.js';
 
 const { combine, timestamp, printf, colorize } = winston.format;
@@ -57,10 +57,11 @@ if (config.get('@opentermsarchive/engine.logger.sendMailOnError')) {
       to: config.get('@opentermsarchive/engine.logger.sendMailOnError.to'),
       from: config.get('@opentermsarchive/engine.logger.sendMailOnError.from'),
       host: config.get('@opentermsarchive/engine.logger.smtp.host'),
+      port: config.get('@opentermsarchive/engine.logger.smtp.port'),
       username: config.get('@opentermsarchive/engine.logger.smtp.username'),
       password: process.env.OTA_ENGINE_SMTP_PASSWORD,
-      ssl: true,
-      timeout: 30 * 1000,
+      tls: true,
+      timeout: 60 * 1000,
       html: false,
       formatter({ message, level }) {
         const isError = level.includes('error');
@@ -141,14 +142,14 @@ if (config.get('@opentermsarchive/engine.logger.sendMailOnError')) {
       },
     };
 
-    transports.push(new winston.transports.Mail({
+    transports.push(new MailTransportWithRetry({
       ...mailerOptions,
       level: 'error',
       subject: `Server error on ${collection.id} collection`,
     }));
 
     if (config.get('@opentermsarchive/engine.logger.sendMailOnError.sendWarnings')) {
-      transports.push(new winston.transports.Mail({
+      transports.push(new MailTransportWithRetry({
         ...mailerOptions,
         level: 'warn',
         subject: `Inaccessible content on ${collection.id} collection`,

package/src/logger/mail-transport-with-retry.js

@@ -0,0 +1,46 @@
+import { once } from 'node:events';
+
+import async from 'async';
+import winston from 'winston';
+
+import 'winston-mail';
+
+export const RETRY_DELAYS = [ 5000, 20000, 60000 ];
+
+const RETRY_OPTIONS = {
+  times: RETRY_DELAYS.length + 1,
+  interval: retryCount => RETRY_DELAYS[retryCount - 1] || RETRY_DELAYS.at(-1),
+  errorFilter: error => {
+    console.warn(`SMTP mail sending failed: ${error.message}; retrying…`);
+
+    return true;
+  },
+};
+
+class MailTransportWithRetry extends winston.Transport {
+  constructor(options) {
+    super(options);
+    this.mailTransport = new winston.transports.Mail(options);
+  }
+
+  async log(info, callback) {
+    try {
+      await async.retry(RETRY_OPTIONS, async () => {
+        const result = Promise.race([
+          once(this.mailTransport, 'logged'),
+          once(this.mailTransport, 'error').then(([err]) => { throw err; }),
+        ]);
+
+        this.mailTransport.log(info, () => {});
+
+        return result;
+      });
+    } catch (error) {
+      console.warn(`SMTP mail sending failed after ${RETRY_OPTIONS.times} attempts: ${error.message}`);
+      this.emit('error', error);
+    }
+    callback();
+  }
+}
+
+export default MailTransportWithRetry;

package/src/logger/mail-transport-with-retry.test.js

@@ -0,0 +1,192 @@
+import { EventEmitter } from 'node:events';
+
+import { expect } from 'chai';
+import sinon from 'sinon';
+import winston from 'winston';
+
+import MailTransportWithRetry, { RETRY_DELAYS } from './mail-transport-with-retry.js';
+
+class MockMailTransport extends EventEmitter {
+  log() {} // eslint-disable-line
+}
+
+describe('MailTransportWithRetry', () => {
+  let clock;
+  let mockMailTransport;
+  let transport;
+  let consoleWarnStub;
+
+  beforeEach(() => {
+    clock = sinon.useFakeTimers();
+    mockMailTransport = new MockMailTransport();
+    sinon.stub(winston.transports, 'Mail').returns(mockMailTransport);
+    consoleWarnStub = sinon.stub(console, 'warn');
+    transport = new MailTransportWithRetry({ to: 'test@example.com' });
+  });
+
+  afterEach(() => {
+    sinon.restore();
+    clock.restore();
+  });
+
+  describe('#log', () => {
+    context('when email is sent successfully on first attempt', () => {
+      it('calls callback without error', async () => {
+        const callback = sinon.spy();
+        const logPromise = transport.log({ message: 'test' }, callback);
+
+        mockMailTransport.emit('logged');
+        await logPromise;
+
+        expect(callback).to.have.been.calledOnce;
+      });
+
+      it('does not emit error event', async () => {
+        const errorHandler = sinon.spy();
+
+        transport.on('error', errorHandler);
+
+        const logPromise = transport.log({ message: 'test' }, () => {});
+
+        mockMailTransport.emit('logged');
+        await logPromise;
+
+        expect(errorHandler).not.to.have.been.called;
+      });
+
+      it('does not log any warning', async () => {
+        const logPromise = transport.log({ message: 'test' }, () => {});
+
+        mockMailTransport.emit('logged');
+        await logPromise;
+
+        expect(consoleWarnStub).not.to.have.been.called;
+      });
+    });
+
+    context('when email fails then succeeds on retry', () => {
+      it('retries and eventually succeeds', async () => {
+        const callback = sinon.spy();
+        const logSpy = sinon.spy(mockMailTransport, 'log');
+        const logPromise = transport.log({ message: 'test' }, callback);
+
+        mockMailTransport.emit('error', new Error('SMTP timeout'));
+
+        await clock.tickAsync(RETRY_DELAYS[0]);
+
+        mockMailTransport.emit('logged');
+        await logPromise;
+
+        expect(logSpy).to.have.been.calledTwice;
+        expect(callback).to.have.been.calledOnce;
+      });
+
+      it('logs warning for failed attempt', async () => {
+        const logPromise = transport.log({ message: 'test' }, () => {});
+
+        mockMailTransport.emit('error', new Error('SMTP timeout'));
+
+        await clock.tickAsync(RETRY_DELAYS[0]);
+
+        mockMailTransport.emit('logged');
+        await logPromise;
+
+        expect(consoleWarnStub).to.have.been.calledOnce;
+        expect(consoleWarnStub.firstCall.args[0]).to.include('SMTP mail sending failed');
+        expect(consoleWarnStub.firstCall.args[0]).to.include('SMTP timeout');
+      });
+    });
+
+    context('when email fails after all retry attempts', () => {
+      it('emits error event after all retries are exhausted', async () => {
+        const errorHandler = sinon.spy();
+
+        transport.on('error', errorHandler);
+
+        const logPromise = transport.log({ message: 'test' }, () => {});
+
+        mockMailTransport.emit('error', new Error('SMTP timeout'));
+        await clock.tickAsync(RETRY_DELAYS[0]);
+
+        mockMailTransport.emit('error', new Error('SMTP timeout'));
+        await clock.tickAsync(RETRY_DELAYS[1]);
+
+        mockMailTransport.emit('error', new Error('SMTP timeout'));
+        await clock.tickAsync(RETRY_DELAYS[2]);
+
+        mockMailTransport.emit('error', new Error('SMTP timeout'));
+        await logPromise;
+
+        expect(errorHandler).to.have.been.calledOnce;
+        expect(errorHandler.firstCall.args[0].message).to.equal('SMTP timeout');
+      });
+
+      it('calls callback even after failure', async () => {
+        const callback = sinon.spy();
+
+        transport.on('error', () => {}); // Prevent unhandled error
+
+        const logPromise = transport.log({ message: 'test' }, callback);
+
+        mockMailTransport.emit('error', new Error('SMTP timeout'));
+        await clock.tickAsync(RETRY_DELAYS[0]);
+
+        mockMailTransport.emit('error', new Error('SMTP timeout'));
+        await clock.tickAsync(RETRY_DELAYS[1]);
+
+        mockMailTransport.emit('error', new Error('SMTP timeout'));
+        await clock.tickAsync(RETRY_DELAYS[2]);
+
+        mockMailTransport.emit('error', new Error('SMTP timeout'));
+        await logPromise;
+
+        expect(callback).to.have.been.calledOnce;
+      });
+
+      it('logs final failure warning', async () => {
+        transport.on('error', () => {}); // Prevent unhandled error
+
+        const logPromise = transport.log({ message: 'test' }, () => {});
+
+        mockMailTransport.emit('error', new Error('SMTP timeout'));
+        await clock.tickAsync(RETRY_DELAYS[0]);
+
+        mockMailTransport.emit('error', new Error('SMTP timeout'));
+        await clock.tickAsync(RETRY_DELAYS[1]);
+
+        mockMailTransport.emit('error', new Error('SMTP timeout'));
+        await clock.tickAsync(RETRY_DELAYS[2]);
+
+        mockMailTransport.emit('error', new Error('SMTP timeout'));
+        await logPromise;
+
+        expect(consoleWarnStub.lastCall.args[0]).to.include(`failed after ${RETRY_DELAYS.length + 1} attempts`);
+      });
+    });
+
+    context('when email succeeds after multiple failures', () => {
+      it('succeeds on third attempt', async () => {
+        const callback = sinon.spy();
+        const errorHandler = sinon.spy();
+        const logSpy = sinon.spy(mockMailTransport, 'log');
+
+        transport.on('error', errorHandler);
+
+        const logPromise = transport.log({ message: 'test' }, callback);
+
+        mockMailTransport.emit('error', new Error('SMTP timeout'));
+        await clock.tickAsync(RETRY_DELAYS[0]);
+
+        mockMailTransport.emit('error', new Error('Connection refused'));
+        await clock.tickAsync(RETRY_DELAYS[1]);
+
+        mockMailTransport.emit('logged');
+        await logPromise;
+
+        expect(logSpy).to.have.been.calledThrice;
+        expect(callback).to.have.been.calledOnce;
+        expect(errorHandler).not.to.have.been.called;
+      });
+    });
+  });
+});