@opentermsarchive/engine 15.3.1 → 16.0.1

package/scripts/dataset/logger/index.test.js

@@ -0,0 +1,32 @@
+import { expect, use } from 'chai';
+import sinon from 'sinon';
+import sinonChai from 'sinon-chai';
+
+import logger, { createModuleLogger } from './index.js';
+
+use(sinonChai);
+
+describe('Dataset logger', () => {
+  describe('#createModuleLogger', () => {
+    const MODULE_NAME = 'test-module';
+    let moduleLogger;
+
+    before(() => {
+      moduleLogger = createModuleLogger(MODULE_NAME);
+    });
+
+    afterEach(() => {
+      sinon.restore();
+    });
+
+    [ 'debug', 'info', 'warn', 'error' ].forEach(level => {
+      it(`forwards ${level} messages to the logger with the module name`, () => {
+        const stub = sinon.stub(logger, level);
+
+        moduleLogger[level]('message');
+
+        expect(stub).to.have.been.calledOnceWithExactly('message', { module: MODULE_NAME });
+      });
+    });
+  });
+});

package/scripts/dataset/publish/datagouv/dataset.js

@@ -8,7 +8,6 @@ import { createModuleLogger } from '../../logger/index.js';
 
 const logger = createModuleLogger('datagouv');
 
-const DATASET_LICENSE = 'odc-odbl';
 const DEFAULT_RESOURCE_DESCRIPTION = 'See README.md inside the archive for dataset structure and usage information.';
 
 const routes = {
@@ -112,11 +111,11 @@ export async function createDataset({ apiBaseUrl, headers, organizationId, title
   return dataset;
 }
 
-export async function updateDatasetMetadata({ apiBaseUrl, headers, datasetId, title, description, stats, frequency }) {
+export async function updateDatasetMetadata({ apiBaseUrl, headers, datasetId, title, description, license, stats, frequency }) {
   const updatePayload = {
     title,
     description,
-    license: DATASET_LICENSE,
+    license,
     frequency,
   };
 

package/scripts/dataset/publish/datagouv/index.js

@@ -1,5 +1,6 @@
 import config from 'config';
 
+import * as license from '../../../../src/dataset/license.js';
 import * as readme from '../../assets/README.template.js';
 import { createModuleLogger } from '../../logger/index.js';
 
@@ -9,7 +10,6 @@ const logger = createModuleLogger('datagouv');
 
 const PRODUCTION_API_BASE_URL = 'https://www.data.gouv.fr/api/1';
 const DEMO_API_BASE_URL = 'https://demo.data.gouv.fr/api/1';
-const DATASET_LICENSE = 'odc-odbl';
 
 export default async function publish({ archivePath, stats }) {
   const { datasetId, organizationIdOrSlug, apiBaseUrl, headers, datasetTitle, frequency } = loadConfiguration();
@@ -19,7 +19,7 @@ export default async function publish({ archivePath, stats }) {
     ? await getDataset({ apiBaseUrl, headers, datasetId })
     : await ensureDatasetExists({ apiBaseUrl, headers, organizationIdOrSlug, datasetTitle, description, frequency });
 
-  await updateDatasetMetadata({ apiBaseUrl, headers, datasetId: dataset.id, title: datasetTitle, description, stats, frequency });
+  await updateDatasetMetadata({ apiBaseUrl, headers, datasetId: dataset.id, title: datasetTitle, description, license: license.DATAGOUV_ID, stats, frequency });
 
   const { resourceId, fileName } = await handleResourceUpload({ apiBaseUrl, headers, datasetId: dataset.id, dataset, archivePath });
 
@@ -73,7 +73,7 @@ async function ensureDatasetExists({ apiBaseUrl, headers, organizationIdOrSlug,
   let dataset = await findDatasetByTitle({ apiBaseUrl, headers, organizationId: organization.id, title: datasetTitle });
 
   if (!dataset) {
-    dataset = await createDataset({ apiBaseUrl, headers, organizationId: organization.id, title: datasetTitle, description, license: DATASET_LICENSE, frequency });
+    dataset = await createDataset({ apiBaseUrl, headers, organizationId: organization.id, title: datasetTitle, description, license: license.DATAGOUV_ID, frequency });
   }
 
   return dataset;

package/scripts/dataset/publish/gitlab/index.js

@@ -51,7 +51,8 @@ export default async function publish({
     projectId = null;
   }
 
-  const tagName = `${path.basename(archivePath, path.extname(archivePath))}`; // use archive filename as Git tag
+  const archiveFilename = path.basename(archivePath);
+  const tagName = path.basename(archiveFilename, path.extname(archiveFilename)); // use archive filename as Git tag
 
   try {
     let options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN);
@@ -88,7 +89,7 @@ export default async function publish({
     // restrict characters to the ones allowed by GitLab APIs
     const packageName = config.get('@opentermsarchive/engine.dataset.title').replace(/[^a-zA-Z0-9.\-_]/g, '-');
     const packageVersion = tagName.replace(/[^a-zA-Z0-9.\-_]/g, '-');
-    const packageFileName = archivePath.replace(/[^a-zA-Z0-9.\-_/]/g, '-');
+    const packageFileName = archiveFilename.replace(/[^a-zA-Z0-9.\-_]/g, '-');
 
     logger.debug(`packageName: ${packageName}, packageVersion: ${packageVersion} packageFileName: ${packageFileName}`);
 
@@ -108,9 +109,9 @@ export default async function publish({
     // Create the release and link the package
     const formData = new FormData();
 
-    formData.append('name', archivePath);
+    formData.append('name', archiveFilename);
     formData.append('url', publishedPackageUrl);
-    formData.append('file', fsApi.createReadStream(archivePath), { filename: path.basename(archivePath) });
+    formData.append('file', fsApi.createReadStream(archivePath), { filename: archiveFilename });
 
     options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN);
     options.method = 'POST';

package/scripts/dataset/publish/gitlab/index.test.js

@@ -0,0 +1,90 @@
+import fs from 'fs/promises';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+import { expect } from 'chai';
+import config from 'config';
+import nock from 'nock';
+
+import publish from './index.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const { origin: API_ORIGIN, pathname: API_PATH } = new URL(config.get('@opentermsarchive/engine.dataset.apiBaseURL'));
+const PROJECT_PATH = 'OpenTermsArchive/sandbox-versions';
+const PROJECT_ID = 1;
+const ARCHIVE_NAME = 'sandbox-2026-01-01';
+const ARCHIVE_FILENAME = `${ARCHIVE_NAME}.zip`;
+const TMP_PATH = path.resolve(__dirname, './tmp');
+const ARCHIVE_PATH = path.join(TMP_PATH, ARCHIVE_FILENAME);
+const DIRECT_ASSET_URL = 'https://gitlab.example.test/download/42';
+const STATS = {
+  servicesCount: 2,
+  firstVersionDate: new Date('2021-01-01T00:00:00Z'),
+  lastVersionDate: new Date('2022-01-06T00:00:00Z'),
+};
+
+describe('GitLab dataset publisher', () => {
+  describe('#publish', () => {
+    let packageUploadScope;
+    let assetLinkScope;
+    let result;
+    let previousToken;
+
+    before(async function () {
+      this.timeout(5000);
+      previousToken = process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN;
+      process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN = 'token';
+
+      await fs.mkdir(TMP_PATH, { recursive: true });
+      await fs.writeFile(ARCHIVE_PATH, 'archive content'); // Plain text keeps the multipart body inspectable: nock hands binary bodies to matchers as hex strings
+
+      nock(API_ORIGIN)
+        .get(`${API_PATH}/projects/${encodeURIComponent(PROJECT_PATH)}`)
+        .reply(200, { id: PROJECT_ID });
+
+      nock(API_ORIGIN)
+        .post(`${API_PATH}/projects/${PROJECT_ID}/releases`)
+        .reply(201, { commit: { id: 'sha' } });
+
+      packageUploadScope = nock(API_ORIGIN)
+        .put(`${API_PATH}/projects/${PROJECT_ID}/packages/generic/sandbox/${ARCHIVE_NAME}/${ARCHIVE_FILENAME}`)
+        .query({ status: 'default', select: 'package_file' })
+        .reply(201, { id: 42 });
+
+      assetLinkScope = nock(API_ORIGIN)
+        .post(`${API_PATH}/projects/${PROJECT_ID}/releases/${ARCHIVE_NAME}/assets/links`, body => body.includes(`name="name"\r\n\r\n${ARCHIVE_FILENAME}\r\n`))
+        .reply(201, { direct_asset_url: DIRECT_ASSET_URL });
+
+      result = await publish({
+        archivePath: ARCHIVE_PATH,
+        releaseDate: new Date('2026-01-01T08:30:00Z'),
+        stats: STATS,
+      });
+    });
+
+    after(async () => {
+      nock.cleanAll();
+
+      if (previousToken === undefined) {
+        delete process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN;
+      } else {
+        process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN = previousToken;
+      }
+
+      await fs.rm(TMP_PATH, { recursive: true, force: true });
+    });
+
+    it('uploads the package under the archive file name', () => {
+      expect(packageUploadScope.isDone()).to.be.true;
+    });
+
+    it('links the release asset under the archive file name', () => {
+      expect(assetLinkScope.isDone()).to.be.true;
+    });
+
+    it('returns the direct asset URL', () => {
+      expect(result).to.equal(DIRECT_ASSET_URL);
+    });
+  });
+});

package/src/archivist/fetcher/fullDomFetcher.js

@@ -1,4 +1,5 @@
-import puppeteer from 'puppeteer-extra';
+import puppeteer from 'puppeteer';
+import { addExtra } from 'puppeteer-extra';
 import stealthPlugin from 'puppeteer-extra-plugin-stealth';
 import navigatorLanguages from 'puppeteer-extra-plugin-stealth/evasions/navigator.languages/index.js';
 import userAgentOverride from 'puppeteer-extra-plugin-stealth/evasions/user-agent-override/index.js';
@@ -39,7 +40,7 @@ export default async function fetch(url, cssSelectors, config) {
     page = await context.newPage();
     client = await page.createCDPSession();
 
-    await configurePage(page, config);
+    await configurePage(page, client, config);
 
     const selectors = [].concat(cssSelectors).filter(Boolean);
 
@@ -47,7 +48,7 @@ export default async function fetch(url, cssSelectors, config) {
     let handled = null;
 
     if (!selectors.length) { // CSS selectors are specified only for HTML content and omitted when fetching a PDF
-      ({ pdf, handled } = setupPdfInterception(client));
+      ({ pdf, handled } = await setupPdfInterception(client));
     }
 
     let response;
@@ -125,13 +126,13 @@ export async function launchHeadlessBrowser(language) {
   }
 
   const { locale, languages } = parseLanguage(language);
+  const puppeteerExtra = addExtra(puppeteer); // Fresh instance for each launch, as plugins registered on the shared instance accumulate across launches and their hooks would run once per copy on every page
   const stealth = stealthPlugin();
 
   stealth.enabledEvasions.delete('user-agent-override');
   stealth.enabledEvasions.delete('navigator.languages');
-  puppeteer.use(stealth);
-  puppeteer.use(userAgentOverride({ locale }));
-  puppeteer.use(navigatorLanguages({ languages }));
+  puppeteerExtra.use(stealth);
+  puppeteerExtra.use(navigatorLanguages({ languages }));
 
   const options = {
     args: [],
@@ -156,12 +157,21 @@ export async function launchHeadlessBrowser(language) {
     options.args.push('--disable-setuid-sandbox');
   }
 
-  browser = await puppeteer.launch(options);
+  const launchedBrowser = await puppeteerExtra.launch(options);
+
+  try {
+    launchedBrowser.userAgentOverride = await captureUserAgentOverride(launchedBrowser, locale); // Computed once since the user agent is the same for every page of the browser
+  } catch (error) {
+    await launchedBrowser.close().catch(() => {});
+    throw error;
+  }
 
   if (proxyCredentials) {
-    browser.proxyCredentials = proxyCredentials;
+    launchedBrowser.proxyCredentials = proxyCredentials;
   }
 
+  browser = launchedBrowser; // Set only once fully configured, so that no caller gets a browser without its user agent override
+
   return browser;
 }
 
@@ -184,22 +194,44 @@ function isValidHttpStatus(status) {
   return (status >= 200 && status < 300) || status === 304;
 }
 
-async function configurePage(page, config) {
+async function configurePage(page, client, config) {
   await page.setViewport({ width: 1920, height: 1080 }); // Realistic viewport to avoid detection based on default Puppeteer dimensions (800x600)
   await page.setDefaultNavigationTimeout(config.navigationTimeout);
+  await client.send('Network.setUserAgentOverride', browser.userAgentOverride);
 
   if (browser.proxyCredentials?.username && browser.proxyCredentials?.password) {
     await page.authenticate(browser.proxyCredentials);
   }
 }
 
-function setupPdfInterception(client) {
+async function captureUserAgentOverride(browser, locale) { // The stealth evasion sends the override without awaiting it, so a failure would crash the process on an unhandled rejection; running it once on a stand-in page records the override for the fetcher to send itself
+  const evasion = userAgentOverride({ locale });
+  let override;
+
+  await evasion.beforeLaunch({ headless: true }); // Otherwise the evasion leaves Accept-Language to browser preferences
+  await evasion.onPageCreated({ // The stand-in page exposes only what the evasion reads
+    browser: () => browser,
+    _client: () => ({
+      send: (method, params) => {
+        if (method === 'Network.setUserAgentOverride') {
+          override = params;
+        }
+      },
+    }),
+  });
+
+  if (!override?.userAgent) {
+    throw new Error('Could not capture the user agent override from the stealth user-agent-override evasion');
+  }
+
+  return override;
+}
+
+async function setupPdfInterception(client) {
   const pdf = { content: null, status: null };
   let onHandled;
   const handled = new Promise(resolve => { onHandled = resolve; });
 
-  client.send('Fetch.enable', { patterns: [{ urlPattern: '*', requestStage: 'Response' }] }); // Intercept all responses before Chrome processes them, allowing to capture PDF content before it's handled by the PDF viewer
-
   client.on('Fetch.requestPaused', async ({ requestId, resourceType, responseHeaders, responseStatusCode }) => {
     try {
       const contentType = responseHeaders?.find(header => header.name.toLowerCase() === 'content-type')?.value;
@@ -234,6 +266,8 @@ function setupPdfInterception(client) {
     }
   });
 
+  await client.send('Fetch.enable', { patterns: [{ urlPattern: '*', requestStage: 'Response' }] }); // Intercept all responses before Chrome processes them, allowing to capture PDF content before it is handled by the PDF viewer; enabled once the listener is registered so that no paused request is missed
+
   return { pdf, handled };
 }
 

package/src/archivist/fetcher/fullDomFetcher.test.js

@@ -17,7 +17,7 @@ use(chaiAsPromised);
 const dynamicHTML = '<!DOCTYPE html><html><head><title>Dynamic Page</title><script>setTimeout(() => { document.body.innerHTML += "<div class=\'dynamic\'>Loaded</div>"; }, 100);</script></head><body></body></html>';
 const delayedContentHTML = '<!DOCTYPE html><html><head><title>Delayed Content</title><script>setTimeout(() => { document.querySelector(".content").textContent = "Final content"; }, 100);</script></head><body><div class="content"></div></body></html>';
 const langEchoHTML = '<!DOCTYPE html><html><body><script>document.body.setAttribute("data-language", navigator.language); document.body.setAttribute("data-languages", navigator.languages.join(","));</script></body></html>';
-const stealthProbeHTML = '<!DOCTYPE html><html><body><script>document.body.setAttribute("data-webdriver", String(navigator.webdriver)); document.body.setAttribute("data-user-agent", navigator.userAgent); document.body.setAttribute("data-plugin-count", String(navigator.plugins.length)); document.body.setAttribute("data-viewport-width", String(window.innerWidth)); document.body.setAttribute("data-viewport-height", String(window.innerHeight)); (() => { const canvas = document.createElement("canvas"); const gl = canvas.getContext("webgl") || canvas.getContext("experimental-webgl"); if (!gl) { document.body.setAttribute("data-webgl-vendor", "none"); return; } const ext = gl.getExtension("WEBGL_debug_renderer_info"); document.body.setAttribute("data-webgl-vendor", ext ? gl.getParameter(ext.UNMASKED_VENDOR_WEBGL) : ""); document.body.setAttribute("data-webgl-renderer", ext ? gl.getParameter(ext.UNMASKED_RENDERER_WEBGL) : ""); })();</script></body></html>';
+const stealthProbeHTML = '<!DOCTYPE html><html><body><script>document.body.setAttribute("data-webdriver", String(navigator.webdriver)); document.body.setAttribute("data-user-agent", navigator.userAgent); document.body.setAttribute("data-platform", navigator.platform); document.body.setAttribute("data-ua-brands", navigator.userAgentData ? navigator.userAgentData.brands.map(entry => entry.brand).join(",") : ""); document.body.setAttribute("data-plugin-count", String(navigator.plugins.length)); document.body.setAttribute("data-viewport-width", String(window.innerWidth)); document.body.setAttribute("data-viewport-height", String(window.innerHeight)); (() => { const canvas = document.createElement("canvas"); const gl = canvas.getContext("webgl") || canvas.getContext("experimental-webgl"); if (!gl) { document.body.setAttribute("data-webgl-vendor", "none"); return; } const ext = gl.getExtension("WEBGL_debug_renderer_info"); document.body.setAttribute("data-webgl-vendor", ext ? gl.getParameter(ext.UNMASKED_VENDOR_WEBGL) : ""); document.body.setAttribute("data-webgl-renderer", ext ? gl.getParameter(ext.UNMASKED_RENDERER_WEBGL) : ""); })();</script></body></html>';
 
 describe('Full DOM Fetcher', function () {
   this.timeout(60000);
@@ -40,6 +40,11 @@ describe('Full DOM Fetcher', function () {
 
         response.writeHead(200, { 'Content-Type': 'text/html' }).write(`<!DOCTYPE html><html><body data-accept-language="${acceptLanguage}"></body></html>`);
       }
+      if (request.url === '/client-hints-header') {
+        const clientHintsBrands = request.headers['sec-ch-ua'] || '';
+
+        response.writeHead(200, { 'Content-Type': 'text/html' }).write(`<!DOCTYPE html><html><body>${clientHintsBrands}</body></html>`);
+      }
       if (request.url === '/lang-echo') {
         response.writeHead(200, { 'Content-Type': 'text/html' }).write(langEchoHTML);
       }
@@ -76,6 +81,16 @@ describe('Full DOM Fetcher', function () {
 
       expect(browser1).to.equal(browser2);
     });
+
+    it('registers plugin hooks once per launch', async () => {
+      const initialListenerCount = (await launchHeadlessBrowser('en')).listenerCount('targetcreated');
+
+      await stopHeadlessBrowser();
+
+      const relaunchedBrowser = await launchHeadlessBrowser('en');
+
+      expect(relaunchedBrowser.listenerCount('targetcreated')).to.equal(initialListenerCount);
+    });
   });
 
   describe('#fetch', () => {
@@ -136,6 +151,25 @@ describe('Full DOM Fetcher', function () {
         expect(content.equals(expectedPDFContent)).to.be.true;
       });
     });
+
+    context('when the user agent override cannot be applied', () => {
+      let browser;
+      let userAgentOverride;
+
+      before(async () => {
+        browser = await launchHeadlessBrowser('en');
+        ({ userAgentOverride } = browser);
+        browser.userAgentOverride = {}; // Rejected by Chrome since the user agent is mandatory
+      });
+
+      after(() => {
+        browser.userAgentOverride = userAgentOverride;
+      });
+
+      it('rejects with the protocol error', async () => {
+        await expect(fetch(`http://127.0.0.1:${SERVER_PORT}/dynamic`, [], config)).to.be.rejectedWith('Network.setUserAgentOverride');
+      });
+    });
   });
 
   describe('Language configuration', () => {
@@ -201,6 +235,23 @@ describe('Full DOM Fetcher', function () {
       expect(content).not.to.match(/HeadlessChrome/);
     });
 
+    it('removes HeadlessChrome from navigator.userAgentData brands', () => {
+      expect(content).to.match(/data-ua-brands="[^"]*Google Chrome[^"]*"/);
+      expect(content).not.to.match(/data-ua-brands="[^"]*Headless[^"]*"/);
+    });
+
+    it('sends the Sec-CH-UA header without HeadlessChrome', async () => {
+      const result = await fetch(`http://127.0.0.1:${SERVER_PORT}/client-hints-header`, [], config);
+
+      expect(result.content).to.match(/"Google Chrome";v="\d+"/);
+      expect(result.content).not.to.match(/Headless/);
+    });
+
+    it('does not expose a Linux host through navigator.platform', () => {
+      expect(content).to.match(/data-platform="[^"]+"/);
+      expect(content).not.to.match(/data-platform="Linux/);
+    });
+
     it('uses a realistic viewport instead of Puppeteer default', () => {
       expect(content).to.match(/data-viewport-width="1920"/);
       expect(content).to.match(/data-viewport-height="1080"/);

package/src/collection-api/middlewares/errors.js

@@ -2,5 +2,10 @@ import logger from '../logger.js';
 
 export default function errorsMiddleware(err, req, res, next) {
   logger.error(err.stack);
+
+  if (res.headersSent) { // A response that already started cannot be restarted with a JSON error; defer to Express' default handler, which ends the request instead of leaving the client waiting
+    return next(err);
+  }
+
   res.status(500).json({ error: 'Internal Server Error' }); // Never echo internal error details: they can expose server internals such as filesystem paths
 }

package/src/collection-api/middlewares/errors.test.js

@@ -0,0 +1,56 @@
+import { expect, use } from 'chai';
+import sinon from 'sinon';
+import sinonChai from 'sinon-chai';
+
+import errorsMiddleware from './errors.js';
+
+use(sinonChai);
+
+describe('Errors middleware', () => {
+  const error = new Error('Boom');
+
+  let res;
+  let next;
+
+  beforeEach(() => {
+    res = {
+      status: sinon.stub().returnsThis(),
+      json: sinon.stub(),
+    };
+    next = sinon.stub();
+  });
+
+  context('when no response has been sent yet', () => {
+    beforeEach(() => {
+      res.headersSent = false;
+      errorsMiddleware(error, {}, res, next);
+    });
+
+    it('responds with 500 status code', () => {
+      expect(res.status).to.have.been.calledOnceWith(500);
+    });
+
+    it('responds with a generic JSON error', () => {
+      expect(res.json).to.have.been.calledOnceWith({ error: 'Internal Server Error' });
+    });
+
+    it('does not call next', () => {
+      expect(next).to.not.have.been.called;
+    });
+  });
+
+  context('when headers have already been sent', () => {
+    beforeEach(() => {
+      res.headersSent = true;
+      errorsMiddleware(error, {}, res, next);
+    });
+
+    it('forwards the error to the default Express handler instead of responding again', () => {
+      expect(next).to.have.been.calledOnceWith(error);
+    });
+
+    it('does not attempt to set a status code', () => {
+      expect(res.status).to.not.have.been.called;
+    });
+  });
+});