@opentermsarchive/engine 15.3.0 → 16.0.0

package/scripts/dataset/logger/index.test.js

@@ -0,0 +1,32 @@
+import { expect, use } from 'chai';
+import sinon from 'sinon';
+import sinonChai from 'sinon-chai';
+
+import logger, { createModuleLogger } from './index.js';
+
+use(sinonChai);
+
+describe('Dataset logger', () => {
+  describe('#createModuleLogger', () => {
+    const MODULE_NAME = 'test-module';
+    let moduleLogger;
+
+    before(() => {
+      moduleLogger = createModuleLogger(MODULE_NAME);
+    });
+
+    afterEach(() => {
+      sinon.restore();
+    });
+
+    [ 'debug', 'info', 'warn', 'error' ].forEach(level => {
+      it(`forwards ${level} messages to the logger with the module name`, () => {
+        const stub = sinon.stub(logger, level);
+
+        moduleLogger[level]('message');
+
+        expect(stub).to.have.been.calledOnceWithExactly('message', { module: MODULE_NAME });
+      });
+    });
+  });
+});

package/scripts/dataset/publish/datagouv/dataset.js

@@ -8,7 +8,6 @@ import { createModuleLogger } from '../../logger/index.js';
 
 const logger = createModuleLogger('datagouv');
 
-const DATASET_LICENSE = 'odc-odbl';
 const DEFAULT_RESOURCE_DESCRIPTION = 'See README.md inside the archive for dataset structure and usage information.';
 
 const routes = {
@@ -112,11 +111,11 @@ export async function createDataset({ apiBaseUrl, headers, organizationId, title
   return dataset;
 }
 
-export async function updateDatasetMetadata({ apiBaseUrl, headers, datasetId, title, description, stats, frequency }) {
+export async function updateDatasetMetadata({ apiBaseUrl, headers, datasetId, title, description, license, stats, frequency }) {
   const updatePayload = {
     title,
     description,
-    license: DATASET_LICENSE,
+    license,
     frequency,
   };
 

package/scripts/dataset/publish/datagouv/index.js

@@ -1,5 +1,6 @@
 import config from 'config';
 
+import * as license from '../../../../src/dataset/license.js';
 import * as readme from '../../assets/README.template.js';
 import { createModuleLogger } from '../../logger/index.js';
 
@@ -9,7 +10,6 @@ const logger = createModuleLogger('datagouv');
 
 const PRODUCTION_API_BASE_URL = 'https://www.data.gouv.fr/api/1';
 const DEMO_API_BASE_URL = 'https://demo.data.gouv.fr/api/1';
-const DATASET_LICENSE = 'odc-odbl';
 
 export default async function publish({ archivePath, stats }) {
   const { datasetId, organizationIdOrSlug, apiBaseUrl, headers, datasetTitle, frequency } = loadConfiguration();
@@ -19,7 +19,7 @@ export default async function publish({ archivePath, stats }) {
     ? await getDataset({ apiBaseUrl, headers, datasetId })
     : await ensureDatasetExists({ apiBaseUrl, headers, organizationIdOrSlug, datasetTitle, description, frequency });
 
-  await updateDatasetMetadata({ apiBaseUrl, headers, datasetId: dataset.id, title: datasetTitle, description, stats, frequency });
+  await updateDatasetMetadata({ apiBaseUrl, headers, datasetId: dataset.id, title: datasetTitle, description, license: license.DATAGOUV_ID, stats, frequency });
 
   const { resourceId, fileName } = await handleResourceUpload({ apiBaseUrl, headers, datasetId: dataset.id, dataset, archivePath });
 
@@ -73,7 +73,7 @@ async function ensureDatasetExists({ apiBaseUrl, headers, organizationIdOrSlug,
   let dataset = await findDatasetByTitle({ apiBaseUrl, headers, organizationId: organization.id, title: datasetTitle });
 
   if (!dataset) {
-    dataset = await createDataset({ apiBaseUrl, headers, organizationId: organization.id, title: datasetTitle, description, license: DATASET_LICENSE, frequency });
+    dataset = await createDataset({ apiBaseUrl, headers, organizationId: organization.id, title: datasetTitle, description, license: license.DATAGOUV_ID, frequency });
   }
 
   return dataset;

package/scripts/dataset/publish/gitlab/index.js

@@ -51,7 +51,8 @@ export default async function publish({
     projectId = null;
   }
 
-  const tagName = `${path.basename(archivePath, path.extname(archivePath))}`; // use archive filename as Git tag
+  const archiveFilename = path.basename(archivePath);
+  const tagName = path.basename(archiveFilename, path.extname(archiveFilename)); // use archive filename as Git tag
 
   try {
     let options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN);
@@ -88,7 +89,7 @@ export default async function publish({
     // restrict characters to the ones allowed by GitLab APIs
     const packageName = config.get('@opentermsarchive/engine.dataset.title').replace(/[^a-zA-Z0-9.\-_]/g, '-');
     const packageVersion = tagName.replace(/[^a-zA-Z0-9.\-_]/g, '-');
-    const packageFileName = archivePath.replace(/[^a-zA-Z0-9.\-_/]/g, '-');
+    const packageFileName = archiveFilename.replace(/[^a-zA-Z0-9.\-_]/g, '-');
 
     logger.debug(`packageName: ${packageName}, packageVersion: ${packageVersion} packageFileName: ${packageFileName}`);
 
@@ -108,9 +109,9 @@ export default async function publish({
     // Create the release and link the package
     const formData = new FormData();
 
-    formData.append('name', archivePath);
+    formData.append('name', archiveFilename);
     formData.append('url', publishedPackageUrl);
-    formData.append('file', fsApi.createReadStream(archivePath), { filename: path.basename(archivePath) });
+    formData.append('file', fsApi.createReadStream(archivePath), { filename: archiveFilename });
 
     options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN);
     options.method = 'POST';

package/scripts/dataset/publish/gitlab/index.test.js

@@ -0,0 +1,90 @@
+import fs from 'fs/promises';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+import { expect } from 'chai';
+import config from 'config';
+import nock from 'nock';
+
+import publish from './index.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const { origin: API_ORIGIN, pathname: API_PATH } = new URL(config.get('@opentermsarchive/engine.dataset.apiBaseURL'));
+const PROJECT_PATH = 'OpenTermsArchive/sandbox-versions';
+const PROJECT_ID = 1;
+const ARCHIVE_NAME = 'sandbox-2026-01-01';
+const ARCHIVE_FILENAME = `${ARCHIVE_NAME}.zip`;
+const TMP_PATH = path.resolve(__dirname, './tmp');
+const ARCHIVE_PATH = path.join(TMP_PATH, ARCHIVE_FILENAME);
+const DIRECT_ASSET_URL = 'https://gitlab.example.test/download/42';
+const STATS = {
+  servicesCount: 2,
+  firstVersionDate: new Date('2021-01-01T00:00:00Z'),
+  lastVersionDate: new Date('2022-01-06T00:00:00Z'),
+};
+
+describe('GitLab dataset publisher', () => {
+  describe('#publish', () => {
+    let packageUploadScope;
+    let assetLinkScope;
+    let result;
+    let previousToken;
+
+    before(async function () {
+      this.timeout(5000);
+      previousToken = process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN;
+      process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN = 'token';
+
+      await fs.mkdir(TMP_PATH, { recursive: true });
+      await fs.writeFile(ARCHIVE_PATH, 'archive content'); // Plain text keeps the multipart body inspectable: nock hands binary bodies to matchers as hex strings
+
+      nock(API_ORIGIN)
+        .get(`${API_PATH}/projects/${encodeURIComponent(PROJECT_PATH)}`)
+        .reply(200, { id: PROJECT_ID });
+
+      nock(API_ORIGIN)
+        .post(`${API_PATH}/projects/${PROJECT_ID}/releases`)
+        .reply(201, { commit: { id: 'sha' } });
+
+      packageUploadScope = nock(API_ORIGIN)
+        .put(`${API_PATH}/projects/${PROJECT_ID}/packages/generic/sandbox/${ARCHIVE_NAME}/${ARCHIVE_FILENAME}`)
+        .query({ status: 'default', select: 'package_file' })
+        .reply(201, { id: 42 });
+
+      assetLinkScope = nock(API_ORIGIN)
+        .post(`${API_PATH}/projects/${PROJECT_ID}/releases/${ARCHIVE_NAME}/assets/links`, body => body.includes(`name="name"\r\n\r\n${ARCHIVE_FILENAME}\r\n`))
+        .reply(201, { direct_asset_url: DIRECT_ASSET_URL });
+
+      result = await publish({
+        archivePath: ARCHIVE_PATH,
+        releaseDate: new Date('2026-01-01T08:30:00Z'),
+        stats: STATS,
+      });
+    });
+
+    after(async () => {
+      nock.cleanAll();
+
+      if (previousToken === undefined) {
+        delete process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN;
+      } else {
+        process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN = previousToken;
+      }
+
+      await fs.rm(TMP_PATH, { recursive: true, force: true });
+    });
+
+    it('uploads the package under the archive file name', () => {
+      expect(packageUploadScope.isDone()).to.be.true;
+    });
+
+    it('links the release asset under the archive file name', () => {
+      expect(assetLinkScope.isDone()).to.be.true;
+    });
+
+    it('returns the direct asset URL', () => {
+      expect(result).to.equal(DIRECT_ASSET_URL);
+    });
+  });
+});

package/scripts/declarations/validate/index.mocha.js

@@ -97,7 +97,7 @@ export default async options => {
           }
         });
 
-        if (!schemaOnly && service) {
+        if (service) {
           service.getTermsTypes()
             .filter(termsType => {
               if (!service.terms[termsType]?.latest) { // If this terms type has been deleted and there is only a historical record for it, but no current valid declaration
@@ -126,6 +126,10 @@ export default async options => {
                   });
                 }
 
+                if (schemaOnly) {
+                  return; // Remaining checks require fetching the source documents
+                }
+
                 terms.sourceDocuments.forEach(sourceDocument => {
                   let filteredContent;
 

package/src/collection-api/middlewares/errors.js

@@ -2,5 +2,10 @@ import logger from '../logger.js';
 
 export default function errorsMiddleware(err, req, res, next) {
   logger.error(err.stack);
+
+  if (res.headersSent) { // A response that already started cannot be restarted with a JSON error; defer to Express' default handler, which ends the request instead of leaving the client waiting
+    return next(err);
+  }
+
   res.status(500).json({ error: 'Internal Server Error' }); // Never echo internal error details: they can expose server internals such as filesystem paths
 }

package/src/collection-api/middlewares/errors.test.js

@@ -0,0 +1,56 @@
+import { expect, use } from 'chai';
+import sinon from 'sinon';
+import sinonChai from 'sinon-chai';
+
+import errorsMiddleware from './errors.js';
+
+use(sinonChai);
+
+describe('Errors middleware', () => {
+  const error = new Error('Boom');
+
+  let res;
+  let next;
+
+  beforeEach(() => {
+    res = {
+      status: sinon.stub().returnsThis(),
+      json: sinon.stub(),
+    };
+    next = sinon.stub();
+  });
+
+  context('when no response has been sent yet', () => {
+    beforeEach(() => {
+      res.headersSent = false;
+      errorsMiddleware(error, {}, res, next);
+    });
+
+    it('responds with 500 status code', () => {
+      expect(res.status).to.have.been.calledOnceWith(500);
+    });
+
+    it('responds with a generic JSON error', () => {
+      expect(res.json).to.have.been.calledOnceWith({ error: 'Internal Server Error' });
+    });
+
+    it('does not call next', () => {
+      expect(next).to.not.have.been.called;
+    });
+  });
+
+  context('when headers have already been sent', () => {
+    beforeEach(() => {
+      res.headersSent = true;
+      errorsMiddleware(error, {}, res, next);
+    });
+
+    it('forwards the error to the default Express handler instead of responding again', () => {
+      expect(next).to.have.been.calledOnceWith(error);
+    });
+
+    it('does not attempt to set a status code', () => {
+      expect(res.status).to.not.have.been.called;
+    });
+  });
+});

package/src/collection-api/routes/dataset.js

@@ -0,0 +1,269 @@
+import http from 'http';
+
+import express from 'express';
+
+import { buildAbsoluteBaseUrl } from '../utils/url.js';
+
+export const NO_DATASET_ERROR = 'No dataset has been generated yet';
+
+const TRANSFER_HEADERS = [ 'Accept-Ranges', 'Cache-Control', 'Content-Disposition', 'Content-Length', 'Content-Range', 'Content-Type', 'ETag', 'Last-Modified' ];
+const METADATA_PATH = '/dataset/latest';
+const DOWNLOAD_PATH = `${METADATA_PATH}/download`;
+
+function handleTransferError(error, res, next) {
+  if (!error || error.code === 'ECONNABORTED') {
+    return;
+  }
+
+  if (res.headersSent) { // The transfer started before failing: `send` destroyed the read stream but never ended the response, so it must be forwarded to end it instead of leaving the client waiting
+    return next(error);
+  }
+
+  TRANSFER_HEADERS.forEach(header => res.removeHeader(header)); // `send` has already described the archive on the response when it fails, whether it rejects the request or cannot read the file; those headers must not describe the JSON error instead
+
+  if (error.status === 404) { // The archive was replaced under another name between the metadata lookup and the transfer; the new metadata is already in place, so a retry succeeds
+    return res.status(404).json({ error: NO_DATASET_ERROR });
+  }
+
+  if (error.status < 500) { // Client errors raised by `send`, such as an unsatisfiable range or a failed precondition
+    return res.status(error.status).set(error.headers ?? {}).json({ error: http.STATUS_CODES[error.status] });
+  }
+
+  return next(error);
+}
+
+/**
+ * @param   {object}         datasetStorage The storage holding the latest dataset of the collection
+ * @returns {express.Router}                The router instance
+ * @swagger
+ * tags:
+ *   name: Dataset
+ *   description: Dataset API
+ * components:
+ *   schemas:
+ *     Dataset:
+ *       type: object
+ *       description: Metadata of the latest dataset generated on this instance. Counts and dates are frozen at generation time, unlike the live counters of the collection metadata.
+ *       additionalProperties: false
+ *       required:
+ *         - filename
+ *         - title
+ *         - license
+ *         - releaseDate
+ *         - firstVersionDate
+ *         - lastVersionDate
+ *         - servicesCount
+ *         - termsCount
+ *         - versionsCount
+ *         - size
+ *         - sha256
+ *         - downloadURL
+ *       properties:
+ *         filename:
+ *           type: string
+ *           description: Name of the archive file.
+ *           example: demo-2026-07-06.zip
+ *         title:
+ *           type: string
+ *           description: Dataset title.
+ *           example: demo
+ *         license:
+ *           type: string
+ *           description: SPDX identifier of the dataset license.
+ *           example: ODbL-1.0
+ *         releaseDate:
+ *           type: string
+ *           format: date-time
+ *           description: Datetime when the dataset was generated.
+ *           example: 2026-07-06T08:30:12Z
+ *         firstVersionDate:
+ *           type: string
+ *           format: date-time
+ *           description: Fetch date of the earliest version in the dataset.
+ *           example: 2022-01-01T12:00:00Z
+ *         lastVersionDate:
+ *           type: string
+ *           format: date-time
+ *           description: Fetch date of the latest version in the dataset.
+ *           example: 2026-07-05T23:12:00Z
+ *         servicesCount:
+ *           type: integer
+ *           description: Number of services in the dataset.
+ *           example: 262
+ *         termsCount:
+ *           type: integer
+ *           description: Number of distinct terms (service and terms type pairs) in the dataset.
+ *           example: 641
+ *         versionsCount:
+ *           type: integer
+ *           description: Total number of version files in the dataset.
+ *           example: 38294
+ *         size:
+ *           type: integer
+ *           description: Size of the archive in bytes.
+ *           example: 104857600
+ *         sha256:
+ *           type: string
+ *           description: SHA-256 checksum of the archive, allowing to verify its integrity and to detect changes without downloading it.
+ *           example: 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08
+ *         downloadURL:
+ *           type: string
+ *           format: uri
+ *           description: Absolute URL of the archive download endpoint.
+ *           example: https://example.org/collection-api/v1/dataset/latest/download
+ */
+export default function datasetRouter(datasetStorage) {
+  const router = express.Router();
+
+  /**
+   * @swagger
+   * /dataset/latest:
+   *   get:
+   *     summary: Get the metadata of the latest dataset.
+   *     tags: [Dataset]
+   *     produces:
+   *       - application/json
+   *     responses:
+   *       200:
+   *         description: A JSON object describing the latest dataset generated on this instance.
+   *         content:
+   *           application/json:
+   *             schema:
+   *               $ref: '#/components/schemas/Dataset'
+   *       404:
+   *         $ref: '#/components/responses/NotFoundError'
+   */
+  router.get(METADATA_PATH, async (req, res) => {
+    const metadata = await datasetStorage.findLatest();
+
+    if (!metadata) {
+      return res.status(404).json({ error: NO_DATASET_ERROR });
+    }
+
+    return res.json({
+      ...metadata,
+      downloadURL: `${buildAbsoluteBaseUrl(req)}${DOWNLOAD_PATH}`,
+    });
+  });
+
+  /**
+   * @swagger
+   * /dataset/latest/download:
+   *   get:
+   *     summary: Download the latest dataset archive.
+   *     description: Streams the ZIP archive of the latest dataset. Supports conditional requests through the `If-None-Match` and `If-Modified-Since` headers, and resumable downloads through `Range` requests.
+   *     tags: [Dataset]
+   *     parameters:
+   *       - in: header
+   *         name: If-None-Match
+   *         description: ETag of an archive already held by the client, to receive `304 Not Modified` when it is still the latest one.
+   *         schema:
+   *           type: string
+   *       - in: header
+   *         name: If-Modified-Since
+   *         description: Date of an archive already held by the client, to receive `304 Not Modified` when no dataset was released since.
+   *         schema:
+   *           type: string
+   *       - in: header
+   *         name: If-Match
+   *         description: ETag the client expects the archive to still match, to receive `412 Precondition Failed` when it changed.
+   *         schema:
+   *           type: string
+   *       - in: header
+   *         name: If-Unmodified-Since
+   *         description: Date the client expects the archive to still match, to receive `412 Precondition Failed` when a newer dataset was released since.
+   *         schema:
+   *           type: string
+   *       - in: header
+   *         name: Range
+   *         description: Byte range to resume an interrupted download.
+   *         schema:
+   *           type: string
+   *           example: bytes=1048576-
+   *       - in: header
+   *         name: If-Range
+   *         description: ETag or date the requested `Range` applies to, to fall back to a full `200` response with the whole archive when it no longer matches.
+   *         schema:
+   *           type: string
+   *     responses:
+   *       200:
+   *         description: The dataset archive.
+   *         headers:
+   *           Content-Disposition:
+   *             description: Attachment disposition carrying the archive file name.
+   *             schema:
+   *               type: string
+   *           Content-Length:
+   *             description: Size of the archive in bytes.
+   *             schema:
+   *               type: integer
+   *           ETag:
+   *             description: Quoted SHA-256 checksum of the archive.
+   *             schema:
+   *               type: string
+   *           Last-Modified:
+   *             description: Release date of the dataset.
+   *             schema:
+   *               type: string
+   *           Accept-Ranges:
+   *             description: Always `bytes`.
+   *             schema:
+   *               type: string
+   *         content:
+   *           application/zip:
+   *             schema:
+   *               type: string
+   *               format: binary
+   *       206:
+   *         description: The requested byte range of the dataset archive.
+   *         headers:
+   *           Content-Range:
+   *             schema:
+   *               type: string
+   *         content:
+   *           application/zip:
+   *             schema:
+   *               type: string
+   *               format: binary
+   *       304:
+   *         description: The client already holds the latest dataset archive.
+   *       404:
+   *         $ref: '#/components/responses/NotFoundError'
+   *       412:
+   *         description: The archive changed since the version referenced by `If-Match` or `If-Unmodified-Since`.
+   *         content:
+   *           application/json:
+   *             schema:
+   *               $ref: '#/components/schemas/ErrorResponse'
+   *       416:
+   *         description: The requested byte range cannot be satisfied.
+   *         headers:
+   *           Content-Range:
+   *             schema:
+   *               type: string
+   *         content:
+   *           application/json:
+   *             schema:
+   *               $ref: '#/components/schemas/ErrorResponse'
+   */
+  router.get(DOWNLOAD_PATH, async (req, res, next) => {
+    const metadata = await datasetStorage.findLatest();
+
+    if (!metadata) {
+      return res.status(404).json({ error: NO_DATASET_ERROR });
+    }
+
+    const options = {
+      root: datasetStorage.path, // Confines the served file to the storage directory and keeps `send` from refusing paths that contain dot-directories
+      dotfiles: 'allow', // Archive names are chosen by operators through `--file`; a name starting with a dot, such as `.weekly`, must still be served
+      headers: { // Applied by `send` once the archive is found and kept over its own defaults, so that the validators describe the dataset rather than the file on disk
+        ETag: `"${metadata.sha256}"`,
+        'Last-Modified': new Date(metadata.releaseDate).toUTCString(),
+      },
+    };
+
+    return res.download(metadata.filename, metadata.filename, options, error => handleTransferError(error, res, next));
+  });
+
+  return router;
+}