@opentermsarchive/engine 15.3.1 → 16.0.1

package/src/collection-api/routes/dataset.js

@@ -0,0 +1,269 @@
+import http from 'http';
+
+import express from 'express';
+
+import { buildAbsoluteBaseUrl } from '../utils/url.js';
+
+export const NO_DATASET_ERROR = 'No dataset has been generated yet';
+
+const TRANSFER_HEADERS = [ 'Accept-Ranges', 'Cache-Control', 'Content-Disposition', 'Content-Length', 'Content-Range', 'Content-Type', 'ETag', 'Last-Modified' ];
+const METADATA_PATH = '/dataset/latest';
+const DOWNLOAD_PATH = `${METADATA_PATH}/download`;
+
+function handleTransferError(error, res, next) {
+  if (!error || error.code === 'ECONNABORTED') {
+    return;
+  }
+
+  if (res.headersSent) { // The transfer started before failing: `send` destroyed the read stream but never ended the response, so it must be forwarded to end it instead of leaving the client waiting
+    return next(error);
+  }
+
+  TRANSFER_HEADERS.forEach(header => res.removeHeader(header)); // `send` has already described the archive on the response when it fails, whether it rejects the request or cannot read the file; those headers must not describe the JSON error instead
+
+  if (error.status === 404) { // The archive was replaced under another name between the metadata lookup and the transfer; the new metadata is already in place, so a retry succeeds
+    return res.status(404).json({ error: NO_DATASET_ERROR });
+  }
+
+  if (error.status < 500) { // Client errors raised by `send`, such as an unsatisfiable range or a failed precondition
+    return res.status(error.status).set(error.headers ?? {}).json({ error: http.STATUS_CODES[error.status] });
+  }
+
+  return next(error);
+}
+
+/**
+ * @param   {object}         datasetStorage The storage holding the latest dataset of the collection
+ * @returns {express.Router}                The router instance
+ * @swagger
+ * tags:
+ *   name: Dataset
+ *   description: Dataset API
+ * components:
+ *   schemas:
+ *     Dataset:
+ *       type: object
+ *       description: Metadata of the latest dataset generated on this instance. Counts and dates are frozen at generation time, unlike the live counters of the collection metadata.
+ *       additionalProperties: false
+ *       required:
+ *         - filename
+ *         - title
+ *         - license
+ *         - releaseDate
+ *         - firstVersionDate
+ *         - lastVersionDate
+ *         - servicesCount
+ *         - termsCount
+ *         - versionsCount
+ *         - size
+ *         - sha256
+ *         - downloadURL
+ *       properties:
+ *         filename:
+ *           type: string
+ *           description: Name of the archive file.
+ *           example: demo-2026-07-06.zip
+ *         title:
+ *           type: string
+ *           description: Dataset title.
+ *           example: demo
+ *         license:
+ *           type: string
+ *           description: SPDX identifier of the dataset license.
+ *           example: ODbL-1.0
+ *         releaseDate:
+ *           type: string
+ *           format: date-time
+ *           description: Datetime when the dataset was generated.
+ *           example: 2026-07-06T08:30:12Z
+ *         firstVersionDate:
+ *           type: string
+ *           format: date-time
+ *           description: Fetch date of the earliest version in the dataset.
+ *           example: 2022-01-01T12:00:00Z
+ *         lastVersionDate:
+ *           type: string
+ *           format: date-time
+ *           description: Fetch date of the latest version in the dataset.
+ *           example: 2026-07-05T23:12:00Z
+ *         servicesCount:
+ *           type: integer
+ *           description: Number of services in the dataset.
+ *           example: 262
+ *         termsCount:
+ *           type: integer
+ *           description: Number of distinct terms (service and terms type pairs) in the dataset.
+ *           example: 641
+ *         versionsCount:
+ *           type: integer
+ *           description: Total number of version files in the dataset.
+ *           example: 38294
+ *         size:
+ *           type: integer
+ *           description: Size of the archive in bytes.
+ *           example: 104857600
+ *         sha256:
+ *           type: string
+ *           description: SHA-256 checksum of the archive, allowing to verify its integrity and to detect changes without downloading it.
+ *           example: 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08
+ *         downloadURL:
+ *           type: string
+ *           format: uri
+ *           description: Absolute URL of the archive download endpoint.
+ *           example: https://example.org/collection-api/v1/dataset/latest/download
+ */
+export default function datasetRouter(datasetStorage) {
+  const router = express.Router();
+
+  /**
+   * @swagger
+   * /dataset/latest:
+   *   get:
+   *     summary: Get the metadata of the latest dataset.
+   *     tags: [Dataset]
+   *     produces:
+   *       - application/json
+   *     responses:
+   *       200:
+   *         description: A JSON object describing the latest dataset generated on this instance.
+   *         content:
+   *           application/json:
+   *             schema:
+   *               $ref: '#/components/schemas/Dataset'
+   *       404:
+   *         $ref: '#/components/responses/NotFoundError'
+   */
+  router.get(METADATA_PATH, async (req, res) => {
+    const metadata = await datasetStorage.findLatest();
+
+    if (!metadata) {
+      return res.status(404).json({ error: NO_DATASET_ERROR });
+    }
+
+    return res.json({
+      ...metadata,
+      downloadURL: `${buildAbsoluteBaseUrl(req)}${DOWNLOAD_PATH}`,
+    });
+  });
+
+  /**
+   * @swagger
+   * /dataset/latest/download:
+   *   get:
+   *     summary: Download the latest dataset archive.
+   *     description: Streams the ZIP archive of the latest dataset. Supports conditional requests through the `If-None-Match` and `If-Modified-Since` headers, and resumable downloads through `Range` requests.
+   *     tags: [Dataset]
+   *     parameters:
+   *       - in: header
+   *         name: If-None-Match
+   *         description: ETag of an archive already held by the client, to receive `304 Not Modified` when it is still the latest one.
+   *         schema:
+   *           type: string
+   *       - in: header
+   *         name: If-Modified-Since
+   *         description: Date of an archive already held by the client, to receive `304 Not Modified` when no dataset was released since.
+   *         schema:
+   *           type: string
+   *       - in: header
+   *         name: If-Match
+   *         description: ETag the client expects the archive to still match, to receive `412 Precondition Failed` when it changed.
+   *         schema:
+   *           type: string
+   *       - in: header
+   *         name: If-Unmodified-Since
+   *         description: Date the client expects the archive to still match, to receive `412 Precondition Failed` when a newer dataset was released since.
+   *         schema:
+   *           type: string
+   *       - in: header
+   *         name: Range
+   *         description: Byte range to resume an interrupted download.
+   *         schema:
+   *           type: string
+   *           example: bytes=1048576-
+   *       - in: header
+   *         name: If-Range
+   *         description: ETag or date the requested `Range` applies to, to fall back to a full `200` response with the whole archive when it no longer matches.
+   *         schema:
+   *           type: string
+   *     responses:
+   *       200:
+   *         description: The dataset archive.
+   *         headers:
+   *           Content-Disposition:
+   *             description: Attachment disposition carrying the archive file name.
+   *             schema:
+   *               type: string
+   *           Content-Length:
+   *             description: Size of the archive in bytes.
+   *             schema:
+   *               type: integer
+   *           ETag:
+   *             description: Quoted SHA-256 checksum of the archive.
+   *             schema:
+   *               type: string
+   *           Last-Modified:
+   *             description: Release date of the dataset.
+   *             schema:
+   *               type: string
+   *           Accept-Ranges:
+   *             description: Always `bytes`.
+   *             schema:
+   *               type: string
+   *         content:
+   *           application/zip:
+   *             schema:
+   *               type: string
+   *               format: binary
+   *       206:
+   *         description: The requested byte range of the dataset archive.
+   *         headers:
+   *           Content-Range:
+   *             schema:
+   *               type: string
+   *         content:
+   *           application/zip:
+   *             schema:
+   *               type: string
+   *               format: binary
+   *       304:
+   *         description: The client already holds the latest dataset archive.
+   *       404:
+   *         $ref: '#/components/responses/NotFoundError'
+   *       412:
+   *         description: The archive changed since the version referenced by `If-Match` or `If-Unmodified-Since`.
+   *         content:
+   *           application/json:
+   *             schema:
+   *               $ref: '#/components/schemas/ErrorResponse'
+   *       416:
+   *         description: The requested byte range cannot be satisfied.
+   *         headers:
+   *           Content-Range:
+   *             schema:
+   *               type: string
+   *         content:
+   *           application/json:
+   *             schema:
+   *               $ref: '#/components/schemas/ErrorResponse'
+   */
+  router.get(DOWNLOAD_PATH, async (req, res, next) => {
+    const metadata = await datasetStorage.findLatest();
+
+    if (!metadata) {
+      return res.status(404).json({ error: NO_DATASET_ERROR });
+    }
+
+    const options = {
+      root: datasetStorage.path, // Confines the served file to the storage directory and keeps `send` from refusing paths that contain dot-directories
+      dotfiles: 'allow', // Archive names are chosen by operators through `--file`; a name starting with a dot, such as `.weekly`, must still be served
+      headers: { // Applied by `send` once the archive is found and kept over its own defaults, so that the validators describe the dataset rather than the file on disk
+        ETag: `"${metadata.sha256}"`,
+        'Last-Modified': new Date(metadata.releaseDate).toUTCString(),
+      },
+    };
+
+    return res.download(metadata.filename, metadata.filename, options, error => handleTransferError(error, res, next));
+  });
+
+  return router;
+}

package/src/collection-api/routes/dataset.test.js

@@ -0,0 +1,395 @@
+import { createHash } from 'crypto';
+import fsApi from 'fs';
+import fs from 'fs/promises';
+import { Readable } from 'stream';
+
+import { expect } from 'chai';
+import config from 'config';
+import sinon from 'sinon';
+import supertest from 'supertest';
+
+import DatasetStorage from '../../dataset/storage.js';
+import app from '../server.js';
+
+import { NO_DATASET_ERROR } from './dataset.js';
+
+const basePath = config.get('@opentermsarchive/engine.collection-api.basePath');
+const request = supertest(app);
+
+function binaryParser(res, callback) { // superagent only buffers text, JSON and media types on its own, so ZIP bodies have to be collected explicitly
+  const chunks = [];
+
+  res.on('data', chunk => chunks.push(chunk));
+  res.on('end', () => callback(null, Buffer.concat(chunks)));
+}
+
+const METADATA_URL = `${basePath}/v1/dataset/latest`;
+const DOWNLOAD_URL = `${basePath}/v1/dataset/latest/download`;
+const ARCHIVE_FILENAME = 'sandbox-2026-01-01.zip';
+const ARCHIVE_CONTENT = Buffer.from('Archive content standing for a ZIP file in tests');
+const METADATA = {
+  filename: ARCHIVE_FILENAME,
+  title: 'sandbox',
+  license: 'ODbL-1.0',
+  releaseDate: '2026-01-01T08:30:12Z',
+  firstVersionDate: '2021-01-01T11:27:00Z',
+  lastVersionDate: '2022-01-06T11:32:47Z',
+  servicesCount: 2,
+  termsCount: 3,
+  versionsCount: 4,
+  size: ARCHIVE_CONTENT.length,
+  sha256: createHash('sha256').update(ARCHIVE_CONTENT).digest('hex'),
+};
+
+describe('Dataset API', () => {
+  const storage = new DatasetStorage(config.get('@opentermsarchive/engine.dataset.storagePath'));
+
+  async function storeDataset() {
+    await fs.mkdir(storage.path, { recursive: true });
+    await fs.writeFile(storage.archivePath(ARCHIVE_FILENAME), ARCHIVE_CONTENT);
+    await storage.save(METADATA);
+  }
+
+  async function removeDataset() {
+    await fs.rm(storage.path, { recursive: true, force: true });
+  }
+
+  function itRespondsWithNoDatasetError(getResponse) {
+    it('responds with 404 status code', () => {
+      expect(getResponse().status).to.equal(404);
+    });
+
+    it('responds with Content-Type application/json', () => {
+      expect(getResponse().type).to.equal('application/json');
+    });
+
+    it('returns an explicit error message', () => {
+      expect(getResponse().body).to.deep.equal({ error: NO_DATASET_ERROR });
+    });
+  }
+
+  describe('GET /dataset/latest', () => {
+    let response;
+
+    context('when no dataset has been generated', () => {
+      before(async () => {
+        await removeDataset();
+        response = await request.get(METADATA_URL);
+      });
+
+      itRespondsWithNoDatasetError(() => response);
+    });
+
+    context('when the archive described by the metadata is missing', () => {
+      before(async () => {
+        await storeDataset();
+        await fs.rm(storage.archivePath(ARCHIVE_FILENAME));
+        response = await request.get(METADATA_URL);
+      });
+
+      after(removeDataset);
+
+      itRespondsWithNoDatasetError(() => response);
+    });
+
+    context('when a dataset exists', () => {
+      before(async () => {
+        await storeDataset();
+        response = await request.get(METADATA_URL);
+      });
+
+      after(removeDataset);
+
+      it('responds with 200 status code', () => {
+        expect(response.status).to.equal(200);
+      });
+
+      it('responds with Content-Type application/json', () => {
+        expect(response.type).to.equal('application/json');
+      });
+
+      it('returns the dataset metadata along with its download URL', () => {
+        const { host } = new URL(response.request.url);
+
+        expect(response.body).to.deep.equal({
+          ...METADATA,
+          downloadURL: `http://${host}${DOWNLOAD_URL}`,
+        });
+      });
+    });
+
+    context('behind a reverse proxy', () => {
+      before(async () => {
+        await storeDataset();
+        response = await request
+          .get(METADATA_URL)
+          .set('X-Forwarded-Proto', 'https')
+          .set('X-Forwarded-Host', 'api.example.com');
+      });
+
+      after(removeDataset);
+
+      it('uses the forwarded protocol and host in the download URL', () => {
+        expect(response.body.downloadURL).to.equal(`https://api.example.com${DOWNLOAD_URL}`);
+      });
+    });
+
+    context('behind a chain of reverse proxies', () => {
+      before(async () => {
+        await storeDataset();
+        response = await request
+          .get(METADATA_URL)
+          .set('X-Forwarded-Proto', 'https')
+          .set('X-Forwarded-Host', 'api.example.com, edge.internal');
+      });
+
+      after(removeDataset);
+
+      it('uses the first host in the forwarded list in the download URL', () => {
+        expect(response.body.downloadURL).to.equal(`https://api.example.com${DOWNLOAD_URL}`);
+      });
+    });
+
+    context('when the stored metadata is corrupted', () => {
+      before(async () => {
+        await fs.mkdir(storage.path, { recursive: true });
+        await fs.writeFile(storage.metadataPath, '{ not json');
+        response = await request.get(METADATA_URL);
+      });
+
+      after(removeDataset);
+
+      it('responds with 500 status code', () => {
+        expect(response.status).to.equal(500);
+      });
+
+      it('responds with Content-Type application/json', () => {
+        expect(response.type).to.equal('application/json');
+      });
+
+      it('returns a generic error message', () => {
+        expect(response.body).to.deep.equal({ error: 'Internal Server Error' });
+      });
+    });
+  });
+
+  describe('GET /dataset/latest/download', () => {
+    let response;
+
+    context('when no dataset has been generated', () => {
+      before(async () => {
+        await removeDataset();
+        response = await request.get(DOWNLOAD_URL);
+      });
+
+      itRespondsWithNoDatasetError(() => response);
+    });
+
+    context('when the archive described by the metadata is missing', () => {
+      before(async () => {
+        await storeDataset();
+        await fs.rm(storage.archivePath(ARCHIVE_FILENAME));
+        response = await request.get(DOWNLOAD_URL);
+      });
+
+      after(removeDataset);
+
+      itRespondsWithNoDatasetError(() => response);
+    });
+
+    context('when a dataset exists', () => {
+      before(storeDataset);
+
+      after(removeDataset);
+
+      describe('without conditions', () => {
+        before(async () => {
+          response = await request.get(DOWNLOAD_URL).buffer(true).parse(binaryParser);
+        });
+
+        it('responds with 200 status code', () => {
+          expect(response.status).to.equal(200);
+        });
+
+        it('responds with Content-Type application/zip', () => {
+          expect(response.type).to.equal('application/zip');
+        });
+
+        it('exposes the archive as an attachment named after the archive file', () => {
+          expect(response.headers['content-disposition']).to.equal(`attachment; filename="${ARCHIVE_FILENAME}"`);
+        });
+
+        it('exposes the archive size as Content-Length', () => {
+          expect(response.headers['content-length']).to.equal(String(ARCHIVE_CONTENT.length));
+        });
+
+        it('exposes the archive checksum as a strong ETag', () => {
+          expect(response.headers.etag).to.equal(`"${METADATA.sha256}"`);
+        });
+
+        it('exposes the release date as Last-Modified', () => {
+          expect(response.headers['last-modified']).to.equal(new Date(METADATA.releaseDate).toUTCString());
+        });
+
+        it('advertises byte range support', () => {
+          expect(response.headers['accept-ranges']).to.equal('bytes');
+        });
+
+        it('returns the archive content', () => {
+          expect(response.body.equals(ARCHIVE_CONTENT)).to.be.true;
+        });
+      });
+
+      describe('with a conditional request', () => {
+        it('returns 304 with no body when If-None-Match matches the archive checksum', async () => {
+          const conditionalResponse = await request.get(DOWNLOAD_URL).set('If-None-Match', `"${METADATA.sha256}"`);
+
+          expect(conditionalResponse.status).to.equal(304);
+          expect(conditionalResponse.text).to.be.empty;
+        });
+
+        it('returns 200 with the archive when If-None-Match does not match', async () => {
+          const conditionalResponse = await request.get(DOWNLOAD_URL).set('If-None-Match', '"another-checksum"');
+
+          expect(conditionalResponse.status).to.equal(200);
+        });
+
+        it('returns 304 with no body when If-Modified-Since is at or after the release date', async () => {
+          const conditionalResponse = await request.get(DOWNLOAD_URL).set('If-Modified-Since', new Date(METADATA.releaseDate).toUTCString());
+
+          expect(conditionalResponse.status).to.equal(304);
+          expect(conditionalResponse.text).to.be.empty;
+        });
+
+        it('returns 200 with the archive when If-Modified-Since is before the release date', async () => {
+          const conditionalResponse = await request.get(DOWNLOAD_URL).set('If-Modified-Since', new Date('2025-12-31T00:00:00Z').toUTCString());
+
+          expect(conditionalResponse.status).to.equal(200);
+        });
+
+        it('returns a JSON error with 412 status code when If-Match does not match the archive checksum', async () => {
+          const conditionalResponse = await request.get(DOWNLOAD_URL).set('If-Match', '"another-checksum"');
+
+          expect(conditionalResponse.status).to.equal(412);
+          expect(conditionalResponse.type).to.equal('application/json');
+          expect(conditionalResponse.headers).to.not.have.any.keys('content-disposition', 'last-modified');
+          expect(conditionalResponse.body).to.deep.equal({ error: 'Precondition Failed' });
+        });
+      });
+
+      describe('with a range request', () => {
+        it('returns the requested bytes with 206 status code', async () => {
+          const rangeResponse = await request.get(DOWNLOAD_URL).set('Range', 'bytes=0-4').buffer(true).parse(binaryParser);
+
+          expect(rangeResponse.status).to.equal(206);
+          expect(rangeResponse.headers['content-range']).to.equal(`bytes 0-4/${ARCHIVE_CONTENT.length}`);
+          expect(rangeResponse.body.equals(ARCHIVE_CONTENT.subarray(0, 5))).to.be.true;
+        });
+
+        it('returns a JSON error with 416 status code when the range cannot be satisfied', async () => {
+          const rangeResponse = await request.get(DOWNLOAD_URL).set('Range', `bytes=${ARCHIVE_CONTENT.length}-`);
+
+          expect(rangeResponse.status).to.equal(416);
+          expect(rangeResponse.type).to.equal('application/json');
+          expect(rangeResponse.headers['content-range']).to.equal(`bytes */${ARCHIVE_CONTENT.length}`);
+          expect(rangeResponse.headers).to.not.have.any.keys('content-disposition', 'last-modified');
+          expect(rangeResponse.headers.etag).to.not.equal(`"${METADATA.sha256}"`);
+          expect(rangeResponse.body).to.deep.equal({ error: 'Range Not Satisfiable' });
+        });
+
+        it('returns 200 with the full archive when If-Range does not match the current archive', async () => {
+          const rangeResponse = await request.get(DOWNLOAD_URL).set('If-Range', '"stale-checksum"').set('Range', 'bytes=0-4').buffer(true)
+            .parse(binaryParser);
+
+          expect(rangeResponse.status).to.equal(200);
+          expect(rangeResponse.body.equals(ARCHIVE_CONTENT)).to.be.true;
+        });
+      });
+
+      describe('with a HEAD request', () => {
+        it('returns the archive headers without its content', async () => {
+          const headResponse = await request.head(DOWNLOAD_URL);
+
+          expect(headResponse.status).to.equal(200);
+          expect(headResponse.headers['content-length']).to.equal(String(ARCHIVE_CONTENT.length));
+          expect(headResponse.headers.etag).to.equal(`"${METADATA.sha256}"`);
+          expect(headResponse.text).to.be.oneOf([ undefined, '' ]);
+        });
+      });
+    });
+
+    context('when the archive file name starts with a dot', () => {
+      const DOTFILE_ARCHIVE_FILENAME = '.weekly-2026-01-01.zip';
+
+      before(async () => {
+        await fs.mkdir(storage.path, { recursive: true });
+        await fs.writeFile(storage.archivePath(DOTFILE_ARCHIVE_FILENAME), ARCHIVE_CONTENT);
+        await storage.save({ ...METADATA, filename: DOTFILE_ARCHIVE_FILENAME });
+        response = await request.get(DOWNLOAD_URL).buffer(true).parse(binaryParser);
+      });
+
+      after(removeDataset);
+
+      it('responds with 200 status code', () => {
+        expect(response.status).to.equal(200);
+      });
+
+      it('returns the archive content', () => {
+        expect(response.body.equals(ARCHIVE_CONTENT)).to.be.true;
+      });
+    });
+
+    context('when the archive cannot be read', () => {
+      before(async () => {
+        await storeDataset();
+        sinon.stub(fsApi, 'createReadStream').returns(new Readable({ read() { this.destroy(new Error('Disk failure')); } })); // `send` opens the archive through the `fs` module at transfer time, once it has already described the archive on the response
+        response = await request.get(DOWNLOAD_URL);
+      });
+
+      after(async () => {
+        sinon.restore();
+        await removeDataset();
+      });
+
+      it('responds with 500 status code', () => {
+        expect(response.status).to.equal(500);
+      });
+
+      it('responds with Content-Type application/json', () => {
+        expect(response.type).to.equal('application/json');
+      });
+
+      it('returns a generic error message', () => {
+        expect(response.body).to.deep.equal({ error: 'Internal Server Error' });
+      });
+
+      it('does not describe the archive', () => {
+        expect(response.headers).to.not.have.any.keys('content-disposition', 'last-modified', 'accept-ranges');
+        expect(response.headers.etag).to.not.equal(`"${METADATA.sha256}"`);
+      });
+    });
+
+    context('when the archive is replaced between the metadata lookup and the transfer', () => {
+      before(async () => {
+        await storeDataset();
+        sinon.stub(DatasetStorage.prototype, 'findLatest').resolves({ ...METADATA, filename: 'sandbox-2026-01-02.zip' }); // The metadata describes an archive that is no longer on disk, as when a generation completes right after the lookup
+        response = await request.get(DOWNLOAD_URL);
+      });
+
+      after(async () => {
+        sinon.restore();
+        await removeDataset();
+      });
+
+      itRespondsWithNoDatasetError(() => response);
+    });
+  });
+
+  describe('GET /dataset', () => {
+    it('responds with 404 status code', async () => {
+      const response = await request.get(`${basePath}/v1/dataset`);
+
+      expect(response.status).to.equal(404);
+    });
+  });
+});

package/src/collection-api/routes/docs.test.js

@@ -54,6 +54,14 @@ describe('Docs API', () => {
           it('/version/{serviceId}/{termsType}/{date}', () => {
             expect(subject).to.have.property('/version/{serviceId}/{termsType}/{date}');
           });
+
+          it('/dataset/latest', () => {
+            expect(subject).to.have.property('/dataset/latest');
+          });
+
+          it('/dataset/latest/download', () => {
+            expect(subject).to.have.property('/dataset/latest/download');
+          });
         });
       });
     });