@opentermsarchive/engine 15.3.1 → 16.0.1

package/src/collection-api/routes/feed.js

@@ -3,6 +3,7 @@ import { js2xml } from 'xml-js';
 
 import { getCollection } from '../../archivist/collection/index.js';
 import { toISODateWithoutMilliseconds } from '../../archivist/utils/date.js';
+import { buildAbsoluteBaseUrl } from '../utils/url.js';
 
 const RECORD_TYPES = {
   firstRecord: 'First record',
@@ -18,12 +19,6 @@ const SCHEMES = Object.freeze({
   recordType: `tag:${TAG_AUTHORITY}:scheme:record-type`,
 });
 
-function buildAbsoluteBaseUrl(req) {
-  const host = req.get('X-Forwarded-Host') ?? req.get('host'); // Behind a trusted reverse proxy, the public host comes from X-Forwarded-Host. req.get('host') only sees the internal Host header, so we read the forwarded value explicitly and fall back to the direct host for non-proxied setups (dev, tests).
-
-  return `${req.protocol}://${host}${req.baseUrl}`;
-}
-
 function classifyRecordType(version) {
   return version.isFirstRecord ? RECORD_TYPES.firstRecord : RECORD_TYPES.change;
 }

package/src/collection-api/routes/index.js

@@ -5,7 +5,9 @@ import helmet from 'helmet';
 import { getCollection } from '../../archivist/collection/index.js';
 import RepositoryFactory from '../../archivist/recorder/repositories/factory.js';
 import * as Services from '../../archivist/services/index.js';
+import DatasetStorage from '../../dataset/storage.js';
 
+import datasetRouter from './dataset.js';
 import docsRouter from './docs.js';
 import feedRouter from './feed.js';
 import metadataRouter from './metadata.js';
@@ -40,6 +42,7 @@ export default async function apiRouter(basePath) {
   const versionsRepository = await RepositoryFactory.create(versionsStorageConfig).initialize();
   const snapshotsRepository = await RepositoryFactory.create(config.get('@opentermsarchive/engine.recorder.snapshots.storage')).initialize();
   const feedConfig = config.get('@opentermsarchive/engine.collection-api.feed');
+  const datasetStorage = new DatasetStorage(config.get('@opentermsarchive/engine.dataset.storagePath'));
 
   if (!collection.metadata?.id) {
     throw new Error('Collection metadata "id" is required to expose feed endpoints, as it is used to build the tag URIs that uniquely identify the feed and its entries. Add an "id" field to the collection metadata file.');
@@ -53,6 +56,7 @@ export default async function apiRouter(basePath) {
   router.use(servicesRouter(services));
   router.use(versionsRouter(versionsRepository, snapshotsRepository));
   router.use(feedRouter(services, versionsRepository, versionsStorageConfig.type, feedConfig.limit, feedConfig.versionUrlTemplate));
+  router.use(datasetRouter(datasetStorage));
 
   return router;
 }

package/src/collection-api/utils/url.js

@@ -0,0 +1,3 @@
+export function buildAbsoluteBaseUrl(req) {
+  return `${req.protocol}://${req.host}${req.baseUrl}`;
+}

package/src/dataset/license.js

@@ -0,0 +1,4 @@
+export const SPDX_ID = 'ODbL-1.0'; // Identifier of the license shipped in scripts/dataset/assets/LICENSE
+export const DATAGOUV_ID = 'odc-odbl'; // data.gouv.fr's own identifier for the same license, distinct from the SPDX scheme
+export const DISPLAY_NAME = 'Open Database (ODbL) License';
+export const INFO_URL = 'https://opendatacommons.org/licenses/odbl/1.0/';

package/src/dataset/storage.js

@@ -0,0 +1,66 @@
+import fs from 'fs/promises';
+import path from 'path';
+
+export const TEMPORARY_SUFFIX = '.tmp';
+
+const METADATA_FILENAME = 'metadata.json';
+const ARCHIVE_EXTENSION = '.zip';
+
+export default class DatasetStorage {
+  constructor(storagePath) {
+    this.path = path.resolve(process.cwd(), storagePath);
+  }
+
+  get metadataPath() {
+    return path.join(this.path, METADATA_FILENAME);
+  }
+
+  archivePath(filename) {
+    return path.join(this.path, filename);
+  }
+
+  async findLatest() {
+    let metadata;
+
+    try {
+      metadata = JSON.parse(await fs.readFile(this.metadataPath, 'utf8'));
+      await fs.access(this.archivePath(metadata.filename));
+    } catch (error) {
+      if (error.code === 'ENOENT') {
+        return null;
+      }
+
+      throw error;
+    }
+
+    return metadata;
+  }
+
+  async save(metadata) {
+    await fs.access(this.archivePath(metadata.filename)); // Never describe an archive that is not in place
+
+    const temporaryPath = `${this.metadataPath}${TEMPORARY_SUFFIX}`;
+
+    await fs.writeFile(temporaryPath, JSON.stringify(metadata, null, 2));
+    await fs.rename(temporaryPath, this.metadataPath); // Readers see either the previous or the new metadata, never a partial file
+  }
+
+  async removePreviousArchives() {
+    const metadata = await this.findLatest();
+
+    if (!metadata) {
+      return;
+    }
+
+    await this.#removeArchivesExcept(metadata.filename);
+  }
+
+  async #removeArchivesExcept(filename) {
+    const entries = await fs.readdir(this.path);
+    const isArchiveOrLeftover = entry => entry.endsWith(ARCHIVE_EXTENSION) || entry.endsWith(`${ARCHIVE_EXTENSION}${TEMPORARY_SUFFIX}`); // Cleanup is restricted to archives and their leftovers: the storage path is user-provided and may hold unrelated files
+
+    await Promise.all(entries
+      .filter(entry => entry !== filename && isArchiveOrLeftover(entry))
+      .map(entry => fs.rm(this.archivePath(entry), { force: true })));
+  }
+}

package/src/dataset/storage.test.js

@@ -0,0 +1,209 @@
+import fs from 'fs/promises';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+import { expect, use } from 'chai';
+import chaiAsPromised from 'chai-as-promised';
+import sinon from 'sinon';
+
+import DatasetStorage, { TEMPORARY_SUFFIX } from './storage.js';
+
+use(chaiAsPromised);
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const TMP_PATH = path.resolve(__dirname, './tmp');
+const ARCHIVE_FILENAME = 'dataset-2026-01-01.zip';
+const PREVIOUS_ARCHIVE_FILENAME = 'dataset-2025-12-25.zip';
+const INTERRUPTED_ARCHIVE_FILENAME = `dataset-2025-12-26.zip${TEMPORARY_SUFFIX}`;
+const UNRELATED_FILENAME = 'notes.txt';
+const METADATA = {
+  filename: ARCHIVE_FILENAME,
+  title: 'sandbox',
+  license: 'ODbL-1.0',
+  releaseDate: '2026-01-01T08:30:00Z',
+  firstVersionDate: '2021-01-01T11:27:00Z',
+  lastVersionDate: '2022-01-06T11:32:47Z',
+  servicesCount: 2,
+  termsCount: 3,
+  versionsCount: 4,
+  size: 15,
+  sha256: 'a'.repeat(64),
+};
+
+describe('DatasetStorage', () => {
+  const storage = new DatasetStorage(TMP_PATH);
+
+  async function removeStorage() {
+    await fs.rm(TMP_PATH, { recursive: true, force: true });
+  }
+
+  before(removeStorage);
+
+  describe('#findLatest', () => {
+    context('when the storage directory does not exist', () => {
+      it('returns null', async () => {
+        expect(await storage.findLatest()).to.be.null;
+      });
+    });
+
+    context('when no metadata file exists', () => {
+      before(async () => {
+        await fs.mkdir(TMP_PATH, { recursive: true });
+      });
+
+      after(removeStorage);
+
+      it('returns null', async () => {
+        expect(await storage.findLatest()).to.be.null;
+      });
+    });
+
+    context('when the metadata file references a missing archive', () => {
+      before(async () => {
+        await fs.mkdir(TMP_PATH, { recursive: true });
+        await fs.writeFile(storage.metadataPath, JSON.stringify(METADATA));
+      });
+
+      after(removeStorage);
+
+      it('returns null', async () => {
+        expect(await storage.findLatest()).to.be.null;
+      });
+    });
+
+    context('when the metadata file is corrupted', () => {
+      before(async () => {
+        await fs.mkdir(TMP_PATH, { recursive: true });
+        await fs.writeFile(storage.metadataPath, '{ not json');
+      });
+
+      after(removeStorage);
+
+      it('rejects', async () => {
+        await expect(storage.findLatest()).to.be.rejected;
+      });
+    });
+
+    context('when the metadata file and the archive exist', () => {
+      before(async () => {
+        await fs.mkdir(TMP_PATH, { recursive: true });
+        await fs.writeFile(storage.archivePath(ARCHIVE_FILENAME), 'archive content');
+        await fs.writeFile(storage.metadataPath, JSON.stringify(METADATA));
+      });
+
+      after(removeStorage);
+
+      it('returns the metadata', async () => {
+        expect(await storage.findLatest()).to.deep.equal(METADATA);
+      });
+    });
+  });
+
+  describe('#save', () => {
+    context('when the archive is missing', () => {
+      before(async () => {
+        await fs.mkdir(TMP_PATH, { recursive: true });
+      });
+
+      after(removeStorage);
+
+      it('rejects', async () => {
+        await expect(storage.save(METADATA)).to.be.rejected;
+      });
+
+      it('writes no metadata file', async () => {
+        await storage.save(METADATA).catch(() => {});
+
+        await expect(fs.access(storage.metadataPath)).to.be.rejected;
+      });
+    });
+
+    context('when the archive exists', () => {
+      before(async () => {
+        await fs.mkdir(TMP_PATH, { recursive: true });
+        await fs.writeFile(storage.archivePath(ARCHIVE_FILENAME), 'archive content');
+        await storage.save(METADATA);
+      });
+
+      after(removeStorage);
+
+      it('writes the metadata file', async () => {
+        expect(JSON.parse(await fs.readFile(storage.metadataPath, 'utf8'))).to.deep.equal(METADATA);
+      });
+
+      it('leaves no temporary metadata file', async () => {
+        await expect(fs.access(`${storage.metadataPath}${TEMPORARY_SUFFIX}`)).to.be.rejected;
+      });
+
+      it('keeps the archive', async () => {
+        await expect(fs.access(storage.archivePath(ARCHIVE_FILENAME))).to.be.fulfilled;
+      });
+    });
+  });
+
+  describe('#removePreviousArchives', () => {
+    context('when no metadata exists', () => {
+      before(async () => {
+        await fs.mkdir(TMP_PATH, { recursive: true });
+      });
+
+      after(removeStorage);
+
+      it('does nothing', async () => {
+        await expect(storage.removePreviousArchives()).to.be.fulfilled;
+      });
+    });
+
+    context('when metadata exists', () => {
+      before(async () => {
+        await fs.mkdir(TMP_PATH, { recursive: true });
+        await fs.writeFile(storage.archivePath(ARCHIVE_FILENAME), 'archive content');
+        await fs.writeFile(storage.archivePath(PREVIOUS_ARCHIVE_FILENAME), 'previous archive');
+        await fs.writeFile(storage.archivePath(INTERRUPTED_ARCHIVE_FILENAME), 'interrupted archive');
+        await fs.writeFile(storage.archivePath(UNRELATED_FILENAME), 'unrelated content');
+        await fs.writeFile(storage.metadataPath, JSON.stringify(METADATA));
+
+        await storage.removePreviousArchives();
+      });
+
+      after(removeStorage);
+
+      it('keeps the current archive', async () => {
+        await expect(fs.access(storage.archivePath(ARCHIVE_FILENAME))).to.be.fulfilled;
+      });
+
+      it('removes previous archives', async () => {
+        await expect(fs.access(storage.archivePath(PREVIOUS_ARCHIVE_FILENAME))).to.be.rejected;
+      });
+
+      it('removes interrupted archives', async () => {
+        await expect(fs.access(storage.archivePath(INTERRUPTED_ARCHIVE_FILENAME))).to.be.rejected;
+      });
+
+      it('keeps unrelated files', async () => {
+        await expect(fs.access(storage.archivePath(UNRELATED_FILENAME))).to.be.fulfilled;
+      });
+    });
+
+    context('when a previous archive fails to be removed', () => {
+      before(async () => {
+        await fs.mkdir(TMP_PATH, { recursive: true });
+        await fs.writeFile(storage.archivePath(ARCHIVE_FILENAME), 'archive content');
+        await fs.writeFile(storage.archivePath(PREVIOUS_ARCHIVE_FILENAME), 'previous archive');
+        await fs.writeFile(storage.metadataPath, JSON.stringify(METADATA));
+
+        sinon.stub(fs, 'rm').rejects(new Error('Permission denied'));
+      });
+
+      after(async () => {
+        sinon.restore(); // Restored first, so that the cleanup below runs the real `fs.rm`
+        await removeStorage();
+      });
+
+      it('rejects', async () => {
+        await expect(storage.removePreviousArchives()).to.be.rejected;
+      });
+    });
+  });
+});