@opentermsarchive/engine 9.2.3 → 10.1.1

package/bin/ota-apply-technical-upgrades.js

@@ -0,0 +1,19 @@
+#! /usr/bin/env node
+import './env.js';
+
+import path from 'path';
+import { fileURLToPath, pathToFileURL } from 'url';
+
+import { program } from 'commander';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const { applyTechnicalUpgrades } = await import(pathToFileURL(path.resolve(__dirname, '../src/index.js'))); // load asynchronously to ensure env.js is loaded before
+
+program
+  .name('ota apply-technical-upgrades')
+  .description('Apply technical upgrades by generating new versions from the latest snapshots using updated declarations, engine logic, or dependencies, and by retrieving any missing snapshots for newly added source documents')
+  .option('-s, --services [serviceId...]', 'service IDs to apply technical upgrades to')
+  .option('-t, --types [termsType...]', 'terms types to apply technical upgrades to');
+
+applyTechnicalUpgrades(program.parse(process.argv).opts());

package/bin/ota-dataset.js

@@ -11,9 +11,9 @@ import logger from '../src/logger/index.js';
 
 program
   .name('ota dataset')
-  .description('Export the versions dataset into a ZIP file and optionally publish it to GitHub releases')
+  .description('Export the versions dataset into a ZIP file and optionally publish it to GitHub releases, GitLab releases, or data.gouv.fr')
   .option('-f, --file <filename>', 'file name of the generated dataset')
-  .option('-p, --publish', 'publish dataset to GitHub releases on versions repository. Mandatory authentication to GitHub is provided through the `OTA_ENGINE_GITHUB_TOKEN` environment variable')
+  .option('-p, --publish', 'publish dataset. Supports GitHub releases (OTA_ENGINE_GITHUB_TOKEN), GitLab releases (OTA_ENGINE_GITLAB_TOKEN), or data.gouv.fr (OTA_ENGINE_DATAGOUV_API_KEY + config)')
   .option('-r, --remove-local-copy', 'remove local copy of dataset after publishing. Works only in combination with --publish option')
   .option('--schedule', 'schedule automatic dataset generation');
 

package/bin/ota-track.js

@@ -15,7 +15,6 @@ program
   .description('Retrieve declared documents, record snapshots, extract versions and publish the resulting records')
   .option('-s, --services [serviceId...]', 'service IDs of services to track')
   .option('-t, --types [termsType...]', 'terms types to track')
-  .option('-e, --extract-only', 'extract versions from existing snapshots with latest declarations and engine, without recording new snapshots')
   .option('--schedule', 'track automatically at a regular interval');
 
 track(program.parse(process.argv).opts());

package/bin/ota.js

@@ -11,6 +11,7 @@ program
   .description(description)
   .version(version)
   .command('track', 'Track the current terms of services according to provided declarations')
+  .command('apply-technical-upgrades', 'Apply technical upgrades by generating new versions from the latest snapshots using updated declarations, engine logic, or dependencies')
   .command('validate', 'Run a series of tests to check the validity of terms declarations')
   .command('lint', 'Check format and stylistic errors in declarations and auto fix them')
   .command('dataset', 'Export the versions dataset into a ZIP file and optionally publish it to GitHub releases')

package/config/default.json

@@ -56,7 +56,7 @@
       }
     },
     "dataset": {
-      "title": "sandbox",
+      "title": "Sandbox collection dataset",
       "versionsRepositoryURL": "https://github.com/OpenTermsArchive/sandbox-declarations",
       "publishingSchedule": "30 8 * * MON"
     }

package/config/development.json

@@ -0,0 +1,60 @@
+{
+  "@opentermsarchive/engine": {
+    "collectionPath": "../collections/pga-declarations",
+    "recorder": {
+      "versions": {
+        "storage": {
+          "git": {
+            "snapshotIdentiferTemplate": "https://github.com/OpenTermsArchive/vlopses-ie-snapshots/commit/%SNAPSHOT_ID",
+            "repository": "git@github.com:OpenTermsArchive/vlopses-ie-versions.git",
+            "path": "./data/versions"
+          }
+        }
+      },
+      "snapshots": {
+        "storage": {
+          "git": {
+            "repository": "git@github.com:OpenTermsArchive/vlopses-ie-snapshots.git",
+            "path": "./data/snapshots"
+          }
+        }
+      }
+    },
+    "fetcher": {
+      "language": "en"
+    },
+    "notifier": {
+      "sendInBlue": {
+        "updatesListId": 0,
+        "updateTemplateId": 0
+      }
+    },
+    "logger": {
+      "sendMailOnError": {
+        "to": "admin@opentermsarchive.org",
+        "from": "noreply@opentermsarchive.org",
+        "sendWarnings": false
+      }
+    },
+    "reporter": {
+      "type": "github",
+      "repositories": {
+        "declarations": "OpenTermsArchive/vlopses-ie-declarations",
+        "versions": "OpenTermsArchive/vlopses-ie-versions",
+        "snapshots": "OpenTermsArchive/vlopses-ie-snapshots"
+      }
+    },
+    "collection-api": {
+      "host": "127.0.0.1",
+      "port": 3003,
+      "basePath": "/collection-api"
+    },
+    "dataset": {
+      "datagouv": {
+        "organizationIdOrSlug": "otatest",
+        "frequency": "weekly",
+        "useDemo": true
+      }
+    }
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "9.2.3",
+  "version": "10.1.1",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {

package/scripts/dataset/assets/README.template.js

@@ -14,7 +14,7 @@ export function title({ releaseDate }) {
 
   const title = config.get('@opentermsarchive/engine.dataset.title');
 
-  return `${title} — ${releaseDate} dataset`;
+  return `${title} — ${releaseDate}`;
 }
 
 export function body({ servicesCount, firstVersionDate, lastVersionDate }) {

package/scripts/dataset/export/test/fixtures/dataset/README.md

@@ -1,4 +1,4 @@
-# Open Terms Archive — sandbox — January 1, 2022 dataset
+# Open Terms Archive — sandbox — January 1, 2022
 
 This dataset consolidates the contractual documents of 2 service providers, in all their versions that were accessible online between January 1, 2021 and January 6, 2022.
 

package/scripts/dataset/index.js

@@ -9,7 +9,7 @@ import publishRelease from './publish/index.js';
 
 export async function release({ shouldPublish, shouldRemoveLocalCopy, fileName }) {
   const releaseDate = new Date();
-  const archiveName = fileName || `dataset-${config.get('@opentermsarchive/engine.dataset.title')}-${releaseDate.toISOString().replace(/T.*/, '')}`;
+  const archiveName = fileName || `${config.get('@opentermsarchive/engine.dataset.title').toLowerCase().replace(/[^a-zA-Z0-9.\-_]/g, '-')}-${releaseDate.toISOString().replace(/T.*/, '')}`;
   const archivePath = `${path.basename(archiveName, '.zip')}.zip`; // allow to pass filename or filename.zip as the archive name and have filename.zip as the result name
 
   logger.info('Start exporting dataset…');
@@ -24,13 +24,18 @@ export async function release({ shouldPublish, shouldRemoveLocalCopy, fileName }
 
   logger.info('Start publishing dataset…');
 
-  const releaseUrl = await publishRelease({
+  const results = await publishRelease({
     archivePath,
     releaseDate,
     stats,
   });
 
-  logger.info(`Dataset published to ${releaseUrl}`);
+  if (results.length > 0) {
+    logger.info('Dataset published to following platforms:');
+    results.forEach(result => {
+      logger.info(`  - ${result.platform}: ${result.url}`);
+    });
+  }
 
   if (!shouldRemoveLocalCopy) {
     return;

package/scripts/dataset/logger/index.js

@@ -8,13 +8,35 @@ const { combine, timestamp, printf, colorize } = winston.format;
 logger.format = combine(
   colorize(),
   timestamp({ format: 'YYYY-MM-DDTHH:mm:ssZ' }),
-  printf(({ level, message, counter, hash, timestamp }) => {
-    const prefix = counter && hash ? `${counter.toString().padEnd(6)} ${hash.padEnd(40)}` : '';
+  printf(({ level, message, counter, hash, timestamp, module }) => {
+    let prefix = counter && hash ? `${counter.toString().padEnd(6)} ${hash.padEnd(40)}` : '';
 
     const timestampPrefix = config.get('@opentermsarchive/engine.logger.timestampPrefix') ? `${timestamp} ` : '';
 
-    return `${timestampPrefix}${level.padEnd(15)} ${prefix.padEnd(50)} ${message}`;
+    prefix = module ? `${module} ${prefix}` : prefix;
+
+    const levelStr = level.padEnd(15);
+    let coloredLevel = levelStr;
+    let coloredMessage = message;
+
+    if (level.includes('warn')) {
+      coloredLevel = `\x1b[33m${levelStr}\x1b[0m`;
+      coloredMessage = `\x1b[33m${message}\x1b[0m`;
+    } else if (level.includes('error')) {
+      coloredLevel = `\x1b[31m${levelStr}\x1b[0m`;
+      coloredMessage = `\x1b[31m${message}\x1b[0m`;
+    }
+
+    return `${timestampPrefix} ${coloredLevel} ${prefix.padEnd(50)} ${coloredMessage}`;
   }),
 );
 
+export function createModuleLogger(moduleName) {
+  return {
+    info: message => logger.info(message, { module: moduleName }),
+    warn: message => logger.warn(message, { module: moduleName }),
+    error: message => logger.error(message, { module: moduleName }),
+  };
+}
+
 export default logger;

package/scripts/dataset/publish/datagouv/dataset.js

@@ -0,0 +1,234 @@
+import fsApi from 'fs';
+import path from 'path';
+
+import FormData from 'form-data';
+import nodeFetch from 'node-fetch';
+
+import { createModuleLogger } from '../../logger/index.js';
+
+const logger = createModuleLogger('datagouv');
+
+const DATASET_LICENSE = 'odc-odbl';
+const DEFAULT_RESOURCE_DESCRIPTION = 'See README.md inside the archive for dataset structure and usage information.';
+
+const routes = {
+  dataset: (apiBaseUrl, datasetId) => `${apiBaseUrl}/datasets/${datasetId}/`,
+  datasets: apiBaseUrl => `${apiBaseUrl}/datasets/`,
+  datasetUpload: (apiBaseUrl, datasetId) => `${apiBaseUrl}/datasets/${datasetId}/upload/`,
+  resource: (apiBaseUrl, datasetId, resourceId) => `${apiBaseUrl}/datasets/${datasetId}/resources/${resourceId}/`,
+  resourceUpload: (apiBaseUrl, datasetId, resourceId) => `${apiBaseUrl}/datasets/${datasetId}/resources/${resourceId}/upload/`,
+  organization: (apiBaseUrl, organizationIdOrSlug) => `${apiBaseUrl}/organizations/${organizationIdOrSlug}/`,
+  organizationDatasets: (apiBaseUrl, organizationId) => `${apiBaseUrl}/organizations/${organizationId}/datasets/?page_size=100`,
+};
+
+export async function getOrganization({ apiBaseUrl, headers, organizationIdOrSlug }) {
+  logger.info(`Fetching organization: ${organizationIdOrSlug}…`);
+
+  const orgResponse = await nodeFetch(routes.organization(apiBaseUrl, organizationIdOrSlug), { headers });
+
+  if (!orgResponse.ok) {
+    const errorText = await orgResponse.text();
+
+    throw new Error(`Failed to retrieve organization: ${orgResponse.status} ${orgResponse.statusText} - ${errorText}`);
+  }
+
+  const orgData = await orgResponse.json();
+
+  logger.info(`Found organization: ${orgData.name} (ID: ${orgData.id})`);
+
+  return orgData;
+}
+
+export async function getDataset({ apiBaseUrl, headers, datasetId }) {
+  const datasetResponse = await nodeFetch(routes.dataset(apiBaseUrl, datasetId), { headers });
+
+  if (!datasetResponse.ok) {
+    const errorText = await datasetResponse.text();
+    const error = new Error(`Failed to retrieve dataset: ${datasetResponse.status} ${datasetResponse.statusText} - ${errorText}`);
+
+    error.statusCode = datasetResponse.status;
+    throw error;
+  }
+
+  const datasetData = await datasetResponse.json();
+
+  return datasetData;
+}
+
+export async function findDatasetByTitle({ apiBaseUrl, headers, organizationId, title }) {
+  logger.info(`Searching for dataset with title "${title}" in organization…`);
+
+  const searchResponse = await nodeFetch(routes.organizationDatasets(apiBaseUrl, organizationId), { headers });
+
+  if (!searchResponse.ok) {
+    const errorText = await searchResponse.text();
+
+    throw new Error(`Failed to search for datasets: ${searchResponse.status} ${searchResponse.statusText} - ${errorText}`);
+  }
+
+  const searchData = await searchResponse.json();
+
+  const dataset = searchData.data.find(ds => ds.title === title);
+
+  if (dataset) {
+    logger.info(`Found existing dataset: ${dataset.title} (ID: ${dataset.id})`);
+
+    return dataset;
+  }
+
+  logger.info('No existing dataset found with this title');
+
+  return null;
+}
+
+export async function createDataset({ apiBaseUrl, headers, organizationId, title, description, license, frequency }) {
+  logger.info(`Creating new dataset: ${title}…`);
+
+  const createResponse = await nodeFetch(routes.datasets(apiBaseUrl), {
+    method: 'POST',
+    headers: {
+      ...headers,
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify({
+      title,
+      description,
+      organization: organizationId,
+      license,
+      frequency,
+    }),
+  });
+
+  if (!createResponse.ok) {
+    const errorText = await createResponse.text();
+
+    throw new Error(`Failed to create dataset: ${createResponse.status} ${createResponse.statusText} - ${errorText}`);
+  }
+
+  const dataset = await createResponse.json();
+
+  logger.info(`Dataset created successfully: ${dataset.title} (ID: ${dataset.id})`);
+
+  return dataset;
+}
+
+export async function updateDatasetMetadata({ apiBaseUrl, headers, datasetId, title, description, stats, frequency }) {
+  const updatePayload = {
+    title,
+    description,
+    license: DATASET_LICENSE,
+    frequency,
+  };
+
+  if (stats?.firstVersionDate && stats?.lastVersionDate) {
+    updatePayload.temporal_coverage = {
+      start: stats.firstVersionDate.toISOString(),
+      end: stats.lastVersionDate.toISOString(),
+    };
+  }
+
+  const updateResponse = await nodeFetch(routes.dataset(apiBaseUrl, datasetId), {
+    method: 'PUT',
+    headers: {
+      ...headers,
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify(updatePayload),
+  });
+
+  if (!updateResponse.ok) {
+    const errorText = await updateResponse.text();
+    const error = new Error(`Failed to update dataset metadata: ${updateResponse.status} ${updateResponse.statusText} - ${errorText}`);
+
+    error.statusCode = updateResponse.status;
+    throw error;
+  }
+
+  logger.info('Dataset metadata updated successfully');
+}
+
+export async function uploadResource({ apiBaseUrl, headers, datasetId, archivePath }) {
+  logger.info('Uploading dataset archive…');
+
+  const { formData, fileName } = createFormDataForFile(archivePath);
+
+  const uploadResponse = await nodeFetch(routes.datasetUpload(apiBaseUrl, datasetId), {
+    method: 'POST',
+    headers: { ...formData.getHeaders(), ...headers },
+    body: formData,
+  });
+
+  if (!uploadResponse.ok) {
+    const errorText = await uploadResponse.text();
+
+    throw new Error(`Failed to upload dataset file: ${uploadResponse.status} ${uploadResponse.statusText} - ${errorText}`);
+  }
+
+  const uploadResult = await uploadResponse.json();
+
+  logger.info(`Dataset file uploaded successfully with resource ID: ${uploadResult.id}`);
+
+  return { resourceId: uploadResult.id, fileName };
+}
+
+export async function replaceResourceFile({ apiBaseUrl, headers, datasetId, resourceId, archivePath }) {
+  logger.info(`Replacing file for existing resource ID: ${resourceId}…`);
+
+  const { formData, fileName } = createFormDataForFile(archivePath);
+
+  const uploadResponse = await nodeFetch(routes.resourceUpload(apiBaseUrl, datasetId, resourceId), {
+    method: 'POST',
+    headers: { ...formData.getHeaders(), ...headers },
+    body: formData,
+  });
+
+  if (!uploadResponse.ok) {
+    const errorText = await uploadResponse.text();
+
+    throw new Error(`Failed to replace resource file: ${uploadResponse.status} ${uploadResponse.statusText} - ${errorText}`);
+  }
+
+  const uploadResult = await uploadResponse.json();
+
+  logger.info('Resource file replaced successfully');
+
+  return { resourceId: uploadResult.id, fileName };
+}
+
+export async function updateResourceMetadata({ apiBaseUrl, headers, datasetId, resourceId, fileName }) {
+  logger.info('Updating resource metadata…');
+
+  const resourceUpdateResponse = await nodeFetch(routes.resource(apiBaseUrl, datasetId, resourceId), {
+    method: 'PUT',
+    headers: { ...headers, 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+      title: fileName,
+      description: DEFAULT_RESOURCE_DESCRIPTION,
+      filetype: 'file',
+      format: 'zip',
+      mime: 'application/zip',
+    }),
+  });
+
+  if (!resourceUpdateResponse.ok) {
+    const errorText = await resourceUpdateResponse.text();
+
+    throw new Error(`Failed to update resource metadata: ${resourceUpdateResponse.status} ${resourceUpdateResponse.statusText} - ${errorText}`);
+  }
+
+  logger.info('Resource metadata updated successfully');
+}
+
+function createFormDataForFile(archivePath) {
+  const formData = new FormData();
+  const fileName = path.basename(archivePath);
+  const fileStats = fsApi.statSync(archivePath);
+
+  formData.append('file', fsApi.createReadStream(archivePath), {
+    filename: fileName,
+    contentType: 'application/zip',
+    knownLength: fileStats.size,
+  });
+
+  return { formData, fileName };
+}

package/scripts/dataset/publish/datagouv/index.js

@@ -0,0 +1,82 @@
+import config from 'config';
+
+import * as readme from '../../assets/README.template.js';
+import { createModuleLogger } from '../../logger/index.js';
+
+import { updateDatasetMetadata, uploadResource, replaceResourceFile, updateResourceMetadata, getDataset, getOrganization, findDatasetByTitle, createDataset } from './dataset.js';
+
+const logger = createModuleLogger('datagouv');
+
+const PRODUCTION_API_BASE_URL = 'https://www.data.gouv.fr/api/1';
+const DEMO_API_BASE_URL = 'https://demo.data.gouv.fr/api/1';
+const DATASET_LICENSE = 'odc-odbl';
+
+export default async function publish({ archivePath, stats }) {
+  const { datasetId, organizationIdOrSlug, apiBaseUrl, headers, datasetTitle, frequency } = loadConfiguration();
+  const description = readme.body(stats);
+
+  const dataset = datasetId
+    ? await getDataset({ apiBaseUrl, headers, datasetId })
+    : await ensureDatasetExists({ apiBaseUrl, headers, organizationIdOrSlug, datasetTitle, description, frequency });
+
+  await updateDatasetMetadata({ apiBaseUrl, headers, datasetId: dataset.id, title: datasetTitle, description, stats, frequency });
+
+  const { resourceId, fileName } = await handleResourceUpload({ apiBaseUrl, headers, datasetId: dataset.id, dataset, archivePath });
+
+  await updateResourceMetadata({ apiBaseUrl, headers, datasetId: dataset.id, resourceId, fileName });
+
+  logger.info(`Dataset published successfully: ${dataset.page}`);
+
+  return dataset.page;
+}
+
+function loadConfiguration() {
+  const apiKey = process.env.OTA_ENGINE_DATAGOUV_API_KEY;
+
+  if (!apiKey) {
+    throw new Error('OTA_ENGINE_DATAGOUV_API_KEY environment variable is required for data.gouv.fr publishing');
+  }
+
+  const datasetId = config.has('@opentermsarchive/engine.dataset.datagouv.datasetId') && config.get('@opentermsarchive/engine.dataset.datagouv.datasetId');
+  const organizationIdOrSlug = config.has('@opentermsarchive/engine.dataset.datagouv.organizationIdOrSlug') && config.get('@opentermsarchive/engine.dataset.datagouv.organizationIdOrSlug');
+
+  if (!datasetId && !organizationIdOrSlug) {
+    throw new Error('Either datasetId or organizationIdOrSlug is required in config at @opentermsarchive/engine.dataset.datagouv');
+  }
+
+  const datasetTitle = config.get('@opentermsarchive/engine.dataset.title');
+  const frequency = config.has('@opentermsarchive/engine.dataset.datagouv.frequency') && config.get('@opentermsarchive/engine.dataset.datagouv.frequency');
+  const useDemo = config.has('@opentermsarchive/engine.dataset.datagouv.useDemo') && config.get('@opentermsarchive/engine.dataset.datagouv.useDemo');
+  const apiBaseUrl = useDemo ? DEMO_API_BASE_URL : PRODUCTION_API_BASE_URL;
+
+  if (useDemo) {
+    logger.warn('Using demo.data.gouv.fr environment for testing');
+  }
+
+  const headers = { 'X-API-KEY': apiKey };
+
+  return { datasetId, organizationIdOrSlug, apiBaseUrl, headers, datasetTitle, frequency };
+}
+
+async function ensureDatasetExists({ apiBaseUrl, headers, organizationIdOrSlug, datasetTitle, description, frequency }) {
+  const organization = await getOrganization({ apiBaseUrl, headers, organizationIdOrSlug });
+  let dataset = await findDatasetByTitle({ apiBaseUrl, headers, organizationId: organization.id, title: datasetTitle });
+
+  if (!dataset) {
+    dataset = await createDataset({ apiBaseUrl, headers, organizationId: organization.id, title: datasetTitle, description, license: DATASET_LICENSE, frequency });
+  }
+
+  return dataset;
+}
+
+function handleResourceUpload({ apiBaseUrl, headers, datasetId, dataset, archivePath }) {
+  if (dataset?.resources?.length > 0) {
+    const existingResource = dataset.resources[0];
+
+    logger.info(`Found existing resource: ${existingResource.title} (ID: ${existingResource.id})`);
+
+    return replaceResourceFile({ apiBaseUrl, headers, datasetId, resourceId: existingResource.id, archivePath });
+  }
+
+  return uploadResource({ apiBaseUrl, headers, datasetId, archivePath });
+}

package/scripts/dataset/publish/github/index.js

@@ -1,19 +1,23 @@
 import fsApi from 'fs';
 import path from 'path';
-import url from 'url';
 
 import config from 'config';
 import { Octokit } from 'octokit'; // eslint-disable-line import/no-unresolved
 
 import * as readme from '../../assets/README.template.js';
+import { createModuleLogger } from '../../logger/index.js';
+
+const logger = createModuleLogger('github');
 
 export default async function publish({ archivePath, releaseDate, stats }) {
   const octokit = new Octokit({ auth: process.env.OTA_ENGINE_GITHUB_TOKEN });
 
-  const [ owner, repo ] = url.parse(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL')).pathname.split('/').filter(component => component);
+  const [ owner, repo ] = new URL(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL')).pathname.split('/').filter(component => component);
 
   const tagName = `${path.basename(archivePath, path.extname(archivePath))}`; // use archive filename as Git tag
 
+  logger.info(`Creating release for ${owner}/${repo}…`);
+
   const { data: { upload_url: uploadUrl, html_url: releaseUrl } } = await octokit.rest.repos.createRelease({
     owner,
     repo,
@@ -22,6 +26,9 @@ export default async function publish({ archivePath, releaseDate, stats }) {
     body: readme.body(stats),
   });
 
+  logger.info(`Release created successfully with tag: ${tagName}`);
+  logger.info('Uploading release asset…');
+
   await octokit.rest.repos.uploadReleaseAsset({
     data: fsApi.readFileSync(archivePath),
     headers: {
@@ -32,5 +39,7 @@ export default async function publish({ archivePath, releaseDate, stats }) {
     url: uploadUrl,
   });
 
+  logger.info(`Release asset uploaded successfully: ${path.basename(archivePath)}`);
+
   return releaseUrl;
 }

package/scripts/dataset/publish/gitlab/index.js

@@ -8,7 +8,9 @@ import nodeFetch from 'node-fetch';
 
 import GitLab from '../../../../src/reporter/gitlab/index.js';
 import * as readme from '../../assets/README.template.js';
-import logger from '../../logger/index.js';
+import { createModuleLogger } from '../../logger/index.js';
+
+const logger = createModuleLogger('gitlab');
 
 dotenv.config({ quiet: true });
 

package/scripts/dataset/publish/index.js

@@ -1,15 +1,49 @@
+import config from 'config';
+
+import logger from '../logger/index.js';
+
+import publishDataGouv from './datagouv/index.js';
 import publishGitHub from './github/index.js';
 import publishGitLab from './gitlab/index.js';
 
-export default function publishRelease({ archivePath, releaseDate, stats }) {
+export default async function publishRelease({ archivePath, releaseDate, stats }) {
+  const platforms = [];
+
   // If both GitHub and GitLab tokens are defined, GitHub takes precedence
   if (process.env.OTA_ENGINE_GITHUB_TOKEN) {
-    return publishGitHub({ archivePath, releaseDate, stats });
+    platforms.push({ name: 'GitHub', publish: () => publishGitHub({ archivePath, releaseDate, stats }) });
+  } else if (process.env.OTA_ENGINE_GITLAB_TOKEN) {
+    platforms.push({ name: 'GitLab', publish: () => publishGitLab({ archivePath, releaseDate, stats }) });
+  }
+
+  if (process.env.OTA_ENGINE_DATAGOUV_API_KEY && (config.has('@opentermsarchive/engine.dataset.datagouv.datasetId') || config.has('@opentermsarchive/engine.dataset.datagouv.organizationIdOrSlug'))) {
+    platforms.push({ name: 'data.gouv.fr', publish: () => publishDataGouv({ archivePath, releaseDate, stats }) });
+  }
+
+  if (!platforms.length) {
+    throw new Error('No publishing platform configured. Please configure at least one of: GitHub (OTA_ENGINE_GITHUB_TOKEN), GitLab (OTA_ENGINE_GITLAB_TOKEN), or data.gouv.fr (OTA_ENGINE_DATAGOUV_API_KEY + datasetId or organizationIdOrSlug in config).');
   }
 
-  if (process.env.OTA_ENGINE_GITLAB_TOKEN) {
-    return publishGitLab({ archivePath, releaseDate, stats });
+  const results = await Promise.allSettled(platforms.map(async platform => {
+    const url = await platform.publish();
+
+    return { platform: platform.name, url };
+  }));
+
+  const succeeded = results.filter(result => result.status === 'fulfilled');
+  const failed = results.filter(result => result.status === 'rejected');
+
+  if (failed.length) {
+    let errorMessage = !succeeded.length ? 'All platforms failed to publish:' : 'Some platforms failed to publish:';
+
+    failed.forEach(rejectedResult => {
+      const index = results.indexOf(rejectedResult);
+
+      errorMessage += `\n  - ${platforms[index].name}: ${rejectedResult.reason.message}`;
+    });
+
+    logger.error(errorMessage);
   }
 
-  throw new Error('No GitHub nor GitLab token found in environment variables (OTA_ENGINE_GITHUB_TOKEN or OTA_ENGINE_GITLAB_TOKEN). Cannot publish the dataset without authentication.');
+  return succeeded.map(result => result.value);
 }