@opentermsarchive/engine 2.5.0 → 2.7.0

package/config/default.json

@@ -48,7 +48,8 @@
         "host": "smtp-relay.sendinblue.com",
         "username": "admin@opentermsarchive.org"
       },
-      "sendMailOnError": false
+      "sendMailOnError": false,
+      "timestampPrefix": true
     },
     "notifier": {
       "sendInBlue": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "2.5.0",
+  "version": "2.7.0",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {

package/scripts/dataset/logger/index.js

@@ -1,3 +1,4 @@
+import config from 'config';
 import winston from 'winston';
 
 import logger from '../../../src/logger/index.js';
@@ -6,11 +7,13 @@ const { combine, timestamp, printf, colorize } = winston.format;
 
 logger.format = combine(
   colorize(),
-  timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
+  timestamp({ format: 'YYYY-MM-DDTHH:MM:SSZ' }),
   printf(({ level, message, counter, hash, timestamp }) => {
     const prefix = counter && hash ? `${counter.toString().padEnd(6)} ${hash.padEnd(40)}` : '';
 
-    return `${timestamp} ${level.padEnd(15)} ${prefix.padEnd(50)} ${message}`;
+    const timestampPrefix = config.get('@opentermsarchive/engine.logger.timestampPrefix') ? `${timestamp} ` : '';
+
+    return `${timestampPrefix}${level.padEnd(15)} ${prefix.padEnd(50)} ${message}`;
   }),
 );
 

package/scripts/dataset/publish/github/index.js

@@ -0,0 +1,36 @@
+import fsApi from 'fs';
+import path from 'path';
+import url from 'url';
+
+import config from 'config';
+import { Octokit } from 'octokit';
+
+import * as readme from '../../assets/README.template.js';
+
+export default async function publish({ archivePath, releaseDate, stats }) {
+  const octokit = new Octokit({ auth: process.env.OTA_ENGINE_GITHUB_TOKEN });
+
+  const [ owner, repo ] = url.parse(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL')).pathname.split('/').filter(component => component);
+
+  const tagName = `${path.basename(archivePath, path.extname(archivePath))}`; // use archive filename as Git tag
+
+  const { data: { upload_url: uploadUrl, html_url: releaseUrl } } = await octokit.rest.repos.createRelease({
+    owner,
+    repo,
+    tag_name: tagName,
+    name: readme.title({ releaseDate }),
+    body: readme.body(stats),
+  });
+
+  await octokit.rest.repos.uploadReleaseAsset({
+    data: fsApi.readFileSync(archivePath),
+    headers: {
+      'content-type': 'application/zip',
+      'content-length': fsApi.statSync(archivePath).size,
+    },
+    name: path.basename(archivePath),
+    url: uploadUrl,
+  });
+
+  return releaseUrl;
+}

package/scripts/dataset/publish/gitlab/index.js

@@ -0,0 +1,133 @@
+import fsApi from 'fs';
+import path from 'path';
+
+import config from 'config';
+import dotenv from 'dotenv';
+import FormData from 'form-data';
+import nodeFetch from 'node-fetch';
+
+import GitLab from '../../../../src/reporter/gitlab/index.js';
+import * as readme from '../../assets/README.template.js';
+import logger from '../../logger/index.js';
+
+dotenv.config();
+
+export default async function publish({
+  archivePath,
+  releaseDate,
+  stats,
+}) {
+  let projectId = null;
+  const gitlabAPIUrl = config.get('@opentermsarchive/engine.dataset.apiBaseURL');
+
+  const [ owner, repo ] = new URL(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL'))
+    .pathname
+    .split('/')
+    .filter(Boolean);
+  const commonParams = { owner, repo };
+
+  try {
+    const repositoryPath = `${commonParams.owner}/${commonParams.repo}`;
+
+    const options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN);
+
+    options.method = 'GET';
+    options.headers = {
+      'Content-Type': 'application/json',
+      ...options.headers,
+    };
+
+    const response = await nodeFetch(
+      `${gitlabAPIUrl}/projects/${encodeURIComponent(repositoryPath)}`,
+      options,
+    );
+    const res = await response.json();
+
+    projectId = res.id;
+  } catch (error) {
+    logger.error(`Error while obtaining projectId: ${error}`);
+    projectId = null;
+  }
+
+  const tagName = `${path.basename(archivePath, path.extname(archivePath))}`; // use archive filename as Git tag
+
+  try {
+    let options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN);
+
+    options.method = 'POST';
+    options.body = {
+      ref: 'main',
+      tag_name: tagName,
+      name: readme.title({ releaseDate }),
+      description: readme.body(stats),
+    };
+    options.headers = {
+      'Content-Type': 'application/json',
+      ...options.headers,
+    };
+
+    options.body = JSON.stringify(options.body);
+
+    const releaseResponse = await nodeFetch(
+      `${gitlabAPIUrl}/projects/${projectId}/releases`,
+      options,
+    );
+    const releaseRes = await releaseResponse.json();
+
+    const releaseId = releaseRes.commit.id;
+
+    logger.info(`Created release with releaseId: ${releaseId}`);
+
+    // Upload the package
+    options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN);
+    options.method = 'PUT';
+    options.body = fsApi.createReadStream(archivePath);
+
+    // restrict characters to the ones allowed by GitLab APIs
+    const packageName = config.get('@opentermsarchive/engine.dataset.title').replace(/[^a-zA-Z0-9.\-_]/g, '-');
+    const packageVersion = tagName.replace(/[^a-zA-Z0-9.\-_]/g, '-');
+    const packageFileName = archivePath.replace(/[^a-zA-Z0-9.\-_/]/g, '-');
+
+    logger.debug(`packageName: ${packageName}, packageVersion: ${packageVersion} packageFileName: ${packageFileName}`);
+
+    const packageResponse = await nodeFetch(
+      `${gitlabAPIUrl}/projects/${projectId}/packages/generic/${packageName}/${packageVersion}/${packageFileName}?status=default&select=package_file`,
+      options,
+    );
+    const packageRes = await packageResponse.json();
+
+    const packageFilesId = packageRes.id;
+
+    logger.debug(`package file id: ${packageFilesId}`);
+
+    // use the package id to build the download url for the release
+    const publishedPackageUrl = `${config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL')}/-/package_files/${packageFilesId}/download`;
+
+    // Create the release and link the package
+    const formData = new FormData();
+
+    formData.append('name', archivePath);
+    formData.append('url', publishedPackageUrl);
+    formData.append('file', fsApi.createReadStream(archivePath), { filename: path.basename(archivePath) });
+
+    options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN);
+    options.method = 'POST';
+    options.headers = {
+      ...formData.getHeaders(),
+      ...options.headers,
+    };
+    options.body = formData;
+
+    const uploadResponse = await nodeFetch(
+      `${gitlabAPIUrl}/projects/${projectId}/releases/${tagName}/assets/links`,
+      options,
+    );
+    const uploadRes = await uploadResponse.json();
+    const releaseUrl = uploadRes.direct_asset_url;
+
+    return releaseUrl;
+  } catch (error) {
+    logger.error('Failed to create release or upload ZIP file:', error);
+    throw error;
+  }
+}

package/scripts/dataset/publish/index.js

@@ -1,36 +1,15 @@
-import fsApi from 'fs';
-import path from 'path';
-import url from 'url';
+import publishGitHub from './github/index.js';
+import publishGitLab from './gitlab/index.js';
 
-import config from 'config';
-import { Octokit } from 'octokit';
+export default function publishRelease({ archivePath, releaseDate, stats }) {
+  // If both GitHub and GitLab tokens are defined, GitHub takes precedence
+  if (process.env.OTA_ENGINE_GITHUB_TOKEN) {
+    return publishGitHub({ archivePath, releaseDate, stats });
+  }
 
-import * as readme from '../assets/README.template.js';
+  if (process.env.OTA_ENGINE_GITLAB_TOKEN) {
+    return publishGitLab({ archivePath, releaseDate, stats });
+  }
 
-export default async function publish({ archivePath, releaseDate, stats }) {
-  const octokit = new Octokit({ auth: process.env.OTA_ENGINE_GITHUB_TOKEN });
-
-  const [ owner, repo ] = url.parse(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL')).pathname.split('/').filter(component => component);
-
-  const tagName = `${path.basename(archivePath, path.extname(archivePath))}`; // use archive filename as Git tag
-
-  const { data: { upload_url: uploadUrl, html_url: releaseUrl } } = await octokit.rest.repos.createRelease({
-    owner,
-    repo,
-    tag_name: tagName,
-    name: readme.title({ releaseDate }),
-    body: readme.body(stats),
-  });
-
-  await octokit.rest.repos.uploadReleaseAsset({
-    data: fsApi.readFileSync(archivePath),
-    headers: {
-      'content-type': 'application/zip',
-      'content-length': fsApi.statSync(archivePath).size,
-    },
-    name: path.basename(archivePath),
-    url: uploadUrl,
-  });
-
-  return releaseUrl;
+  throw new Error('No GitHub nor GitLab token found in environment variables (OTA_ENGINE_GITHUB_TOKEN or OTA_ENGINE_GITLAB_TOKEN). Cannot publish the dataset without authentication.');
 }

package/src/collection-api/logger.js

@@ -30,8 +30,12 @@ if (config.get('@opentermsarchive/engine.logger.sendMailOnError')) {
 const logger = winston.createLogger({
   format: combine(
     colorize(),
-    timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
-    printf(({ level, message, timestamp }) => `${timestamp} ${level.padEnd(15)} ${message}`),
+    timestamp({ format: 'YYYY-MM-DDTHH:MM:SSZ' }),
+    printf(({ level, message, timestamp }) => {
+      const timestampPrefix = config.get('@opentermsarchive/engine.logger.timestampPrefix') ? `${timestamp} ` : '';
+
+      return `${timestampPrefix}${level.padEnd(15)} ${message}`;
+    }),
   ),
   transports,
   rejectionHandlers: transports,

package/src/index.js

@@ -55,21 +55,17 @@ export default async function track({ services, types, extractOnly, schedule })
     logger.warn('Environment variable "OTA_ENGINE_SENDINBLUE_API_KEY" was not found; the Notifier module will be ignored');
   }
 
-  if (process.env.OTA_ENGINE_GITHUB_TOKEN) {
-    if (config.has('@opentermsarchive/engine.reporter.githubIssues.repositories.declarations')) {
-      try {
-        const reporter = new Reporter(config.get('@opentermsarchive/engine.reporter'));
-
-        await reporter.initialize();
-        archivist.attach(reporter);
-      } catch (error) {
-        logger.error('Cannot instantiate the Reporter module; it will be ignored:', error);
-      }
-    } else {
-      logger.warn('Configuration key "reporter.githubIssues.repositories.declarations" was not found; issues on the declarations repository cannot be created');
+  if (process.env.OTA_ENGINE_GITHUB_TOKEN || process.env.OTA_ENGINE_GITLAB_TOKEN) {
+    try {
+      const reporter = new Reporter(config.get('@opentermsarchive/engine.reporter'));
+
+      await reporter.initialize();
+      archivist.attach(reporter);
+    } catch (error) {
+      logger.error('Cannot instantiate the Reporter module; it will be ignored:', error);
     }
   } else {
-    logger.warn('Environment variable "OTA_ENGINE_GITHUB_TOKEN" was not found; the Reporter module will be ignored');
+    logger.warn('Environment variable with token for GitHub or GitLab was not found; the Reporter module will be ignored');
   }
 
   if (!schedule) {

package/src/logger/index.js

@@ -8,23 +8,17 @@ const { combine, timestamp, printf, colorize } = winston.format;
 
 const alignedWithColorsAndTime = combine(
   colorize(),
-  timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
+  timestamp({ format: 'YYYY-MM-DDTHH:MM:SSZ' }),
   printf(({ level, message, timestamp, serviceId, termsType, documentId }) => {
-    let prefix = '';
+    const servicePrefix = serviceId && termsType
+      ? `${serviceId} — ${termsType}${documentId ? `:${documentId}` : ''}`
+      : '';
 
-    if (serviceId && termsType) {
-      prefix = `${serviceId} — ${termsType}`;
-    }
-
-    if (documentId) {
-      prefix = `${prefix}:${documentId}`;
-    }
+    const truncatedPrefix = servicePrefix.length > 75 ? `${servicePrefix.slice(0, 74)}…` : servicePrefix;
 
-    if (prefix.length > 75) {
-      prefix = `${prefix.substring(0, 74)}…`;
-    }
+    const timestampPrefix = config.get('@opentermsarchive/engine.logger.timestampPrefix') ? `${timestamp} ` : '';
 
-    return `${timestamp} ${level.padEnd(15)} ${prefix.padEnd(75)} ${message}`;
+    return `${timestampPrefix}${level.padEnd(15)} ${truncatedPrefix.padEnd(75)} ${message}`;
   }),
 );
 

package/src/reporter/factory.js

@@ -0,0 +1,13 @@
+import GitHub from './github/index.js';
+import GitLab from './gitlab/index.js';
+
+export function createReporter(config) {
+  switch (config.type) {
+  case 'github':
+    return new GitHub(config.repositories.declarations);
+  case 'gitlab':
+    return new GitLab(config.repositories.declarations, config.baseURL, config.apiBaseURL);
+  default:
+    throw new Error(`Unsupported reporter type: ${config.type}`);
+  }
+}

package/src/reporter/{github.js → github/index.js}

@@ -2,7 +2,7 @@ import { createRequire } from 'module';
 
 import { Octokit } from 'octokit';
 
-import logger from '../logger/index.js';
+import logger from '../../logger/index.js';
 
 const require = createRequire(import.meta.url);
 
@@ -14,7 +14,7 @@ export default class GitHub {
   static ISSUE_STATE_ALL = 'all';
 
   constructor(repository) {
-    const { version } = require('../../package.json');
+    const { version } = require('../../../package.json');
 
     this.octokit = new Octokit({
       auth: process.env.OTA_ENGINE_GITHUB_TOKEN,
@@ -198,4 +198,16 @@ export default class GitHub {
       logger.error(`Failed to update issue "${title}": ${error.stack}`);
     }
   }
+
+  generateDeclarationURL(serviceName) {
+    return `https://github.com/${this.commonParams.owner}/${this.commonParams.repo}/blob/main/declarations/${encodeURIComponent(serviceName)}.json`;
+  }
+
+  generateVersionURL(serviceName, termsType) {
+    return `https://github.com/${this.commonParams.owner}/${this.commonParams.repo}/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}.md`;
+  }
+
+  generateSnapshotsBaseUrl(serviceName, termsType) {
+    return `https://github.com/${this.commonParams.owner}/${this.commonParams.repo}/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}`;
+  }
 }

package/src/reporter/{github.test.js → github/index.test.js}

@@ -3,7 +3,7 @@ import { createRequire } from 'module';
 import { expect } from 'chai';
 import nock from 'nock';
 
-import GitHub from './github.js';
+import GitHub from './index.js';
 
 const require = createRequire(import.meta.url);
 

package/src/reporter/{labels.test.js → github/labels.test.js}

@@ -2,7 +2,7 @@ import { createRequire } from 'module';
 
 import chai from 'chai';
 
-import { MANAGED_BY_OTA_MARKER } from './github.js';
+import { MANAGED_BY_OTA_MARKER } from './index.js';
 
 const require = createRequire(import.meta.url);