@opentermsarchive/engine 2.4.0 → 2.6.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "2.4.0",
+  "version": "2.6.0",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {

package/scripts/dataset/publish/github/index.js

@@ -0,0 +1,36 @@
+import fsApi from 'fs';
+import path from 'path';
+import url from 'url';
+
+import config from 'config';
+import { Octokit } from 'octokit';
+
+import * as readme from '../../assets/README.template.js';
+
+export default async function publish({ archivePath, releaseDate, stats }) {
+  const octokit = new Octokit({ auth: process.env.OTA_ENGINE_GITHUB_TOKEN });
+
+  const [ owner, repo ] = url.parse(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL')).pathname.split('/').filter(component => component);
+
+  const tagName = `${path.basename(archivePath, path.extname(archivePath))}`; // use archive filename as Git tag
+
+  const { data: { upload_url: uploadUrl, html_url: releaseUrl } } = await octokit.rest.repos.createRelease({
+    owner,
+    repo,
+    tag_name: tagName,
+    name: readme.title({ releaseDate }),
+    body: readme.body(stats),
+  });
+
+  await octokit.rest.repos.uploadReleaseAsset({
+    data: fsApi.readFileSync(archivePath),
+    headers: {
+      'content-type': 'application/zip',
+      'content-length': fsApi.statSync(archivePath).size,
+    },
+    name: path.basename(archivePath),
+    url: uploadUrl,
+  });
+
+  return releaseUrl;
+}

package/scripts/dataset/publish/gitlab/index.js

@@ -0,0 +1,133 @@
+import fsApi from 'fs';
+import path from 'path';
+
+import config from 'config';
+import dotenv from 'dotenv';
+import FormData from 'form-data';
+import nodeFetch from 'node-fetch';
+
+import GitLab from '../../../../src/reporter/gitlab/index.js';
+import * as readme from '../../assets/README.template.js';
+import logger from '../../logger/index.js';
+
+dotenv.config();
+
+export default async function publish({
+  archivePath,
+  releaseDate,
+  stats,
+}) {
+  let projectId = null;
+  const gitlabAPIUrl = config.get('@opentermsarchive/engine.dataset.apiBaseURL');
+
+  const [ owner, repo ] = new URL(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL'))
+    .pathname
+    .split('/')
+    .filter(Boolean);
+  const commonParams = { owner, repo };
+
+  try {
+    const repositoryPath = `${commonParams.owner}/${commonParams.repo}`;
+
+    const options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN);
+
+    options.method = 'GET';
+    options.headers = {
+      'Content-Type': 'application/json',
+      ...options.headers,
+    };
+
+    const response = await nodeFetch(
+      `${gitlabAPIUrl}/projects/${encodeURIComponent(repositoryPath)}`,
+      options,
+    );
+    const res = await response.json();
+
+    projectId = res.id;
+  } catch (error) {
+    logger.error(`Error while obtaining projectId: ${error}`);
+    projectId = null;
+  }
+
+  const tagName = `${path.basename(archivePath, path.extname(archivePath))}`; // use archive filename as Git tag
+
+  try {
+    let options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN);
+
+    options.method = 'POST';
+    options.body = {
+      ref: 'main',
+      tag_name: tagName,
+      name: readme.title({ releaseDate }),
+      description: readme.body(stats),
+    };
+    options.headers = {
+      'Content-Type': 'application/json',
+      ...options.headers,
+    };
+
+    options.body = JSON.stringify(options.body);
+
+    const releaseResponse = await nodeFetch(
+      `${gitlabAPIUrl}/projects/${projectId}/releases`,
+      options,
+    );
+    const releaseRes = await releaseResponse.json();
+
+    const releaseId = releaseRes.commit.id;
+
+    logger.info(`Created release with releaseId: ${releaseId}`);
+
+    // Upload the package
+    options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN);
+    options.method = 'PUT';
+    options.body = fsApi.createReadStream(archivePath);
+
+    // restrict characters to the ones allowed by GitLab APIs
+    const packageName = config.get('@opentermsarchive/engine.dataset.title').replace(/[^a-zA-Z0-9.\-_]/g, '-');
+    const packageVersion = tagName.replace(/[^a-zA-Z0-9.\-_]/g, '-');
+    const packageFileName = archivePath.replace(/[^a-zA-Z0-9.\-_/]/g, '-');
+
+    logger.debug(`packageName: ${packageName}, packageVersion: ${packageVersion} packageFileName: ${packageFileName}`);
+
+    const packageResponse = await nodeFetch(
+      `${gitlabAPIUrl}/projects/${projectId}/packages/generic/${packageName}/${packageVersion}/${packageFileName}?status=default&select=package_file`,
+      options,
+    );
+    const packageRes = await packageResponse.json();
+
+    const packageFilesId = packageRes.id;
+
+    logger.debug(`package file id: ${packageFilesId}`);
+
+    // use the package id to build the download url for the release
+    const publishedPackageUrl = `${config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL')}/-/package_files/${packageFilesId}/download`;
+
+    // Create the release and link the package
+    const formData = new FormData();
+
+    formData.append('name', archivePath);
+    formData.append('url', publishedPackageUrl);
+    formData.append('file', fsApi.createReadStream(archivePath), { filename: path.basename(archivePath) });
+
+    options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN);
+    options.method = 'POST';
+    options.headers = {
+      ...formData.getHeaders(),
+      ...options.headers,
+    };
+    options.body = formData;
+
+    const uploadResponse = await nodeFetch(
+      `${gitlabAPIUrl}/projects/${projectId}/releases/${tagName}/assets/links`,
+      options,
+    );
+    const uploadRes = await uploadResponse.json();
+    const releaseUrl = uploadRes.direct_asset_url;
+
+    return releaseUrl;
+  } catch (error) {
+    logger.error('Failed to create release or upload ZIP file:', error);
+    throw error;
+  }
+}

package/scripts/dataset/publish/index.js

@@ -1,36 +1,15 @@
-import fsApi from 'fs';
-import path from 'path';
-import url from 'url';
+import publishGitHub from './github/index.js';
+import publishGitLab from './gitlab/index.js';
 
-import config from 'config';
-import { Octokit } from 'octokit';
+export default function publishRelease({ archivePath, releaseDate, stats }) {
+  // If both GitHub and GitLab tokens are defined, GitHub takes precedence
+  if (process.env.OTA_ENGINE_GITHUB_TOKEN) {
+    return publishGitHub({ archivePath, releaseDate, stats });
+  }
 
-import * as readme from '../assets/README.template.js';
+  if (process.env.OTA_ENGINE_GITLAB_TOKEN) {
+    return publishGitLab({ archivePath, releaseDate, stats });
+  }
 
-export default async function publish({ archivePath, releaseDate, stats }) {
-  const octokit = new Octokit({ auth: process.env.OTA_ENGINE_GITHUB_TOKEN });
-
-  const [ owner, repo ] = url.parse(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL')).pathname.split('/').filter(component => component);
-
-  const tagName = `${path.basename(archivePath, path.extname(archivePath))}`; // use archive filename as Git tag
-
-  const { data: { upload_url: uploadUrl, html_url: releaseUrl } } = await octokit.rest.repos.createRelease({
-    owner,
-    repo,
-    tag_name: tagName,
-    name: readme.title({ releaseDate }),
-    body: readme.body(stats),
-  });
-
-  await octokit.rest.repos.uploadReleaseAsset({
-    data: fsApi.readFileSync(archivePath),
-    headers: {
-      'content-type': 'application/zip',
-      'content-length': fsApi.statSync(archivePath).size,
-    },
-    name: path.basename(archivePath),
-    url: uploadUrl,
-  });
-
-  return releaseUrl;
+  throw new Error('No GitHub nor GitLab token found in environment variables (OTA_ENGINE_GITHUB_TOKEN or OTA_ENGINE_GITLAB_TOKEN). Cannot publish the dataset without authentication.');
 }

package/scripts/reporter/duplicate/README.md

@@ -0,0 +1,37 @@
+# Duplicate issues removal script
+
+This script helps remove duplicate issues from a GitHub repository by closing issues that have the same title as any older issue.
+
+## Prerequisites
+
+1. Set up environment variables:
+   - Create a `.env` file in the root directory
+   - Add the GitHub personal access token of the bot that manages issues on your collection, with `repo` permissions:
+
+```shell
+OTA_ENGINE_GITHUB_TOKEN=your_github_token
+```
+
+2. Configure the target repository in your chosen configuration file within the `config` folder:
+
+```json
+{
+  "@opentermsarchive/engine": {
+    "reporter": {
+      "githubIssues": {
+        "repositories": {
+          "declarations": "owner/repository"
+        }
+      }
+    }
+  }
+}
+```
+
+## Usage
+
+Run the script using:
+
+```shell
+node scripts/reporter/duplicate/index.js
+```

package/scripts/reporter/duplicate/index.js

@@ -0,0 +1,73 @@
+import 'dotenv/config';
+import config from 'config';
+import { Octokit } from 'octokit';
+
+async function removeDuplicateIssues() {
+  const repository = config.get('@opentermsarchive/engine.reporter.githubIssues.repositories.declarations');
+
+  if (!repository.includes('/') || repository.includes('https://')) {
+    throw new Error(`Configuration entry "reporter.githubIssues.repositories.declarations" is expected to be a string in the format <owner>/<repo>, but received: "${repository}"`);
+  }
+
+  const [ owner, repo ] = repository.split('/');
+
+  const octokit = new Octokit({ auth: process.env.OTA_ENGINE_GITHUB_TOKEN });
+
+  console.log(`Getting issues from repository ${repository}…`);
+
+  const issues = await octokit.paginate('GET /repos/{owner}/{repo}/issues', {
+    owner,
+    repo,
+    state: 'open',
+    per_page: 100,
+  });
+
+  const onlyIssues = issues.filter(issue => !issue.pull_request);
+  const issuesByTitle = new Map();
+  let counter = 0;
+
+  console.log(`Found ${onlyIssues.length} issues`);
+
+  for (const issue of onlyIssues) {
+    if (!issuesByTitle.has(issue.title)) {
+      issuesByTitle.set(issue.title, [issue]);
+    } else {
+      issuesByTitle.get(issue.title).push(issue);
+    }
+  }
+
+  for (const [ title, duplicateIssues ] of issuesByTitle) {
+    if (duplicateIssues.length === 1) continue;
+
+    const originalIssue = duplicateIssues.reduce((oldest, current) => (new Date(current.created_at) < new Date(oldest.created_at) ? current : oldest));
+
+    console.log(`\nFound ${duplicateIssues.length - 1} duplicates for issue #${originalIssue.number} "${title}"`);
+
+    for (const issue of duplicateIssues) {
+      if (issue.number === originalIssue.number) {
+        continue;
+      }
+
+      await octokit.request('PATCH /repos/{owner}/{repo}/issues/{issue_number}', { /* eslint-disable-line no-await-in-loop */
+        owner,
+        repo,
+        issue_number: issue.number,
+        state: 'closed',
+      });
+
+      await octokit.request('POST /repos/{owner}/{repo}/issues/{issue_number}/comments', { /* eslint-disable-line no-await-in-loop */
+        owner,
+        repo,
+        issue_number: issue.number,
+        body: `This issue is detected as duplicate as it has the same title as #${originalIssue.number}. It most likely was created accidentally by an engine older than [v2.3.2](https://github.com/OpenTermsArchive/engine/releases/tag/v2.3.2). Closing automatically.`,
+      });
+
+      counter++;
+      console.log(`Closed issue #${issue.number}: ${issue.html_url}`);
+    }
+  }
+
+  console.log(`\nDuplicate removal process completed; ${counter} issues closed`);
+}
+
+removeDuplicateIssues();

package/src/index.js

@@ -55,21 +55,17 @@ export default async function track({ services, types, extractOnly, schedule })
     logger.warn('Environment variable "OTA_ENGINE_SENDINBLUE_API_KEY" was not found; the Notifier module will be ignored');
   }
 
-  if (process.env.OTA_ENGINE_GITHUB_TOKEN) {
-    if (config.has('@opentermsarchive/engine.reporter.githubIssues.repositories.declarations')) {
-      try {
-        const reporter = new Reporter(config.get('@opentermsarchive/engine.reporter'));
-
-        await reporter.initialize();
-        archivist.attach(reporter);
-      } catch (error) {
-        logger.error('Cannot instantiate the Reporter module; it will be ignored:', error);
-      }
-    } else {
-      logger.warn('Configuration key "reporter.githubIssues.repositories.declarations" was not found; issues on the declarations repository cannot be created');
+  if (process.env.OTA_ENGINE_GITHUB_TOKEN || process.env.OTA_ENGINE_GITLAB_TOKEN) {
+    try {
+      const reporter = new Reporter(config.get('@opentermsarchive/engine.reporter'));
+
+      await reporter.initialize();
+      archivist.attach(reporter);
+    } catch (error) {
+      logger.error('Cannot instantiate the Reporter module; it will be ignored:', error);
     }
   } else {
-    logger.warn('Environment variable "OTA_ENGINE_GITHUB_TOKEN" was not found; the Reporter module will be ignored');
+    logger.warn('Environment variable with token for GitHub or GitLab was not found; the Reporter module will be ignored');
   }
 
   if (!schedule) {

package/src/reporter/factory.js

@@ -0,0 +1,13 @@
+import GitHub from './github/index.js';
+import GitLab from './gitlab/index.js';
+
+export function createReporter(config) {
+  switch (config.type) {
+  case 'github':
+    return new GitHub(config.repositories.declarations);
+  case 'gitlab':
+    return new GitLab(config.repositories.declarations, config.baseURL, config.apiBaseURL);
+  default:
+    throw new Error(`Unsupported reporter type: ${config.type}`);
+  }
+}

package/src/reporter/{github.js → github/index.js}

@@ -2,7 +2,7 @@ import { createRequire } from 'module';
 
 import { Octokit } from 'octokit';
 
-import logger from '../logger/index.js';
+import logger from '../../logger/index.js';
 
 const require = createRequire(import.meta.url);
 
@@ -14,7 +14,7 @@ export default class GitHub {
   static ISSUE_STATE_ALL = 'all';
 
   constructor(repository) {
-    const { version } = require('../../package.json');
+    const { version } = require('../../../package.json');
 
     this.octokit = new Octokit({
       auth: process.env.OTA_ENGINE_GITHUB_TOKEN,
@@ -198,4 +198,16 @@ export default class GitHub {
       logger.error(`Failed to update issue "${title}": ${error.stack}`);
     }
   }
+
+  generateDeclarationURL(serviceName) {
+    return `https://github.com/${this.commonParams.owner}/${this.commonParams.repo}/blob/main/declarations/${encodeURIComponent(serviceName)}.json`;
+  }
+
+  generateVersionURL(serviceName, termsType) {
+    return `https://github.com/${this.commonParams.owner}/${this.commonParams.repo}/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}.md`;
+  }
+
+  generateSnapshotsBaseUrl(serviceName, termsType) {
+    return `https://github.com/${this.commonParams.owner}/${this.commonParams.repo}/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}`;
+  }
 }

package/src/reporter/{github.test.js → github/index.test.js}

@@ -3,7 +3,7 @@ import { createRequire } from 'module';
 import { expect } from 'chai';
 import nock from 'nock';
 
-import GitHub from './github.js';
+import GitHub from './index.js';
 
 const require = createRequire(import.meta.url);
 

package/src/reporter/{labels.test.js → github/labels.test.js}

@@ -2,7 +2,7 @@ import { createRequire } from 'module';
 
 import chai from 'chai';
 
-import { MANAGED_BY_OTA_MARKER } from './github.js';
+import { MANAGED_BY_OTA_MARKER } from './index.js';
 
 const require = createRequire(import.meta.url);