@opentermsarchive/engine 5.6.1 → 6.0.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "5.6.1",
+  "version": "6.0.0",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {

package/src/archivist/fetcher/fullDomFetcher.js

@@ -20,6 +20,11 @@ export default async function fetch(url, cssSelectors, config) {
     await page.setDefaultNavigationTimeout(config.navigationTimeout);
     await page.setExtraHTTPHeaders({ 'Accept-Language': config.language });
 
+    await page.setCacheEnabled(false); // Disable cache to ensure fresh content on each fetch and prevent stale data from previous requests
+    const client = await page.target().createCDPSession();
+
+    await client.send('Network.clearBrowserCookies'); // Clear cookies to ensure clean state between fetches and prevent session persistence across different URLs
+
     response = await page.goto(url, { waitUntil: 'load' }); // Using `load` instead of `networkidle0` as it's more reliable and faster. The 'load' event fires when the page and all its resources (stylesheets, scripts, images) have finished loading. `networkidle0` can be problematic as it waits for 500ms of network inactivity, which may never occur on dynamic pages and then triggers a navigation timeout.
 
     if (!response) {

package/src/reporter/github/index.js

@@ -3,15 +3,15 @@ import { createRequire } from 'module';
 import { Octokit } from 'octokit';
 
 import logger from '../../logger/index.js';
+import { LABELS, MANAGED_BY_OTA_MARKER, DEPRECATED_MANAGED_BY_OTA_MARKER } from '../labels.js';
 
 const require = createRequire(import.meta.url);
 
-export const MANAGED_BY_OTA_MARKER = '[managed by OTA]';
-
 export default class GitHub {
   static ISSUE_STATE_CLOSED = 'closed';
   static ISSUE_STATE_OPEN = 'open';
   static ISSUE_STATE_ALL = 'all';
+  static MAX_LABEL_DESCRIPTION_LENGTH = 100;
 
   constructor(repository) {
     const { version } = require('../../../package.json');
@@ -49,9 +49,19 @@ export default class GitHub {
   }
 
   async initialize() {
-    this.MANAGED_LABELS = require('./labels.json');
+    this.MANAGED_LABELS = Object.values(LABELS);
     try {
       const existingLabels = await this.getRepositoryLabels();
+      const labelsToRemove = existingLabels.filter(label => label.description && label.description.includes(DEPRECATED_MANAGED_BY_OTA_MARKER));
+
+      if (labelsToRemove.length) {
+        logger.info(`Removing labels with deprecated markers: ${labelsToRemove.map(label => `"${label.name}"`).join(', ')}`);
+
+        for (const label of labelsToRemove) {
+          await this.deleteLabel(label.name); /* eslint-disable-line no-await-in-loop */
+        }
+      }
+
       const existingLabelsNames = existingLabels.map(label => label.name);
       const missingLabels = this.MANAGED_LABELS.filter(label => !existingLabelsNames.includes(label.name));
 
@@ -116,6 +126,13 @@ export default class GitHub {
     });
   }
 
+  async deleteLabel(name) {
+    await this.octokit.request('DELETE /repos/{owner}/{repo}/labels/{name}', {
+      ...this.commonParams,
+      name,
+    });
+  }
+
   async getIssue(title) {
     return (await this.issues).get(title);
   }
@@ -174,25 +191,25 @@ export default class GitHub {
     }
   }
 
-  async createOrUpdateIssue({ title, description, label }) {
+  async createOrUpdateIssue({ title, description, labels }) {
     try {
       const issue = await this.getIssue(title);
 
       if (!issue) {
-        const createdIssue = await this.createIssue({ title, description, labels: [label] });
+        const createdIssue = await this.createIssue({ title, description, labels });
 
         return logger.info(`Created issue #${createdIssue.number} "${title}": ${createdIssue.html_url}`);
       }
 
       const managedLabelsNames = this.MANAGED_LABELS.map(label => label.name);
       const labelsNotManagedToKeep = issue.labels.map(label => label.name).filter(label => !managedLabelsNames.includes(label));
-      const [managedLabel] = issue.labels.filter(label => managedLabelsNames.includes(label.name)); // It is assumed that only one specific reason for failure is possible at a time, making managed labels mutually exclusive
+      const managedLabels = issue.labels.filter(label => managedLabelsNames.includes(label.name));
 
-      if (issue.state !== GitHub.ISSUE_STATE_CLOSED && managedLabel?.name === label) {
-        return;
+      if (issue.state !== GitHub.ISSUE_STATE_CLOSED && labels.every(label => managedLabels.some(managedLabel => managedLabel.name === label))) {
+        return; // if all requested labels are already assigned to the issue, the error is redundant with the one already reported and no further action is necessary
       }
 
-      const updatedIssue = await this.updateIssue(issue, { state: GitHub.ISSUE_STATE_OPEN, labels: [ label, ...labelsNotManagedToKeep ] });
+      const updatedIssue = await this.updateIssue(issue, { state: GitHub.ISSUE_STATE_OPEN, labels: [ ...labels, ...labelsNotManagedToKeep ] });
 
       await this.addCommentToIssue({ issue, comment: description });
 

package/src/reporter/github/index.test.js

@@ -1,22 +1,20 @@
-import { createRequire } from 'module';
-
 import { expect } from 'chai';
 import nock from 'nock';
 
-import GitHub from './index.js';
+import { LABELS } from '../labels.js';
 
-const require = createRequire(import.meta.url);
+import GitHub from './index.js';
 
 describe('GitHub', function () {
   this.timeout(5000);
 
   let MANAGED_LABELS;
   let github;
-  const EXISTING_OPEN_ISSUE = { number: 1, title: 'Opened issue', description: 'Issue description', state: GitHub.ISSUE_STATE_OPEN, labels: [{ name: 'location' }] };
-  const EXISTING_CLOSED_ISSUE = { number: 2, title: 'Closed issue', description: 'Issue description', state: GitHub.ISSUE_STATE_CLOSED, labels: [{ name: '403' }] };
+  const EXISTING_OPEN_ISSUE = { number: 1, title: 'Opened issue', description: 'Issue description', state: GitHub.ISSUE_STATE_OPEN, labels: [{ name: 'page access restriction' }, { name: 'server error' }] };
+  const EXISTING_CLOSED_ISSUE = { number: 2, title: 'Closed issue', description: 'Issue description', state: GitHub.ISSUE_STATE_CLOSED, labels: [{ name: 'empty content' }] };
 
   before(async () => {
-    MANAGED_LABELS = require('./labels.json');
+    MANAGED_LABELS = Object.values(LABELS);
     github = new GitHub('owner/repo');
     nock('https://api.github.com')
       .get('/repos/owner/repo/issues')
@@ -274,7 +272,7 @@ describe('GitHub', function () {
 
   describe('#createOrUpdateIssue', () => {
     before(() => {
-      github.MANAGED_LABELS = require('./labels.json');
+      github.MANAGED_LABELS = Object.values(LABELS);
     });
 
     context('when the issue does not exist', () => {
@@ -282,7 +280,7 @@ describe('GitHub', function () {
       const ISSUE_TO_CREATE = {
         title: 'New Issue',
         description: 'Description of the new issue',
-        label: 'bug',
+        labels: ['empty response'],
       };
 
       before(async () => {
@@ -290,7 +288,7 @@ describe('GitHub', function () {
           .post('/repos/owner/repo/issues', {
             title: ISSUE_TO_CREATE.title,
             body: ISSUE_TO_CREATE.description,
-            labels: [ISSUE_TO_CREATE.label],
+            labels: ISSUE_TO_CREATE.labels,
           })
           .reply(200, { number: 123 });
 
@@ -313,14 +311,14 @@ describe('GitHub', function () {
 
         before(async () => {
           updateIssueScope = nock('https://api.github.com')
-            .patch(`/repos/owner/repo/issues/${EXISTING_CLOSED_ISSUE.number}`, { state: GitHub.ISSUE_STATE_OPEN, labels: ['location'] })
+            .patch(`/repos/owner/repo/issues/${EXISTING_CLOSED_ISSUE.number}`, { state: GitHub.ISSUE_STATE_OPEN, labels: ['page access restriction'] })
             .reply(200);
 
           addCommentScope = nock('https://api.github.com')
             .post(`/repos/owner/repo/issues/${EXISTING_CLOSED_ISSUE.number}/comments`, { body: EXISTING_CLOSED_ISSUE.description })
             .reply(200);
 
-          await github.createOrUpdateIssue({ title: EXISTING_CLOSED_ISSUE.title, description: EXISTING_CLOSED_ISSUE.description, label: 'location' });
+          await github.createOrUpdateIssue({ title: EXISTING_CLOSED_ISSUE.title, description: EXISTING_CLOSED_ISSUE.description, labels: ['page access restriction'] });
         });
 
         after(() => {
@@ -344,14 +342,14 @@ describe('GitHub', function () {
 
           before(async () => {
             updateIssueScope = nock('https://api.github.com')
-              .patch(`/repos/owner/repo/issues/${EXISTING_OPEN_ISSUE.number}`, { state: GitHub.ISSUE_STATE_OPEN, labels: ['404'] })
+              .patch(`/repos/owner/repo/issues/${EXISTING_OPEN_ISSUE.number}`, { state: GitHub.ISSUE_STATE_OPEN, labels: ['empty content'] })
               .reply(200);
 
             addCommentScope = nock('https://api.github.com')
               .post(`/repos/owner/repo/issues/${EXISTING_OPEN_ISSUE.number}/comments`, { body: EXISTING_OPEN_ISSUE.description })
               .reply(200);
 
-            await github.createOrUpdateIssue({ title: EXISTING_OPEN_ISSUE.title, description: EXISTING_OPEN_ISSUE.description, label: '404' });
+            await github.createOrUpdateIssue({ title: EXISTING_OPEN_ISSUE.title, description: EXISTING_OPEN_ISSUE.description, labels: ['empty content'] });
           });
 
           after(() => {
@@ -367,25 +365,27 @@ describe('GitHub', function () {
             expect(addCommentScope.isDone()).to.be.true;
           });
         });
-        context('when the reason did not change', () => {
+
+        context('when all requested labels are already present', () => {
           let addCommentScope;
           let updateIssueScope;
 
           before(async () => {
             updateIssueScope = nock('https://api.github.com')
-              .patch(`/repos/owner/repo/issues/${EXISTING_OPEN_ISSUE.number}`, { state: GitHub.ISSUE_STATE_OPEN, labels: EXISTING_OPEN_ISSUE.labels })
+              .patch(`/repos/owner/repo/issues/${EXISTING_OPEN_ISSUE.number}`)
               .reply(200);
 
             addCommentScope = nock('https://api.github.com')
-              .post(`/repos/owner/repo/issues/${EXISTING_OPEN_ISSUE.number}/comments`, { body: EXISTING_OPEN_ISSUE.description })
+              .post(`/repos/owner/repo/issues/${EXISTING_OPEN_ISSUE.number}/comments`)
               .reply(200);
 
-            await github.createOrUpdateIssue({ title: EXISTING_OPEN_ISSUE.title, description: EXISTING_OPEN_ISSUE.description, label: EXISTING_OPEN_ISSUE.labels[0].name });
+            await github.createOrUpdateIssue({ title: EXISTING_OPEN_ISSUE.title, description: EXISTING_OPEN_ISSUE.description, labels: [ 'page access restriction', 'server error' ] });
           });
 
           after(() => {
-            github.issuesCache.delete(EXISTING_OPEN_ISSUE.title);
             github.issuesCache.set(EXISTING_OPEN_ISSUE.title, EXISTING_OPEN_ISSUE);
+
+            nock.cleanAll();
           });
 
           it('does not attempt to updates the issue’s labels', () => {
@@ -396,6 +396,37 @@ describe('GitHub', function () {
             expect(addCommentScope.isDone()).to.be.false;
           });
         });
+
+        context('when some but not all requested labels are present', () => {
+          let addCommentScope;
+          let updateIssueScope;
+
+          before(async () => {
+            updateIssueScope = nock('https://api.github.com')
+              .patch(`/repos/owner/repo/issues/${EXISTING_OPEN_ISSUE.number}`, { state: GitHub.ISSUE_STATE_OPEN, labels: [ 'page access restriction', 'empty content' ] })
+              .reply(200);
+
+            addCommentScope = nock('https://api.github.com')
+              .post(`/repos/owner/repo/issues/${EXISTING_OPEN_ISSUE.number}/comments`, { body: EXISTING_OPEN_ISSUE.description })
+              .reply(200);
+
+            await github.createOrUpdateIssue({ title: EXISTING_OPEN_ISSUE.title, description: EXISTING_OPEN_ISSUE.description, labels: [ 'page access restriction', 'empty content' ] });
+          });
+
+          after(() => {
+            github.issuesCache.set(EXISTING_OPEN_ISSUE.title, EXISTING_OPEN_ISSUE);
+
+            nock.cleanAll();
+          });
+
+          it("updates the issue's labels", () => {
+            expect(updateIssueScope.isDone()).to.be.true;
+          });
+
+          it('adds a comment to the issue', () => {
+            expect(addCommentScope.isDone()).to.be.true;
+          });
+        });
       });
     });
   });

package/src/reporter/gitlab/index.js

@@ -1,14 +1,10 @@
-import { createRequire } from 'module';
-
 import HttpProxyAgent from 'http-proxy-agent';
 import HttpsProxyAgent from 'https-proxy-agent';
 import nodeFetch from 'node-fetch';
 
 import logger from '../../logger/index.js';
+import { LABELS, MANAGED_BY_OTA_MARKER, DEPRECATED_MANAGED_BY_OTA_MARKER } from '../labels.js';
 
-const require = createRequire(import.meta.url);
-
-export const MANAGED_BY_OTA_MARKER = '[managed by OTA]';
 const BASE_URL = 'https://gitlab.com';
 const API_BASE_URL = 'https://gitlab.com/api/v4';
 
@@ -16,12 +12,12 @@ export default class GitLab {
   static ISSUE_STATE_CLOSED = 'closed';
   static ISSUE_STATE_OPEN = 'opened';
   static ISSUE_STATE_ALL = 'all';
+  static MAX_LABEL_DESCRIPTION_LENGTH = 255;
 
   constructor(repository, baseURL = BASE_URL, apiBaseURL = API_BASE_URL) {
     this.repositoryPath = repository;
     this.projectId = null;
     this.baseURL = baseURL;
-    console.log('this.baseURL', this.baseURL);
     this.apiBaseURL = apiBaseURL;
   }
 
@@ -47,22 +43,35 @@ export default class GitLab {
       this.projectId = null;
     }
 
-    this.MANAGED_LABELS = require('./labels.json');
+    this.MANAGED_LABELS = Object.values(LABELS);
+    try {
+      const existingLabels = await this.getRepositoryLabels();
+      const labelsToRemove = existingLabels.filter(label => label.description && label.description.includes(DEPRECATED_MANAGED_BY_OTA_MARKER));
+
+      if (labelsToRemove.length) {
+        logger.info(`Removing labels with deprecated markers: ${labelsToRemove.map(label => `"${label.name}"`).join(', ')}`);
+
+        for (const label of labelsToRemove) {
+          await this.deleteLabel(label.name); /* eslint-disable-line no-await-in-loop */
+        }
+      }
 
-    const existingLabels = await this.getRepositoryLabels();
-    const existingLabelsNames = existingLabels.map(label => label.name);
-    const missingLabels = this.MANAGED_LABELS.filter(label => !existingLabelsNames.includes(label.name));
+      const existingLabelsNames = existingLabels.map(label => label.name);
+      const missingLabels = this.MANAGED_LABELS.filter(label => !existingLabelsNames.includes(label.name));
 
-    if (missingLabels.length) {
-      logger.info(`The following required labels are not present on the repository: ${missingLabels.map(label => `"${label.name}"`).join(', ')}. Creating them…`);
+      if (missingLabels.length) {
+        logger.info(`Following required labels are not present on the repository: ${missingLabels.map(label => `"${label.name}"`).join(', ')}. Creating them…`);
 
-      for (const label of missingLabels) {
-        await this.createLabel({ /* eslint-disable-line no-await-in-loop */
-          name: label.name,
-          color: label.color,
-          description: `${label.description} ${MANAGED_BY_OTA_MARKER}`,
-        });
+        for (const label of missingLabels) {
+          await this.createLabel({ /* eslint-disable-line no-await-in-loop */
+            name: label.name,
+            color: `#${label.color}`,
+            description: `${label.description} ${MANAGED_BY_OTA_MARKER}`,
+          });
+        }
       }
+    } catch (error) {
+      logger.error(`Failed to handle repository labels: ${error.message}`);
     }
   }
 
@@ -122,6 +131,29 @@ export default class GitLab {
     }
   }
 
+  async deleteLabel(name) {
+    try {
+      const options = GitLab.baseOptionsHttpReq();
+
+      options.method = 'DELETE';
+
+      const response = await nodeFetch(
+        `${this.apiBaseURL}/projects/${this.projectId}/labels/${encodeURIComponent(name)}`,
+        options,
+      );
+
+      if (response.ok) {
+        logger.info(`Label deleted: ${name}`);
+      } else {
+        const res = await response.json();
+
+        logger.error(`deleteLabel response: ${JSON.stringify(res)}`);
+      }
+    } catch (error) {
+      logger.error(`Failed to delete label: ${error}`);
+    }
+  }
+
   async createIssue({ title, description, labels }) {
     try {
       const issue = {
@@ -318,35 +350,39 @@ export default class GitLab {
     return this.closeIssue(issue);
   }
 
-  async createOrUpdateIssue({ title, description, label }) {
-    const issue = await this.getIssue({ title, state: GitLab.ISSUE_STATE_ALL });
+  async createOrUpdateIssue({ title, description, labels }) {
+    try {
+      const issue = await this.getIssue({ title, state: GitLab.ISSUE_STATE_ALL });
 
-    if (!issue) {
-      return this.createIssue({ title, description, labels: [label] });
-    }
+      if (!issue) {
+        const createdIssue = await this.createIssue({ title, description, labels });
 
-    if (issue.state == GitLab.ISSUE_STATE_CLOSED) {
-      await this.openIssue(issue);
-    }
+        return logger.info(`Created GitLab issue #${createdIssue.iid} "${title}": ${createdIssue.web_url}`);
+      }
 
-    const managedLabelsNames = this.MANAGED_LABELS.map(label => label.name);
-    const [managedLabel] = issue.labels.filter(label =>
-      managedLabelsNames.includes(label.name)); // it is assumed that only one specific reason for failure is possible at a time, making managed labels mutually exclusive
+      const managedLabelsNames = this.MANAGED_LABELS.map(label => label.name);
+      const labelsNotManagedToKeep = issue.labels.map(label => label.name).filter(label => !managedLabelsNames.includes(label));
+      const managedLabels = issue.labels.filter(label => managedLabelsNames.includes(label.name));
 
-    if (managedLabel?.name == label) {
-      // if the label is already assigned to the issue, the error is redundant with the one already reported and no further action is necessary
-      return;
-    }
+      if (issue.state !== GitLab.ISSUE_STATE_CLOSED && labels.every(label => managedLabels.some(managedLabel => managedLabel.name === label))) {
+        return; // if all requested labels are already assigned to the issue, the error is redundant with the one already reported and no further action is necessary
+      }
 
-    const labelsNotManagedToKeep = issue.labels
-      .map(label => label.name)
-      .filter(label => !managedLabelsNames.includes(label));
+      if (issue.state == GitLab.ISSUE_STATE_CLOSED) {
+        await this.openIssue(issue);
+      }
 
-    await this.setIssueLabels({
-      issue,
-      labels: [ label, ...labelsNotManagedToKeep ],
-    });
-    await this.addCommentToIssue({ issue, comment: description });
+      await this.setIssueLabels({
+        issue,
+        labels: [ ...labels, ...labelsNotManagedToKeep ],
+      });
+
+      await this.addCommentToIssue({ issue, comment: description });
+
+      logger.info(`Updated GitLab issue with comment #${issue.iid}: ${issue.web_url}`);
+    } catch (error) {
+      logger.error(`Failed to update GitLab issue "${title}": ${error.stack}`);
+    }
   }
 
   static baseOptionsHttpReq(token = process.env.OTA_ENGINE_GITLAB_TOKEN) {
@@ -368,7 +404,7 @@ export default class GitLab {
   }
 
   generateVersionURL(serviceName, termsType) {
-    return `${this.baseURL}/${this.repositoryPath}/-/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(serviceName, termsType)}.md`;
+    return `${this.baseURL}/${this.repositoryPath}/-/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}.md`;
   }
 
   generateSnapshotsBaseUrl(serviceName, termsType) {

package/src/reporter/gitlab/index.test.js

@@ -1,11 +1,9 @@
-import { createRequire } from 'module';
-
 import { expect } from 'chai';
 import nock from 'nock';
 
-import GitLab from './index.js';
+import { LABELS } from '../labels.js';
 
-const require = createRequire(import.meta.url);
+import GitLab from './index.js';
 
 describe('GitLab', function () {
   this.timeout(5000);
@@ -15,7 +13,7 @@ describe('GitLab', function () {
   const PROJECT_ID = '4';
 
   before(() => {
-    MANAGED_LABELS = require('./labels.json');
+    MANAGED_LABELS = Object.values(LABELS);
     gitlab = new GitLab('owner/repo');
   });
 
@@ -400,12 +398,14 @@ describe('GitLab', function () {
       await gitlab.initialize();
     });
 
+    after(nock.cleanAll);
+
     context('when the issue does not exist', () => {
       let createIssueScope;
       const ISSUE_TO_CREATE = {
         title: 'New Issue',
         description: 'Description of the new issue',
-        label: 'bug',
+        labels: ['empty response'],
       };
 
       before(async () => {
@@ -419,7 +419,7 @@ describe('GitLab', function () {
             {
               title: ISSUE_TO_CREATE.title,
               description: ISSUE_TO_CREATE.description,
-              labels: [ISSUE_TO_CREATE.label],
+              labels: ISSUE_TO_CREATE.labels,
             },
           )
           .reply(200, { iid: 123, web_url: 'https://example.com/test/test' });
@@ -436,7 +436,7 @@ describe('GitLab', function () {
       const ISSUE = {
         title: 'Existing Issue',
         description: 'New comment',
-        label: 'location',
+        labels: ['page access restriction'],
       };
 
       context('when issue is closed', () => {
@@ -448,7 +448,7 @@ describe('GitLab', function () {
           iid: 123,
           title: ISSUE.title,
           description: ISSUE.description,
-          labels: [{ name: 'selectors' }],
+          labels: [{ name: 'empty content' }],
           state: GitLab.ISSUE_STATE_CLOSED,
         };
 
@@ -456,7 +456,7 @@ describe('GitLab', function () {
         const responseIssuereopened = { iid: 123 };
         const responseSetLabels = {
           iid: 123,
-          labels: ['location'],
+          labels: ['page access restriction'],
         };
         const responseAddcomment = { iid: 123, id: 23, body: ISSUE.description };
         const { iid } = GITLAB_RESPONSE_FOR_EXISTING_ISSUE;
@@ -471,7 +471,7 @@ describe('GitLab', function () {
             .reply(200, responseIssuereopened);
 
           setIssueLabelsScope = nock(gitlab.apiBaseURL)
-            .put(`/projects/${PROJECT_ID}/issues/${iid}`, { labels: ['location'] })
+            .put(`/projects/${PROJECT_ID}/issues/${iid}`, { labels: ['page access restriction'] })
             .reply(200, responseSetLabels);
 
           addCommentScope = nock(gitlab.apiBaseURL)
@@ -499,53 +499,170 @@ describe('GitLab', function () {
         let addCommentScope;
         let openIssueScope;
 
-        const GITLAB_RESPONSE_FOR_EXISTING_ISSUE = {
-          number: 123,
-          title: ISSUE.title,
-          description: ISSUE.description,
-          labels: [{ name: 'selectors' }],
-          state: GitLab.ISSUE_STATE_OPEN,
-        };
-
-        const EXPECTED_REQUEST_BODY = { state_event: 'reopen' };
-        const responseIssuereopened = { iid: 123 };
-        const responseSetLabels = {
-          iid: 123,
-          labels: ['location'],
-        };
-        const responseAddcomment = { iid: 123, id: 23, body: ISSUE.description };
-        const { iid } = GITLAB_RESPONSE_FOR_EXISTING_ISSUE;
-
-        before(async () => {
-          nock(gitlab.apiBaseURL)
-            .get(`/projects/${PROJECT_ID}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`)
-            .reply(200, [GITLAB_RESPONSE_FOR_EXISTING_ISSUE]);
-
-          openIssueScope = nock(gitlab.apiBaseURL)
-            .put(`/projects/${PROJECT_ID}/issues/${iid}`, EXPECTED_REQUEST_BODY)
-            .reply(200, responseIssuereopened);
-
-          setIssueLabelsScope = nock(gitlab.apiBaseURL)
-            .put(`/projects/${PROJECT_ID}/issues/${iid}`, { labels: ['location'] })
-            .reply(200, responseSetLabels);
-
-          addCommentScope = nock(gitlab.apiBaseURL)
-            .post(`/projects/${PROJECT_ID}/issues/${iid}/notes`, { body: ISSUE.description })
-            .reply(200, responseAddcomment);
-
-          await gitlab.createOrUpdateIssue(ISSUE);
-        });
-
-        it('does not change the issue state', () => {
-          expect(openIssueScope.isDone()).to.be.false;
+        context('when the reason is new', () => {
+          const GITLAB_RESPONSE_FOR_EXISTING_ISSUE = {
+            iid: 123,
+            title: ISSUE.title,
+            description: ISSUE.description,
+            labels: [{ name: 'empty content' }],
+            state: GitLab.ISSUE_STATE_OPEN,
+          };
+
+          const EXPECTED_REQUEST_BODY = { state_event: 'reopen' };
+          const responseIssuereopened = { iid: 123 };
+          const responseSetLabels = {
+            iid: 123,
+            labels: ['page access restriction'],
+          };
+          const responseAddcomment = { iid: 123, id: 23, body: ISSUE.description };
+          const { iid } = GITLAB_RESPONSE_FOR_EXISTING_ISSUE;
+
+          before(async () => {
+            nock(gitlab.apiBaseURL)
+              .get(`/projects/${PROJECT_ID}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`)
+              .reply(200, [GITLAB_RESPONSE_FOR_EXISTING_ISSUE]);
+
+            openIssueScope = nock(gitlab.apiBaseURL)
+              .put(`/projects/${PROJECT_ID}/issues/${iid}`, EXPECTED_REQUEST_BODY)
+              .reply(200, responseIssuereopened);
+
+            setIssueLabelsScope = nock(gitlab.apiBaseURL)
+              .put(`/projects/${PROJECT_ID}/issues/${iid}`, { labels: ['page access restriction'] })
+              .reply(200, responseSetLabels);
+
+            addCommentScope = nock(gitlab.apiBaseURL)
+              .post(`/projects/${PROJECT_ID}/issues/${iid}/notes`, { body: ISSUE.description })
+              .reply(200, responseAddcomment);
+
+            await gitlab.createOrUpdateIssue(ISSUE);
+          });
+
+          it('does not change the issue state', () => {
+            expect(openIssueScope.isDone()).to.be.false;
+          });
+
+          it("updates the issue's label", () => {
+            expect(setIssueLabelsScope.isDone()).to.be.true;
+          });
+
+          it('adds comment to the issue', () => {
+            expect(addCommentScope.isDone()).to.be.true;
+          });
         });
 
-        it("updates the issue's label", () => {
-          expect(setIssueLabelsScope.isDone()).to.be.true;
+        context('when all requested labels are already present', () => {
+          let setIssueLabelsScope;
+          let addCommentScope;
+          let openIssueScope;
+
+          const GITLAB_RESPONSE_FOR_EXISTING_ISSUE = {
+            iid: 123,
+            title: ISSUE.title,
+            description: ISSUE.description,
+            labels: [{ name: 'page access restriction' }, { name: 'server error' }],
+            state: GitLab.ISSUE_STATE_OPEN,
+          };
+
+          const { iid } = GITLAB_RESPONSE_FOR_EXISTING_ISSUE;
+
+          before(async () => {
+            nock(gitlab.apiBaseURL)
+              .get(`/projects/${PROJECT_ID}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`)
+              .reply(200, [GITLAB_RESPONSE_FOR_EXISTING_ISSUE]);
+
+            openIssueScope = nock(gitlab.apiBaseURL)
+              .put(`/projects/${PROJECT_ID}/issues/${iid}`)
+              .reply(200);
+
+            setIssueLabelsScope = nock(gitlab.apiBaseURL)
+              .put(`/projects/${PROJECT_ID}/issues/${iid}`)
+              .reply(200);
+
+            addCommentScope = nock(gitlab.apiBaseURL)
+              .post(`/projects/${PROJECT_ID}/issues/${iid}/notes`)
+              .reply(200);
+
+            await gitlab.createOrUpdateIssue({
+              title: ISSUE.title,
+              description: ISSUE.description,
+              labels: [ 'page access restriction', 'server error' ],
+            });
+          });
+
+          it('does not change the issue state', () => {
+            expect(openIssueScope.isDone()).to.be.false;
+          });
+
+          it('does not attempt to update the issue labels', () => {
+            expect(setIssueLabelsScope.isDone()).to.be.false;
+          });
+
+          it('does not attempt to add any comment to the issue', () => {
+            expect(addCommentScope.isDone()).to.be.false;
+          });
         });
 
-        it('adds comment to the issue', () => {
-          expect(addCommentScope.isDone()).to.be.true;
+        context('when some but not all requested labels are present', () => {
+          let setIssueLabelsScope;
+          let addCommentScope;
+          let openIssueScope;
+
+          before(() => {
+            nock.cleanAll();
+          });
+
+          const GITLAB_RESPONSE_FOR_EXISTING_ISSUE = {
+            iid: 123,
+            title: ISSUE.title,
+            description: ISSUE.description,
+            labels: [{ name: 'page access restriction' }],
+            state: GitLab.ISSUE_STATE_OPEN,
+          };
+
+          const EXPECTED_REQUEST_BODY = { state_event: 'reopen' };
+          const responseIssuereopened = { iid: 123 };
+          const responseSetLabels = {
+            iid: 123,
+            labels: [ 'page access restriction', 'empty content' ],
+          };
+          const responseAddcomment = { iid: 123, id: 23, body: ISSUE.description };
+          const { iid } = GITLAB_RESPONSE_FOR_EXISTING_ISSUE;
+
+          before(async () => {
+            nock(gitlab.apiBaseURL)
+              .get(`/projects/${PROJECT_ID}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`)
+              .reply(200, [GITLAB_RESPONSE_FOR_EXISTING_ISSUE]);
+
+            openIssueScope = nock(gitlab.apiBaseURL)
+              .put(`/projects/${PROJECT_ID}/issues/${iid}`, EXPECTED_REQUEST_BODY)
+              .reply(200, responseIssuereopened);
+
+            setIssueLabelsScope = nock(gitlab.apiBaseURL)
+              .put(`/projects/${PROJECT_ID}/issues/${iid}`, { labels: [ 'page access restriction', 'empty content' ] })
+              .reply(200, responseSetLabels);
+
+            addCommentScope = nock(gitlab.apiBaseURL)
+              .post(`/projects/${PROJECT_ID}/issues/${iid}/notes`, { body: ISSUE.description })
+              .reply(200, responseAddcomment);
+
+            await gitlab.createOrUpdateIssue({
+              title: ISSUE.title,
+              description: ISSUE.description,
+              labels: [ 'page access restriction', 'empty content' ],
+            });
+          });
+
+          it('does not change the issue state', () => {
+            expect(openIssueScope.isDone()).to.be.false;
+          });
+
+          it("updates the issue's labels", () => {
+            expect(setIssueLabelsScope.isDone()).to.be.true;
+          });
+
+          it('adds comment to the issue', () => {
+            expect(addCommentScope.isDone()).to.be.true;
+          });
         });
       });
     });

package/src/reporter/index.js

@@ -4,31 +4,32 @@ import { toISODateWithoutMilliseconds } from '../archivist/utils/date.js';
 import logger from '../logger/index.js';
 
 import { createReporter } from './factory.js';
+import { LABELS } from './labels.js';
 
 const CONTRIBUTION_TOOL_URL = 'https://contribute.opentermsarchive.org/en/service';
 const DOC_URL = 'https://docs.opentermsarchive.org';
 
-const ERROR_MESSAGE_TO_ISSUE_LABEL_MAP = {
-  'has no match': 'selectors',
-  'HTTP code 404': 'location',
-  'HTTP code 403': '403',
-  'HTTP code 429': '429',
-  'HTTP code 500': '500',
-  'HTTP code 502': '502',
-  'HTTP code 503': '503',
-  'Timed out after': 'timeout',
-  EAI_AGAIN: 'EAI_AGAIN',
-  ENOTFOUND: 'ENOTFOUND',
-  'Response is empty': 'empty response',
-  'unable to verify the first certificate': 'first certificate',
-  'certificate has expired': 'certificate expired',
-  'maximum redirect reached': 'redirects',
-  'not a valid selector': 'invalid selector',
-  'empty content': 'empty content',
+const ERROR_MESSAGE_TO_ISSUE_LABELS_MAP = {
+  'has no match': [ LABELS.PAGE_STRUCTURE_CHANGE.name, LABELS.NEEDS_INTERVENTION.name ],
+  'HTTP code 404': [ LABELS.PAGE_NOT_FOUND.name, LABELS.NEEDS_INTERVENTION.name ],
+  'HTTP code 403': [LABELS.HTTP_403.name],
+  'HTTP code 429': [LABELS.HTTP_429.name],
+  'HTTP code 500': [LABELS.HTTP_500.name],
+  'HTTP code 502': [LABELS.HTTP_502.name],
+  'HTTP code 503': [LABELS.HTTP_503.name],
+  'Timed out after': [LABELS.PAGE_LOAD_TIMEOUT.name],
+  EAI_AGAIN: [LABELS.DNS_LOOKUP_FAILURE.name],
+  ENOTFOUND: [LABELS.DNS_RESOLUTION_FAILURE.name],
+  'Response is empty': [LABELS.EMPTY_RESPONSE.name],
+  'unable to verify the first certificate': [LABELS.SSL_INVALID.name],
+  'certificate has expired': [LABELS.SSL_EXPIRED.name],
+  'maximum redirect reached': [LABELS.TOO_MANY_REDIRECTS.name],
+  'not a valid selector': [LABELS.INVALID_SELECTOR.name],
+  'empty content': [LABELS.EMPTY_CONTENT.name],
 };
 
-function getLabelNameFromError(error) {
-  return ERROR_MESSAGE_TO_ISSUE_LABEL_MAP[Object.keys(ERROR_MESSAGE_TO_ISSUE_LABEL_MAP).find(substring => error.toString().includes(substring))] || 'to clarify';
+function getLabelNamesFromError(error) {
+  return ERROR_MESSAGE_TO_ISSUE_LABELS_MAP[Object.keys(ERROR_MESSAGE_TO_ISSUE_LABELS_MAP).find(substring => error.toString().includes(substring))] || [LABELS.UNKNOWN_FAILURE.name];
 }
 
 // In the following class, it is assumed that each issue is managed using its title as a unique identifier
@@ -118,7 +119,7 @@ No changes were found in the last run, so no new version has been recorded.`,
     await this.reporter.createOrUpdateIssue({
       title: Reporter.generateTitleID(terms.service.id, terms.type),
       description: this.generateDescription({ error, terms }),
-      label: getLabelNameFromError(error),
+      labels: getLabelNamesFromError(error),
     });
   }
 

package/src/reporter/labels.js

@@ -0,0 +1,96 @@
+export const DEPRECATED_MANAGED_BY_OTA_MARKER = '[managed by OTA]';
+
+export const MANAGED_BY_OTA_MARKER = '- Auto-managed by OTA engine';
+
+export const LABELS = {
+  HTTP_403: {
+    name: 'page access restriction',
+    color: 'FFFFFF',
+    description: 'Fetching failed with a 403 (forbidden) HTTP code',
+  },
+  HTTP_429: {
+    name: 'request limit exceeded',
+    color: 'FFFFFF',
+    description: 'Fetching failed with a 429 (too many requests) HTTP code',
+  },
+  HTTP_500: {
+    name: 'server error',
+    color: 'FFFFFF',
+    description: 'Fetching failed with a 500 (internal server error) HTTP code',
+  },
+  HTTP_502: {
+    name: 'server response failure',
+    color: 'FFFFFF',
+    description: 'Fetching failed with a 502 (bad gateway) HTTP code',
+  },
+  HTTP_503: {
+    name: 'server unavailability',
+    color: 'FFFFFF',
+    description: 'Fetching failed with a 503 (service unavailable) HTTP code',
+  },
+  SSL_EXPIRED: {
+    name: 'SSL certificate expiration',
+    color: 'FFFFFF',
+    description: 'Fetching failed because the domain SSL certificate has expired',
+  },
+  DNS_LOOKUP_FAILURE: {
+    name: 'DNS lookup failure',
+    color: 'FFFFFF',
+    description: 'Fetching failed because the domain temporarily failed to resolve on DNS',
+  },
+  DNS_RESOLUTION_FAILURE: {
+    name: 'DNS resolution failure',
+    color: 'FFFFFF',
+    description: 'Fetching failed because the domain fails to resolve on DNS',
+  },
+  EMPTY_RESPONSE: {
+    name: 'empty response',
+    color: 'FFFFFF',
+    description: 'Fetching failed because server returned an empty response body',
+  },
+  SSL_INVALID: {
+    name: 'SSL certificate invalidity',
+    color: 'FFFFFF',
+    description: 'Fetching failed because SSL certificate verification failed',
+  },
+  TOO_MANY_REDIRECTS: {
+    name: 'too many redirects',
+    color: 'FFFFFF',
+    description: 'Fetching failed because too many redirects detected',
+  },
+  PAGE_LOAD_TIMEOUT: {
+    name: 'page load timeout',
+    color: 'FFFFFF',
+    description: 'Fetching failed with a timeout error',
+  },
+  EMPTY_CONTENT: {
+    name: 'empty content',
+    color: 'FFFFFF',
+    description: 'Fetching failed because the server returns an empty content',
+  },
+  UNKNOWN_FAILURE: {
+    name: 'unknown failure reason',
+    color: 'FFFFFF',
+    description: 'Fetching failed for an undetermined reason that needs investigation',
+  },
+  PAGE_STRUCTURE_CHANGE: {
+    name: 'page structure change',
+    color: 'FFFFFF',
+    description: 'Extraction selectors are outdated',
+  },
+  PAGE_NOT_FOUND: {
+    name: 'page not found',
+    color: 'FFFFFF',
+    description: 'Fetch location is outdated',
+  },
+  INVALID_SELECTOR: {
+    name: 'invalid selector',
+    color: 'FFFFFF',
+    description: 'Some selectors cannot be understood by the engine',
+  },
+  NEEDS_INTERVENTION: {
+    name: '⚠ needs intervention',
+    color: 'FDD42A',
+    description: 'Requires contributors intervention to restore tracking',
+  },
+};

package/src/reporter/labels.test.js

@@ -0,0 +1,24 @@
+import chai from 'chai';
+
+import GitHub from './github/index.js';
+import GitLab from './gitlab/index.js';
+import { MANAGED_BY_OTA_MARKER, LABELS } from './labels.js';
+
+const { expect } = chai;
+
+const MAX_LABEL_DESCRIPTION_LENGTH = Math.min(
+  GitHub.MAX_LABEL_DESCRIPTION_LENGTH,
+  GitLab.MAX_LABEL_DESCRIPTION_LENGTH,
+);
+
+describe('Reporter GitHub labels', () => {
+  Object.values(LABELS).forEach(label => {
+    describe(`"${label.name}"`, () => {
+      it('complies with the max description length of supported platforms', () => {
+        const descriptionLength = label.description.length + MANAGED_BY_OTA_MARKER.length;
+
+        expect(descriptionLength).to.be.lessThan(MAX_LABEL_DESCRIPTION_LENGTH);
+      });
+    });
+  });
+});

package/src/reporter/github/labels.json

@@ -1,87 +0,0 @@
-[
-  {
-    "name": "403",
-    "color": "0b08a0",
-    "description": "Fetching fails with a 403 (forbidden) HTTP code"
-  },
-  {
-    "name": "429",
-    "color": "0b08a0",
-    "description": "Fetching fails with a 429 (too many requests) HTTP code"
-  },
-  {
-    "name": "500",
-    "color": "0b08a0",
-    "description": "Fetching fails with a 500 (internal server error) HTTP code"
-  },
-  {
-    "name": "502",
-    "color": "0b08a0",
-    "description": "Fetching fails with a 502 (bad gateway) HTTP code"
-  },
-  {
-    "name": "503",
-    "color": "0b08a0",
-    "description": "Fetching fails with a 503 (service unavailable) HTTP code"
-  },
-  {
-    "name": "certificate expired",
-    "color": "0b08a0",
-    "description": "Fetching fails because the domain SSL certificate has expired"
-  },
-  {
-    "name": "EAI_AGAIN",
-    "color": "0b08a0",
-    "description": "Fetching fails because the domain fails to resolve on DNS"
-  },
-  {
-    "name": "ENOTFOUND",
-    "color": "0b08a0",
-    "description": "Fetching fails because the domain fails to resolve on DNS"
-  },
-  {
-    "name": "empty response",
-    "color": "0b08a0",
-    "description": "Fetching fails with a “response is empty” error"
-  },
-  {
-    "name": "first certificate",
-    "color": "0b08a0",
-    "description": "Fetching fails with an “unable to verify the first certificate” error"
-  },
-  {
-    "name": "redirects",
-    "color": "0b08a0",
-    "description": "Fetching fails with a “too many redirects” error"
-  },
-  {
-    "name": "timeout",
-    "color": "0b08a0",
-    "description": "Fetching fails with a timeout error"
-  },
-  {
-    "name": "empty content",
-    "color": "0b08a0",
-    "description": "Fetching fails because the server returns an empty content"
-  },
-  {
-    "name": "to clarify",
-    "color": "0496ff",
-    "description": "Default failure label"
-  },
-  {
-    "name": "selectors",
-    "color": "FBCA04",
-    "description": "Extraction selectors are outdated"
-  },
-  {
-    "name": "location",
-    "color": "FBCA04",
-    "description": "Fetch location is outdated"
-  },
-  {
-    "name": "invalid selector",
-    "color": "FBCA04",
-    "description": "Some selectors cannot be understood by the engine"
-  }
-]

package/src/reporter/github/labels.test.js

@@ -1,30 +0,0 @@
-import { createRequire } from 'module';
-
-import chai from 'chai';
-
-import { MANAGED_BY_OTA_MARKER } from './index.js';
-
-const require = createRequire(import.meta.url);
-
-const { expect } = chai;
-const labels = require('./labels.json');
-
-const GITHUB_LABEL_DESCRIPTION_MAX_LENGTH = 100;
-
-describe('Reporter GitHub labels', () => {
-  labels.forEach(label => {
-    describe(`"${label.name}"`, () => {
-      it('complies with the GitHub character limit for descriptions', () => {
-        const descriptionLength = label.description.length + MANAGED_BY_OTA_MARKER.length;
-
-        expect(descriptionLength).to.be.lessThan(GITHUB_LABEL_DESCRIPTION_MAX_LENGTH);
-      });
-
-      it('complies with the GitHub constraints for color', () => {
-        const validHexColorRegex = /^[0-9a-fA-F]{6}$/; // Regex for a valid 6-digit hexadecimal color code without the `#`
-
-        expect(validHexColorRegex.test(label.color)).to.be.true;
-      });
-    });
-  });
-});

package/src/reporter/gitlab/labels.json

@@ -1,77 +0,0 @@
-[
-  {
-    "name": "403",
-    "color": "#0b08a0",
-    "description": "Fetching fails with a 403 (forbidden) HTTP code"
-  },
-  {
-    "name": "429",
-    "color": "#0b08a0",
-    "description": "Fetching fails with a 429 (too many requests) HTTP code"
-  },
-  {
-    "name": "500",
-    "color": "#0b08a0",
-    "description": "Fetching fails with a 500 (internal server error) HTTP code"
-  },
-  {
-    "name": "502",
-    "color": "#0b08a0",
-    "description": "Fetching fails with a 502 (bad gateway) HTTP code"
-  },
-  {
-    "name": "503",
-    "color": "#0b08a0",
-    "description": "Fetching fails with a 503 (service unavailable) HTTP code"
-  },
-  {
-    "name": "certificate expired",
-    "color": "#0b08a0",
-    "description": "Fetching fails because the domain SSL certificate has expired"
-  },
-  {
-    "name": "EAI_AGAIN",
-    "color": "#0b08a0",
-    "description": "Fetching fails because the domain fails to resolve on DNS"
-  },
-  {
-    "name": "ENOTFOUND",
-    "color": "#0b08a0",
-    "description": "Fetching fails because the domain fails to resolve on DNS"
-  },
-  {
-    "name": "empty response",
-    "color": "#0b08a0",
-    "description": "Fetching fails with a “response is empty” error"
-  },
-  {
-    "name": "first certificate",
-    "color": "#0b08a0",
-    "description": "Fetching fails with an “unable to verify the first certificate” error"
-  },
-  {
-    "name": "redirects",
-    "color": "#0b08a0",
-    "description": "Fetching fails with a “too many redirects” error"
-  },
-  {
-    "name": "timeout",
-    "color": "#0b08a0",
-    "description": "Fetching fails with a timeout error"
-  },
-  {
-    "name": "to clarify",
-    "color": "#0496ff",
-    "description": "Default failure label"
-  },
-  {
-    "name": "selectors",
-    "color": "#FBCA04",
-    "description": "Extraction selectors are outdated"
-  },
-  {
-    "name": "location",
-    "color": "#FBCA04",
-    "description": "Fetch location is outdated"
-  }
-]

package/src/reporter/gitlab/labels.test.js

@@ -1,30 +0,0 @@
-import { createRequire } from 'module';
-
-import chai from 'chai';
-
-import { MANAGED_BY_OTA_MARKER } from './index.js';
-
-const require = createRequire(import.meta.url);
-
-const { expect } = chai;
-const labels = require('./labels.json');
-
-const GITLAB_LABEL_DESCRIPTION_MAX_LENGTH = 255;
-
-describe('Reporter GitLab labels', () => {
-  labels.forEach(label => {
-    describe(`"${label.name}"`, () => {
-      it('complies with the GitLab character limit for descriptions', () => {
-        const descriptionLength = label.description.length + MANAGED_BY_OTA_MARKER.length;
-
-        expect(descriptionLength).to.be.lessThan(GITLAB_LABEL_DESCRIPTION_MAX_LENGTH);
-      });
-
-      it('complies with the GitLab constraints for color', () => {
-        const validHexColorRegex = /^#[0-9a-fA-F]{6}$/;
-
-        expect(validHexColorRegex.test(label.color)).to.be.true;
-      });
-    });
-  });
-});