@opentermsarchive/engine 1.0.0 → 1.1.1

package/.eslintrc.yaml

@@ -100,6 +100,7 @@ rules:
   new-cap:
     - error
     - properties: false
+  require-await: 1
 
 overrides:
   - files:

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "1.0.0",
+  "version": "1.1.1",
   "description": "Tracks and makes visible changes to the terms of online services",
-  "homepage": "https://github.com/OpenTermsArchive/engine#readme",
+  "homepage": "https://opentermsarchive.org",
   "bugs": {
     "url": "https://github.com/OpenTermsArchive/engine/issues"
   },
@@ -11,7 +11,7 @@
     "url": "git+https://github.com/OpenTermsArchive/engine.git"
   },
   "license": "EUPL-1.2",
-  "author": "ambanum",
+  "author": "Open Terms Archive contributors",
   "type": "module",
   "exports": {
     ".": "./src/exports.js",

package/src/archivist/index.js

@@ -29,6 +29,7 @@ export const EVENTS = [
   'trackingCompleted',
   'inaccessibleContent',
   'error',
+  'pluginError',
 ];
 
 export default class Archivist extends events.EventEmitter {
@@ -72,10 +73,6 @@ export default class Archivist extends events.EventEmitter {
   initQueue() {
     this.trackingQueue = async.queue(this.trackTermsChanges.bind(this), MAX_PARALLEL_TRACKING);
     this.trackingQueue.error(async (error, { terms }) => {
-      if (error.toString().includes('HttpError: API rate limit exceeded for user ID')) {
-        return; // This is an error due to SendInBlue quota, bypass
-      }
-
       if (error instanceof InaccessibleContentError) {
         this.emit('inaccessibleContent', error, terms);
 
@@ -91,7 +88,13 @@ export default class Archivist extends events.EventEmitter {
       const handlerName = `on${event[0].toUpperCase()}${event.substring(1)}`;
 
       if (listener[handlerName]) {
-        this.on(event, listener[handlerName].bind(listener));
+        this.on(event, async (...params) => {
+          try {
+            await listener[handlerName](...params); // Prefer try...catch over .catch() for handling errors to account for both synchronous and asynchronous functions, as .catch() cannot be applied to synchronous functions
+          } catch (error) {
+            this.emit('pluginError', error, listener.constructor.name);
+          }
+        });
       }
     });
   }

package/src/archivist/index.test.js

@@ -245,6 +245,80 @@ describe('Archivist', function () {
     });
   });
 
+  describe('Plugin system', () => {
+    const plugin = {};
+
+    describe('#attach', () => {
+      before(async () => {
+        app = new Archivist({
+          recorderConfig: config.get('recorder'),
+          fetcherConfig: config.get('fetcher'),
+        });
+        await app.initialize();
+
+        EVENTS.forEach(event => {
+          const handlerName = `on${event[0].toUpperCase()}${event.substring(1)}`;
+
+          plugin[handlerName] = sinon.spy();
+        });
+
+        app.removeAllListeners('error');
+        expect(app.eventNames()).to.be.empty;
+        app.attach(plugin);
+      });
+
+      EVENTS.forEach(event => {
+        const handlerName = `on${event[0].toUpperCase()}${event.substring(1)}`;
+
+        it(`attaches plugin "${event}" handler`, () => {
+          app.emit(event);
+          expect(plugin[handlerName].calledOnce).to.be.true;
+        });
+      });
+    });
+
+    context('when errors occur within a plugin', () => {
+      let error;
+      let listeners;
+      let plugin;
+
+      before(async () => {
+        nock.cleanAll();
+        nock('https://www.servicea.example').get('/tos').reply(200, serviceASnapshotExpectedContent, { 'Content-Type': 'text/html' });
+
+        app = new Archivist({
+          recorderConfig: config.get('recorder'),
+          fetcherConfig: config.get('fetcher'),
+        });
+        await app.initialize();
+
+        plugin = { onFirstVersionRecorded: () => { throw new Error('Plugin error'); } };
+
+        listeners = process.listeners('unhandledRejection'); // back up listeners
+        process.removeAllListeners('unhandledRejection'); // remove all listeners to avoid exit the process
+
+        process.on('unhandledRejection', reason => { error = reason; });
+
+        app.attach(plugin);
+      });
+
+      after(async () => {
+        process.removeAllListeners('unhandledRejection');
+        listeners.forEach(listener => process.on('unhandledRejection', listener));
+        await resetGitRepositories();
+      });
+
+      it('does not crash the tracking process', done => {
+        app.track({ services: [services[0]] }).then(() => {
+          if (error) {
+            return done(error);
+          }
+          done();
+        });
+      });
+    });
+  });
+
   describe('events', () => {
     const spies = {};
 

package/src/index.js

@@ -39,17 +39,25 @@ export default async function track({ services, types, extractOnly, schedule })
   }
 
   if (process.env.SENDINBLUE_API_KEY) {
-    archivist.attach(new Notifier(archivist.services));
+    try {
+      archivist.attach(new Notifier(archivist.services));
+    } catch (error) {
+      logger.error('Cannot instantiate the Notifier module; it will be ignored:', error);
+    }
   } else {
     logger.warn('Environment variable "SENDINBLUE_API_KEY" was not found; the Notifier module will be ignored');
   }
 
   if (process.env.GITHUB_TOKEN) {
     if (config.has('reporter.githubIssues.repositories.declarations')) {
-      const reporter = new Reporter(config.get('reporter'));
+      try {
+        const reporter = new Reporter(config.get('reporter'));
 
-      await reporter.initialize();
-      archivist.attach(reporter);
+        await reporter.initialize();
+        archivist.attach(reporter);
+      } catch (error) {
+        logger.error('Cannot instantiate the Reporter module; it will be ignored:', error);
+      }
     } else {
       logger.warn('Configuration key "reporter.githubIssues.repositories.declarations" was not found; issues on the declarations repository cannot be created');
     }

package/src/logger/index.js

@@ -136,4 +136,8 @@ logger.onError = (error, terms) => {
   logger.error({ message: error.stack, serviceId: terms.serviceId, termsType: terms.type });
 };
 
+logger.onPluginError = (error, pluginName) => {
+  logger.error({ message: `Error in "${pluginName}" plugin: ${error.stack}` });
+};
+
 export default logger;

package/src/reporter/github.js

@@ -178,7 +178,7 @@ export default class GitHub {
       return this.createIssue({ title, description, labels: [label] });
     }
 
-    if (issue.state == this.ISSUE_STATE_CLOSED) {
+    if (issue.state == GitHub.ISSUE_STATE_CLOSED) {
       await this.openIssue(issue);
     }
 

package/src/reporter/github.test.js

@@ -0,0 +1,473 @@
+import { createRequire } from 'module';
+
+import { expect } from 'chai';
+import nock from 'nock';
+
+import GitHub from './github.js';
+
+const require = createRequire(import.meta.url);
+
+describe('GitHub', function () {
+  this.timeout(5000);
+
+  let MANAGED_LABELS;
+  let github;
+
+  before(() => {
+    MANAGED_LABELS = require('./labels.json');
+    github = new GitHub('owner/repo');
+  });
+
+  describe('#initialize', () => {
+    const scopes = [];
+
+    before(async () => {
+      const existingLabels = MANAGED_LABELS.slice(0, -2);
+
+      nock('https://api.github.com')
+        .get('/repos/owner/repo/labels')
+        .reply(200, existingLabels);
+
+      const missingLabels = MANAGED_LABELS.slice(-2);
+
+      for (const label of missingLabels) {
+        scopes.push(nock('https://api.github.com')
+          .post('/repos/owner/repo/labels', body => body.name === label.name)
+          .reply(200, label));
+      }
+
+      await github.initialize();
+    });
+
+    after(nock.cleanAll);
+
+    it('should create missing labels', () => {
+      scopes.forEach(scope => expect(scope.isDone()).to.be.true);
+    });
+  });
+
+  describe('#getRepositoryLabels', () => {
+    let scope;
+    let result;
+    const LABELS = [{ name: 'bug' }, { name: 'enhancement' }];
+
+    before(async () => {
+      scope = nock('https://api.github.com')
+        .get('/repos/owner/repo/labels')
+        .reply(200, LABELS);
+
+      result = await github.getRepositoryLabels();
+    });
+
+    after(nock.cleanAll);
+
+    it('fetches repository labels', () => {
+      expect(scope.isDone()).to.be.true;
+    });
+
+    it('returns the repository labels', () => {
+      expect(result).to.deep.equal(LABELS);
+    });
+  });
+
+  describe('#createLabel', () => {
+    let scope;
+    const LABEL = { name: 'new_label', color: 'ffffff' };
+
+    before(async () => {
+      scope = nock('https://api.github.com')
+        .post('/repos/owner/repo/labels', body => body.name === LABEL.name)
+        .reply(200, LABEL);
+
+      await github.createLabel(LABEL);
+    });
+
+    after(nock.cleanAll);
+
+    it('creates the new label', () => {
+      expect(scope.isDone()).to.be.true;
+    });
+  });
+
+  describe('#createIssue', () => {
+    let scope;
+    let result;
+    const ISSUE = {
+      title: 'New Issue',
+      description: 'Description of the new issue',
+      labels: ['bug'],
+    };
+    const CREATED_ISSUE = { number: 123, ...ISSUE };
+
+    before(async () => {
+      scope = nock('https://api.github.com')
+        .post('/repos/owner/repo/issues', request => request.title === ISSUE.title && request.body === ISSUE.description && request.labels[0] === ISSUE.labels[0])
+        .reply(200, CREATED_ISSUE);
+
+      result = await github.createIssue(ISSUE);
+    });
+
+    after(nock.cleanAll);
+
+    it('creates the new issue', () => {
+      expect(scope.isDone()).to.be.true;
+    });
+
+    it('returns the created issue', () => {
+      expect(result).to.deep.equal(CREATED_ISSUE);
+    });
+  });
+
+  describe('#setIssueLabels', () => {
+    let scope;
+    const ISSUE_NUMBER = 123;
+    const LABELS = [ 'bug', 'enhancement' ];
+
+    before(async () => {
+      scope = nock('https://api.github.com')
+        .put(`/repos/owner/repo/issues/${ISSUE_NUMBER}/labels`, { labels: LABELS })
+        .reply(200);
+
+      await github.setIssueLabels({ issue: { number: ISSUE_NUMBER }, labels: LABELS });
+    });
+
+    after(nock.cleanAll);
+
+    it('sets labels on the issue', () => {
+      expect(scope.isDone()).to.be.true;
+    });
+  });
+
+  describe('#openIssue', () => {
+    let scope;
+    const ISSUE = { number: 123 };
+    const EXPECTED_REQUEST_BODY = { state: 'open' };
+
+    before(async () => {
+      scope = nock('https://api.github.com')
+        .patch(`/repos/owner/repo/issues/${ISSUE.number}`, EXPECTED_REQUEST_BODY)
+        .reply(200);
+
+      await github.openIssue(ISSUE);
+    });
+
+    after(nock.cleanAll);
+
+    it('opens the issue', () => {
+      expect(scope.isDone()).to.be.true;
+    });
+  });
+
+  describe('#closeIssue', () => {
+    let scope;
+    const ISSUE = { number: 123 };
+    const EXPECTED_REQUEST_BODY = { state: 'closed' };
+
+    before(async () => {
+      scope = nock('https://api.github.com')
+        .patch(`/repos/owner/repo/issues/${ISSUE.number}`, EXPECTED_REQUEST_BODY)
+        .reply(200);
+
+      await github.closeIssue(ISSUE);
+    });
+
+    after(nock.cleanAll);
+
+    it('closes the issue', () => {
+      expect(scope.isDone()).to.be.true;
+    });
+  });
+
+  describe('#getIssue', () => {
+    let scope;
+    let result;
+
+    const ISSUE = { number: 123, title: 'Test Issue' };
+    const ANOTHER_ISSUE = { number: 124, title: 'Test Issue 2' };
+
+    before(async () => {
+      scope = nock('https://api.github.com')
+        .get('/repos/owner/repo/issues')
+        .query(true)
+        .reply(200, [ ISSUE, ANOTHER_ISSUE ]);
+
+      result = await github.getIssue({ title: ISSUE.title });
+    });
+
+    after(nock.cleanAll);
+
+    it('searches for the issue', () => {
+      expect(scope.isDone()).to.be.true;
+    });
+
+    it('returns the expected issue', () => {
+      expect(result).to.deep.equal(ISSUE);
+    });
+  });
+
+  describe('#addCommentToIssue', () => {
+    let scope;
+    const ISSUE_NUMBER = 123;
+    const COMMENT = 'Test comment';
+
+    before(async () => {
+      scope = nock('https://api.github.com')
+        .post(`/repos/owner/repo/issues/${ISSUE_NUMBER}/comments`, { body: COMMENT })
+        .reply(200);
+
+      await github.addCommentToIssue({ issue: { number: ISSUE_NUMBER }, comment: COMMENT });
+    });
+
+    after(nock.cleanAll);
+
+    it('adds the comment to the issue', () => {
+      expect(scope.isDone()).to.be.true;
+    });
+  });
+
+  describe('#closeIssueWithCommentIfExists', () => {
+    after(nock.cleanAll);
+
+    context('when the issue exists and is open', () => {
+      const ISSUE = {
+        number: 123,
+        title: 'Open Issue',
+        state: GitHub.ISSUE_STATE_OPEN,
+      };
+      let addCommentScope;
+      let closeIssueScope;
+
+      before(async () => {
+        nock('https://api.github.com')
+          .get('/repos/owner/repo/issues')
+          .query(true)
+          .reply(200, [ISSUE]);
+
+        addCommentScope = nock('https://api.github.com')
+          .post(`/repos/owner/repo/issues/${ISSUE.number}/comments`)
+          .reply(200);
+
+        closeIssueScope = nock('https://api.github.com')
+          .patch(`/repos/owner/repo/issues/${ISSUE.number}`, { state: GitHub.ISSUE_STATE_CLOSED })
+          .reply(200);
+
+        await github.closeIssueWithCommentIfExists({ title: ISSUE.title, comment: 'Closing comment' });
+      });
+
+      it('adds comment to the issue', () => {
+        expect(addCommentScope.isDone()).to.be.true;
+      });
+
+      it('closes the issue', () => {
+        expect(closeIssueScope.isDone()).to.be.true;
+      });
+    });
+
+    context('when the issue exists and is closed', () => {
+      const ISSUE = {
+        number: 123,
+        title: 'Closed Issue',
+        state: GitHub.ISSUE_STATE_CLOSED,
+      };
+      let addCommentScope;
+      let closeIssueScope;
+
+      before(async () => {
+        nock('https://api.github.com')
+          .get('/repos/owner/repo/issues')
+          .query(true)
+          .reply(200, []);
+
+        addCommentScope = nock('https://api.github.com')
+          .post(`/repos/owner/repo/issues/${ISSUE.number}/comments`)
+          .reply(200);
+
+        closeIssueScope = nock('https://api.github.com')
+          .patch(`/repos/owner/repo/issues/${ISSUE.number}`, { state: GitHub.ISSUE_STATE_CLOSED })
+          .reply(200);
+
+        await github.closeIssueWithCommentIfExists({ title: ISSUE.title, comment: 'Closing comment' });
+      });
+
+      it('does not add comment', () => {
+        expect(addCommentScope.isDone()).to.be.false;
+      });
+
+      it('does not attempt to close the issue', () => {
+        expect(closeIssueScope.isDone()).to.be.false;
+      });
+    });
+
+    context('when the issue does not exist', () => {
+      let addCommentScope;
+      let closeIssueScope;
+
+      before(async () => {
+        nock('https://api.github.com')
+          .get('/repos/owner/repo/issues')
+          .query(true)
+          .reply(200, []);
+
+        addCommentScope = nock('https://api.github.com')
+          .post(/\/repos\/owner\/repo\/issues\/\d+\/comments/)
+          .reply(200);
+
+        closeIssueScope = nock('https://api.github.com')
+          .patch(/\/repos\/owner\/repo\/issues\/\d+/, { state: GitHub.ISSUE_STATE_CLOSED })
+          .reply(200);
+
+        await github.closeIssueWithCommentIfExists({ title: 'Non-existent Issue', comment: 'Closing comment' });
+      });
+
+      it('does not attempt to add comment', () => {
+        expect(addCommentScope.isDone()).to.be.false;
+      });
+
+      it('does not attempt to close the issue', () => {
+        expect(closeIssueScope.isDone()).to.be.false;
+      });
+    });
+  });
+
+  describe('#createOrUpdateIssue', () => {
+    before(async () => {
+      nock('https://api.github.com')
+        .get('/repos/owner/repo/labels')
+        .reply(200, MANAGED_LABELS);
+
+      await github.initialize();
+    });
+
+    context('when the issue does not exist', () => {
+      let createIssueScope;
+      const ISSUE_TO_CREATE = {
+        title: 'New Issue',
+        description: 'Description of the new issue',
+        label: 'bug',
+      };
+
+      before(async () => {
+        nock('https://api.github.com')
+          .get('/repos/owner/repo/issues')
+          .query(true)
+          .reply(200, []); // Simulate that there is no issues on the repository
+
+        createIssueScope = nock('https://api.github.com')
+          .post('/repos/owner/repo/issues', {
+            title: ISSUE_TO_CREATE.title,
+            body: ISSUE_TO_CREATE.description,
+            labels: [ISSUE_TO_CREATE.label],
+          })
+          .reply(200, { number: 123 });
+
+        await github.createOrUpdateIssue(ISSUE_TO_CREATE);
+      });
+
+      it('creates the issue', () => {
+        expect(createIssueScope.isDone()).to.be.true;
+      });
+    });
+
+    context('when the issue already exists', () => {
+      const ISSUE = {
+        title: 'Existing Issue',
+        description: 'New comment',
+        label: 'location',
+      };
+
+      context('when issue is closed', () => {
+        let setIssueLabelsScope;
+        let addCommentScope;
+        let openIssueScope;
+
+        const GITHUB_RESPONSE_FOR_EXISTING_ISSUE = {
+          number: 123,
+          title: ISSUE.title,
+          description: ISSUE.description,
+          labels: [{ name: 'selectors' }],
+          state: GitHub.ISSUE_STATE_CLOSED,
+        };
+
+        before(async () => {
+          nock('https://api.github.com')
+            .get('/repos/owner/repo/issues')
+            .query(true)
+            .reply(200, [GITHUB_RESPONSE_FOR_EXISTING_ISSUE]);
+
+          openIssueScope = nock('https://api.github.com')
+            .patch(`/repos/owner/repo/issues/${GITHUB_RESPONSE_FOR_EXISTING_ISSUE.number}`, { state: GitHub.ISSUE_STATE_OPEN })
+            .reply(200);
+
+          setIssueLabelsScope = nock('https://api.github.com')
+            .put(`/repos/owner/repo/issues/${GITHUB_RESPONSE_FOR_EXISTING_ISSUE.number}/labels`, { labels: ['location'] })
+            .reply(200);
+
+          addCommentScope = nock('https://api.github.com')
+            .post(`/repos/owner/repo/issues/${GITHUB_RESPONSE_FOR_EXISTING_ISSUE.number}/comments`, { body: ISSUE.description })
+            .reply(200);
+
+          await github.createOrUpdateIssue(ISSUE);
+        });
+
+        it('reopens the issue', () => {
+          expect(openIssueScope.isDone()).to.be.true;
+        });
+
+        it("updates the issue's label", () => {
+          expect(setIssueLabelsScope.isDone()).to.be.true;
+        });
+
+        it('adds comment to the issue', () => {
+          expect(addCommentScope.isDone()).to.be.true;
+        });
+      });
+
+      context('when issue is already opened', () => {
+        let setIssueLabelsScope;
+        let addCommentScope;
+        let openIssueScope;
+
+        const GITHUB_RESPONSE_FOR_EXISTING_ISSUE = {
+          number: 123,
+          title: ISSUE.title,
+          description: ISSUE.description,
+          labels: [{ name: 'selectors' }],
+          state: GitHub.ISSUE_STATE_OPEN,
+        };
+
+        before(async () => {
+          nock('https://api.github.com')
+            .get('/repos/owner/repo/issues')
+            .query(true)
+            .reply(200, [GITHUB_RESPONSE_FOR_EXISTING_ISSUE]);
+
+          openIssueScope = nock('https://api.github.com')
+            .patch(`/repos/owner/repo/issues/${GITHUB_RESPONSE_FOR_EXISTING_ISSUE.number}`, { state: GitHub.ISSUE_STATE_OPEN })
+            .reply(200);
+
+          setIssueLabelsScope = nock('https://api.github.com')
+            .put(`/repos/owner/repo/issues/${GITHUB_RESPONSE_FOR_EXISTING_ISSUE.number}/labels`, { labels: ['location'] })
+            .reply(200);
+
+          addCommentScope = nock('https://api.github.com')
+            .post(`/repos/owner/repo/issues/${GITHUB_RESPONSE_FOR_EXISTING_ISSUE.number}/comments`, { body: ISSUE.description })
+            .reply(200);
+
+          await github.createOrUpdateIssue(ISSUE);
+        });
+
+        it('does not change the issue state', () => {
+          expect(openIssueScope.isDone()).to.be.false;
+        });
+
+        it("updates the issue's label", () => {
+          expect(setIssueLabelsScope.isDone()).to.be.true;
+        });
+
+        it('adds comment to the issue', () => {
+          expect(addCommentScope.isDone()).to.be.true;
+        });
+      });
+    });
+  });
+});