@opentermsarchive/engine 1.2.0 → 1.2.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "1.2.0",
+  "version": "1.2.2",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {
@@ -71,6 +71,7 @@
     "eslint-plugin-chai-friendly": "^0.7.2",
     "eslint-plugin-import": "^2.29.0",
     "eslint-plugin-json-format": "^2.0.1",
+    "eslint-plugin-no-only-tests": "^3.1.0",
     "express": "^4.18.2",
     "express-async-errors": "^3.1.1",
     "fs-extra": "^10.0.0",
@@ -90,9 +91,9 @@
     "node-fetch": "^3.1.0",
     "octokit": "^1.7.0",
     "pdfjs-dist": "^2.9.359",
-    "puppeteer": "14.3.0",
-    "puppeteer-extra": "^3.2.3",
-    "puppeteer-extra-plugin-stealth": "^2.9.0",
+    "puppeteer": "^22.8.1",
+    "puppeteer-extra": "^3.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2",
     "sib-api-v3-sdk": "^8.2.1",
     "simple-git": "^3.8.0",
     "swagger-jsdoc": "^6.2.8",
@@ -103,7 +104,6 @@
   "devDependencies": {
     "@commitlint/cli": "^19.0.3",
     "dir-compare": "^4.0.0",
-    "eslint-plugin-no-only-tests": "^3.1.0",
     "keep-a-changelog": "^2.5.3",
     "nock": "^13.2.1",
     "node-stream-zip": "^1.15.0",

package/scripts/declarations/lint/index.mocha.js

@@ -36,7 +36,7 @@ export default async options => {
 
   const lintFile = lintAndFixFile(options.fix);
 
-  describe('Service declarations lint validation', async function () {
+  describe('Service declarations lint validation', function () {
     this.timeout(30000);
 
     servicesToValidate.forEach(serviceId => {
@@ -46,8 +46,8 @@ export default async options => {
       const filtersFilePath = path.join(declarationsPath, `${serviceId}.filters.js`);
       const filtersHistoryFilePath = path.join(declarationsPath, `${serviceId}.filters.history.js`);
 
-      context(serviceId, async () => {
-        before(async function () {
+      context(serviceId, () => {
+        before(function () {
           if (!service) {
             console.log('      (Tests skipped as declaration has been archived)');
             this.skip();

package/scripts/declarations/validate/index.mocha.js

@@ -40,7 +40,7 @@ export default async options => {
     ({ services: servicesToValidate, servicesTermsTypes } = await declarationUtils.getModifiedServicesAndTermsTypes());
   }
 
-  describe('Service declarations validation', async function () {
+  describe('Service declarations validation', function () {
     this.timeout(60000);
     this.slow(SLOW_DOCUMENT_THRESHOLD);
 
@@ -53,8 +53,8 @@ export default async options => {
 
       after(stopHeadlessBrowser);
 
-      context(serviceId, async () => {
-        before(async function () {
+      context(serviceId, () => {
+        before(function () {
           if (!service) {
             console.log('      (Tests skipped as declaration has been archived)');
             this.skip();
@@ -100,7 +100,7 @@ export default async options => {
                   let filteredContent;
 
                   context(sourceDocument.location, () => {
-                    before(async function () {
+                    before(function () {
                       if (!terms) {
                         console.log('      (Tests skipped as declaration has been archived)');
                         this.skip();
@@ -129,7 +129,7 @@ export default async options => {
                       expect(filteredContent).to.not.be.empty;
                     });
 
-                    it(`filtered content has at least ${MIN_DOC_LENGTH} characters`, async function checkContentLength() {
+                    it(`filtered content has at least ${MIN_DOC_LENGTH} characters`, function checkContentLength() {
                       if (!sourceDocument.content) {
                         console.log('          [Tests skipped as URL is not fetchable]');
                         this.skip();

package/scripts/history/migrate-services.js

@@ -198,7 +198,7 @@ async function initialize(migration) {
   ]);
 }
 
-async function finalize(migration) {
+function finalize(migration) {
   return Promise.all([
     migration.from.snapshots.source.finalize(),
     migration.from.snapshots.destination.finalize(),

package/scripts/import/index.js

@@ -67,7 +67,7 @@ async function initialize() {
   await renamer.loadRules();
 }
 
-async function queueWorker({ commit, index, total }) {
+function queueWorker({ commit, index, total }) {
   return async.retry({
     times: MAX_RETRY,
     interval(retryCount) {

package/src/api/routes/docs.test.js

@@ -26,7 +26,7 @@ describe('Docs API', () => {
       describe('body response defines', () => {
         let subject;
 
-        before(async () => {
+        before(() => {
           subject = response.body;
         });
 
@@ -39,7 +39,7 @@ describe('Docs API', () => {
         });
 
         describe('with endpoints', () => {
-          before(async () => {
+          before(() => {
             subject = response.body.paths;
           });
 

package/src/api/routes/versions.test.js

@@ -57,7 +57,7 @@ describe('Versions API', () => {
       };
     });
 
-    after(async () => versionsRepository.removeAll());
+    after(() => versionsRepository.removeAll());
 
     let response;
 

package/src/archivist/extract/index.test.js

@@ -525,7 +525,7 @@ describe('Extract', () => {
 
       context('when PDF contains no text', () => {
         it('throws an ExtractDocumentError error', async () => {
-          await expect(extract({ content: await fs.readFile(path.resolve(__dirname, '../../../test/fixtures/termsNoText.pdf')), mimeType: mime.getType('pdf') })).to.be.rejectedWith(ExtractDocumentError, /contains no text/);
+          await expect(extract({ content: await fs.readFile(path.resolve(__dirname, '../../../test/fixtures/termsWithoutText.pdf')), mimeType: mime.getType('pdf') })).to.be.rejectedWith(ExtractDocumentError, /contains no text/);
         });
       });
     });

package/src/archivist/fetcher/fullDomFetcher.js

@@ -1,8 +1,8 @@
-import puppeteer from 'puppeteer';
-import puppeteerExtra from 'puppeteer-extra';
+import { TimeoutError } from 'puppeteer';
+import puppeteer from 'puppeteer-extra';
 import stealthPlugin from 'puppeteer-extra-plugin-stealth';
 
-puppeteerExtra.use(stealthPlugin());
+puppeteer.use(stealthPlugin());
 
 let browser;
 
@@ -51,7 +51,7 @@ export default async function fetch(url, cssSelectors, config) {
       content: await page.content(),
     };
   } catch (error) {
-    if (error instanceof puppeteer.errors.TimeoutError) {
+    if (error instanceof TimeoutError) {
       throw new Error(`Timed out after ${config.navigationTimeout / 1000} seconds when trying to fetch '${url}'`);
     }
     throw new Error(error.message);
@@ -67,7 +67,7 @@ export async function launchHeadlessBrowser() {
     return browser;
   }
 
-  browser = await puppeteerExtra.launch({ headless: true });
+  browser = await puppeteer.launch({ headless: true });
 
   return browser;
 }

package/src/archivist/fetcher/index.test.js

@@ -70,7 +70,7 @@ describe('Fetcher', function () {
 
         context('when expected selectors are present', () => {
           before(async () => {
-            ({ content, mimeType } = await fetch({ url, selectors: 'body' }));
+            ({ content, mimeType } = await fetch({ url, cssSelectors: 'body' }));
           });
 
           it('returns the web page content of the given URL', () => {
@@ -83,7 +83,7 @@ describe('Fetcher', function () {
 
           context('with client script enabled', () => {
             before(async () => {
-              ({ content, mimeType } = await fetch({ url, selectors: 'body', executeClientScripts: true }));
+              ({ content, mimeType } = await fetch({ url, cssSelectors: 'body', executeClientScripts: true }));
             });
 
             it('returns the web page content of the given URL', () => {
@@ -100,7 +100,7 @@ describe('Fetcher', function () {
           const NOT_PRESENT_SELECTOR = 'h2';
 
           before(async () => {
-            ({ content, mimeType } = await fetch({ url, selectors: NOT_PRESENT_SELECTOR }));
+            ({ content, mimeType } = await fetch({ url, cssSelectors: NOT_PRESENT_SELECTOR }));
           });
 
           it('returns the web page content of the given URL', () => {
@@ -113,7 +113,7 @@ describe('Fetcher', function () {
 
           context('with client script enabled', () => {
             before(async () => {
-              ({ content, mimeType } = await fetch({ url, selectors: NOT_PRESENT_SELECTOR, executeClientScripts: true }));
+              ({ content, mimeType } = await fetch({ url, cssSelectors: NOT_PRESENT_SELECTOR, executeClientScripts: true }));
             });
 
             it('returns the web page content of the given URL', () => {
@@ -133,7 +133,7 @@ describe('Fetcher', function () {
 
         context('when expected selectors are present', () => {
           before(async () => {
-            ({ content } = await fetch({ url, selectors: 'body' }));
+            ({ content } = await fetch({ url, cssSelectors: 'body' }));
           });
 
           it('returns the web page content of the given URL', () => {

package/src/archivist/index.test.js

@@ -28,7 +28,7 @@ const FETCH_DATE = new Date('2000-01-02T12:00:00.000Z');
 let gitVersion;
 let app;
 
-async function resetGitRepositories() {
+function resetGitRepositories() {
   return Promise.all([ app.recorder.snapshotsRepository.removeAll(), app.recorder.versionsRepository.removeAll() ]);
 }
 
@@ -79,7 +79,7 @@ describe('Archivist', function () {
     });
 
     context('when everything works fine', () => {
-      before(async () => app.track({ services }));
+      before(() => app.track({ services }));
 
       after(resetGitRepositories);
 
@@ -118,11 +118,11 @@ describe('Archivist', function () {
 
       after(resetGitRepositories);
 
-      it('records no snapshot for service A', async () => {
+      it('records no snapshot for service A', () => {
         expect(fsApi.existsSync(path.resolve(__dirname, SERVICE_A_EXPECTED_SNAPSHOT_FILE_PATH))).to.be.false;
       });
 
-      it('records no version for service A', async () => {
+      it('records no version for service A', () => {
         expect(fsApi.existsSync(path.resolve(__dirname, SERVICE_A_EXPECTED_VERSION_FILE_PATH))).to.be.false;
       });
 
@@ -182,11 +182,11 @@ describe('Archivist', function () {
             expect(serviceAContent).to.equal('Terms of service with UTF-8 \'çhãràčtęrs"\n========================================');
           });
 
-          it('generates a new version id', async () => {
+          it('generates a new version id', () => {
             expect(reExtractedVersionId).to.not.equal(firstVersionId);
           });
 
-          it('mentions the snapshot id in the changelog', async () => {
+          it('mentions the snapshot id in the changelog', () => {
             expect(reExtractedVersionMessageBody).to.include(originalSnapshotId);
           });
 
@@ -233,11 +233,11 @@ describe('Archivist', function () {
 
           after(resetGitRepositories);
 
-          it('emits an inaccessibleContent event', async () => {
+          it('emits an inaccessibleContent event', () => {
             expect(inaccessibleContentSpy).to.have.been.called;
           });
 
-          it('still extracts the terms of other services', async () => {
+          it('still extracts the terms of other services', () => {
             expect(versionNotChangedSpy).to.have.been.calledWith(versionB);
           });
         });
@@ -355,10 +355,10 @@ describe('Archivist', function () {
       let terms;
       let snapshot;
 
-      before(async () => {
+      before(() => {
         terms = app.services.service·A.getTerms({ type: SERVICE_A_TYPE });
         terms.fetchDate = FETCH_DATE;
-        terms.sourceDocuments.forEach(async sourceDocument => {
+        terms.sourceDocuments.forEach(sourceDocument => {
           sourceDocument.content = serviceASnapshotExpectedContent;
           sourceDocument.mimeType = MIME_TYPE;
         });
@@ -376,7 +376,7 @@ describe('Archivist', function () {
           return resetGitRepositories();
         });
 
-        it('emits "firstSnapshotRecorded" event', async () => {
+        it('emits "firstSnapshotRecorded" event', () => {
           expect(spies.onFirstSnapshotRecorded).to.have.been.calledWith(snapshot);
         });
 
@@ -424,7 +424,7 @@ describe('Archivist', function () {
             return resetGitRepositories();
           });
 
-          it('emits "snapshotNotChanged" event', async () => {
+          it('emits "snapshotNotChanged" event', () => {
             expect(spies.onSnapshotNotChanged).to.have.been.calledWith(snapshot);
           });
 
@@ -472,7 +472,7 @@ describe('Archivist', function () {
           before(async () => {
             await app.recordVersion(terms);
             resetSpiesHistory();
-            terms.sourceDocuments.forEach(async sourceDocument => {
+            terms.sourceDocuments.forEach(sourceDocument => {
               sourceDocument.content = serviceBSnapshotExpectedContent;
             });
             changedVersion = await app.recordVersion(terms);
@@ -484,7 +484,7 @@ describe('Archivist', function () {
             return resetGitRepositories();
           });
 
-          it('emits "versionRecorded" event', async () => {
+          it('emits "versionRecorded" event', () => {
             expect(spies.onVersionRecorded).to.have.been.calledWith(changedVersion);
           });
 
@@ -506,7 +506,7 @@ describe('Archivist', function () {
             return resetGitRepositories();
           });
 
-          it('emits "versionNotChanged" event', async () => {
+          it('emits "versionNotChanged" event', () => {
             expect(spies.onVersionNotChanged).to.have.been.calledWith(version);
           });
 
@@ -516,7 +516,7 @@ describe('Archivist', function () {
     });
 
     context('when tracking changes on new services', () => {
-      before(async () => {
+      before(() => {
         nock('https://www.servicea.example').get('/tos').reply(200, serviceASnapshotExpectedContent, { 'Content-Type': 'text/html' });
         nock('https://www.serviceb.example').get('/privacy').reply(200, serviceBSnapshotExpectedContent, { 'Content-Type': 'application/pdf' });
 

package/src/archivist/recorder/index.js

@@ -8,19 +8,19 @@ export default class Recorder {
     this.snapshotsRepository = RepositoryFactory.create(config.snapshots.storage);
   }
 
-  async initialize() {
+  initialize() {
     return Promise.all([ this.versionsRepository.initialize(), this.snapshotsRepository.initialize() ]);
   }
 
-  async finalize() {
+  finalize() {
     return Promise.all([ this.versionsRepository.finalize(), this.snapshotsRepository.finalize() ]);
   }
 
-  async getLatestSnapshot(terms, sourceDocumentId) {
+  getLatestSnapshot(terms, sourceDocumentId) {
     return this.snapshotsRepository.findLatest(terms.service.id, terms.type, terms.hasMultipleSourceDocuments && sourceDocumentId);
   }
 
-  async record(record) {
+  record(record) {
     record.validate();
 
     switch (record.constructor) { // eslint-disable-line default-case

package/src/archivist/recorder/index.test.js

@@ -30,7 +30,7 @@ describe('Recorder', () => {
         await recorder.initialize();
       });
 
-      after(async () => recorder.finalize());
+      after(() => recorder.finalize());
 
       context('Snapshot', () => {
         describe('#record', () => {
@@ -41,7 +41,7 @@ describe('Recorder', () => {
           let record;
 
           context('when a required param is missing', () => {
-            after(async () => recorder.snapshotsRepository.removeAll());
+            after(() => recorder.snapshotsRepository.removeAll());
 
             Snapshot.REQUIRED_PARAMS.forEach(testedRequiredParam => {
               context(`when "${testedRequiredParam}" is missing`, () => {
@@ -82,17 +82,17 @@ describe('Recorder', () => {
               record = await recorder.snapshotsRepository.findLatest(SERVICE_ID, TYPE);
             });
 
-            after(async () => recorder.snapshotsRepository.removeAll());
+            after(() => recorder.snapshotsRepository.removeAll());
 
             it('records the snapshot with the proper content', async () => {
               expect(await record.content).to.equal(CONTENT);
             });
 
-            it('returns the record id', async () => {
+            it('returns the record id', () => {
               expect(record.id).to.include(id);
             });
 
-            it('states that it is the first record', async () => {
+            it('states that it is the first record', () => {
               expect(isFirstRecord).to.be.true;
             });
           });
@@ -120,17 +120,17 @@ describe('Recorder', () => {
               record = await recorder.snapshotsRepository.findLatest(SERVICE_ID, TYPE);
             });
 
-            after(async () => recorder.snapshotsRepository.removeAll());
+            after(() => recorder.snapshotsRepository.removeAll());
 
             it('records the snapshot with the proper content', async () => {
               expect(await record.content).to.equal(UPDATED_CONTENT);
             });
 
-            it('returns the record id', async () => {
+            it('returns the record id', () => {
               expect(record.id).to.include(id);
             });
 
-            it('states that it is not the first record', async () => {
+            it('states that it is not the first record', () => {
               expect(isFirstRecord).to.be.false;
             });
           });
@@ -156,9 +156,9 @@ describe('Recorder', () => {
               record = await recorder.snapshotsRepository.findLatest(SERVICE_ID, TYPE);
             });
 
-            after(async () => recorder.snapshotsRepository.removeAll());
+            after(() => recorder.snapshotsRepository.removeAll());
 
-            it('does not record the snapshot', async () => {
+            it('does not record the snapshot', () => {
               expect(id).to.not.be.ok;
             });
           });
@@ -175,7 +175,7 @@ describe('Recorder', () => {
           let record;
 
           context('when a required param is missing', () => {
-            after(async () => recorder.versionsRepository.removeAll());
+            after(() => recorder.versionsRepository.removeAll());
 
             Version.REQUIRED_PARAMS.forEach(testedRequiredParam => {
               context(`when "${testedRequiredParam}" is missing`, () => {
@@ -216,17 +216,17 @@ describe('Recorder', () => {
               record = await recorder.versionsRepository.findLatest(SERVICE_ID, TYPE);
             });
 
-            after(async () => recorder.versionsRepository.removeAll());
+            after(() => recorder.versionsRepository.removeAll());
 
             it('records the version with the proper content', async () => {
               expect(await record.content).to.equal(CONTENT);
             });
 
-            it('returns the record id', async () => {
+            it('returns the record id', () => {
               expect(record.id).to.include(id);
             });
 
-            it('states that it is the first record', async () => {
+            it('states that it is the first record', () => {
               expect(isFirstRecord).to.be.true;
             });
           });
@@ -254,21 +254,21 @@ describe('Recorder', () => {
               record = await recorder.versionsRepository.findLatest(SERVICE_ID, TYPE);
             });
 
-            after(async () => recorder.versionsRepository.removeAll());
+            after(() => recorder.versionsRepository.removeAll());
 
             it('records the version with the proper content', async () => {
               expect(await record.content).to.equal(UPDATED_CONTENT);
             });
 
-            it('records in the version that it is not an extracted only version', async () => {
+            it('records in the version that it is not an extracted only version', () => {
               expect(record.isExtractOnly).to.equal(false);
             });
 
-            it('returns the record id', async () => {
+            it('returns the record id', () => {
               expect(record.id).to.include(id);
             });
 
-            it('states that it is not the first record', async () => {
+            it('states that it is not the first record', () => {
               expect(isFirstRecord).to.be.false;
             });
           });
@@ -294,9 +294,9 @@ describe('Recorder', () => {
               record = await recorder.versionsRepository.findLatest(SERVICE_ID, TYPE);
             });
 
-            after(async () => recorder.versionsRepository.removeAll());
+            after(() => recorder.versionsRepository.removeAll());
 
-            it('does not record any version', async () => {
+            it('does not record any version', () => {
               expect(id).to.not.be.ok;
             });
           });
@@ -323,17 +323,17 @@ describe('Recorder', () => {
                 record = await recorder.versionsRepository.findLatest(SERVICE_ID, TYPE);
               });
 
-              after(async () => recorder.versionsRepository.removeAll()); after(async () => recorder.versionsRepository.removeAll());
+              after(() => recorder.versionsRepository.removeAll()); after(() => recorder.versionsRepository.removeAll());
 
               it('records the version with the proper content', async () => {
                 expect(await record.content).to.equal(CONTENT);
               });
 
-              it('returns the record id', async () => {
+              it('returns the record id', () => {
                 expect(record.id).to.include(id);
               });
 
-              it('states that it is the first record', async () => {
+              it('states that it is the first record', () => {
                 expect(isFirstRecord).to.be.true;
               });
             });
@@ -362,21 +362,21 @@ describe('Recorder', () => {
                 record = await recorder.versionsRepository.findLatest(SERVICE_ID, TYPE);
               });
 
-              after(async () => recorder.versionsRepository.removeAll());
+              after(() => recorder.versionsRepository.removeAll());
 
               it('records the version with the proper content', async () => {
                 expect(await record.content).to.equal(UPDATED_CONTENT);
               });
 
-              it('records in the version that it is an extracted only version', async () => {
+              it('records in the version that it is an extracted only version', () => {
                 expect(record.isExtractOnly).to.equal(true);
               });
 
-              it('returns the record id', async () => {
+              it('returns the record id', () => {
                 expect(record.id).to.include(id);
               });
 
-              it('states that it is not the first record', async () => {
+              it('states that it is not the first record', () => {
                 expect(isFirstRecord).to.be.false;
               });
             });
@@ -403,9 +403,9 @@ describe('Recorder', () => {
                 record = await recorder.versionsRepository.findLatest(SERVICE_ID, TYPE);
               });
 
-              after(async () => recorder.versionsRepository.removeAll());
+              after(() => recorder.versionsRepository.removeAll());
 
-              it('does not record any version', async () => {
+              it('does not record any version', () => {
                 expect(id).to.not.be.ok;
               });
             });

package/src/archivist/recorder/repositories/git/git.js

@@ -30,7 +30,7 @@ export default class Git {
       .addConfig('core.quotePath', false); // disable Git's encoding of special characters in pathnames. For example, `service·A` will be encoded as `service\302\267A` without this setting, leading to issues. See https://git-scm.com/docs/git-config#Documentation/git-config.txt-corequotePath
   }
 
-  async add(filePath) {
+  add(filePath) {
     return this.git.add(this.relativePath(filePath));
   }
 
@@ -55,11 +55,11 @@ export default class Git {
     return summary.commit;
   }
 
-  async pushChanges() {
+  pushChanges() {
     return this.git.push();
   }
 
-  async listCommits(options = []) {
+  listCommits(options = []) {
     return this.log([ '--reverse', '--no-merges', '--name-only', ...options ]);
   }
 
@@ -89,11 +89,11 @@ export default class Git {
     return Boolean(result);
   }
 
-  async checkout(options) {
+  checkout(options) {
     return this.git.checkout(options);
   }
 
-  async show(options) {
+  show(options) {
     return this.git.show(options);
   }
 
@@ -107,7 +107,7 @@ export default class Git {
     return (await this.git.show([ shortHash, '--pretty=%H', '-s' ])).trim();
   }
 
-  async restore(path, commit) {
+  restore(path, commit) {
     return this.git.raw([ 'restore', '-s', commit, '--', path ]);
   }
 

package/src/archivist/recorder/repositories/git/index.js

@@ -100,7 +100,7 @@ export default class GitRepository extends RepositoryInterface {
     }
   }
 
-  async removeAll() {
+  removeAll() {
     return this.git.destroyHistory();
   }
 
@@ -129,7 +129,9 @@ export default class GitRepository extends RepositoryInterface {
 
   async #getCommits() {
     return (await this.git.listCommits())
-      .filter(({ message }) => message.match(DataMapper.COMMIT_MESSAGE_PREFIXES_REGEXP)) // Skip commits which are not a record (README, LICENSE…)
+      .filter(commit => // Skip non-record commits (e.g., README or LICENSE updates)
+        DataMapper.COMMIT_MESSAGE_PREFIXES_REGEXP.test(commit.message) // Commits generated by the engine have messages that match predefined prefixes
+        && path.dirname(commit.diff.files[0].file) !== '.') // Assumes one record per commit; records must be in a serviceId folder, not root
       .sort((commitA, commitB) => new Date(commitA.date) - new Date(commitB.date)); // Make sure that the commits are sorted in ascending chronological order
   }