@opentermsarchive/engine 0.25.1 → 0.26.0

package/README.md

@@ -226,12 +226,12 @@ Run [`ota validate`](#ota-validate) only on files that have been modified in Git
 #### `ota lint`
 
 ```sh
-npx ota lint [--services <service_id>...]
+npx ota lint [--services <service_id>...] [--fix] [--modified]
 ```
 
-Normalise the format of declarations.
+Test the the format of declarations' normalisation.
 
-Automatically correct formatting mistakes and ensure that all declarations are standardised.
+Use `--fix` to automatically correct formatting mistakes and ensure that all declarations are standardised.
 
 If one or several `<service_id>` are provided, check only those services.
 

package/bin/ota-lint.js

@@ -2,18 +2,55 @@
 import './env.js';
 
 import path from 'path';
-import { fileURLToPath, pathToFileURL } from 'url';
+import { fileURLToPath } from 'url';
 
 import { program } from 'commander';
+import Mocha from 'mocha';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
+const LINT_TEST_FILEPATH = '../scripts/declarations/lint/index.mocha.js';
+const LINT_PATH = path.resolve(__dirname, LINT_TEST_FILEPATH);
+
+// Mocha catches unhandled rejection from the user code and re-emits them to the process (see https://github.com/mochajs/mocha/blob/master/lib/runner.js#L198)
+process.on('unhandledRejection', reason => {
+  // Re-throw them so that the validation command fails in these cases (for example, if there is a syntax error when parsing JSON declaration files)
+  throw reason;
+});
+
 program
   .name('ota lint')
   .description('Check format and stylistic errors in declarations and auto fix them')
   .option('-s, --services [serviceId...]', 'service IDs of services to lint')
-  .option('-m, --modified', 'to only lint modified services already commited to git');
+  .option('-m, --modified', 'to only lint modified services already commited to git')
+  .option('-f, --fix', 'to fix the declarations');
+
+const mocha = new Mocha({
+  delay: true, // as the validation script performs an asynchronous load before running the tests, the execution of the tests are delayed until run() is called
+  failZero: true, // consider that being called with no service to validate is a failure
+});
+
+(async () => {
+  mocha.addFile(LINT_PATH); // As `delay` has been called, this statement will not load the file directly, `loadFilesAsync` is required.
+  await mocha.loadFilesAsync() // Load files previously added to the Mocha cache with `addFile`.
+    .catch(error => {
+      console.error(error);
+      process.exit(2);
+    });
+
+  let hasFailedTests = false;
+
+  const lintFiles = (await import(LINT_TEST_FILEPATH)).default;
+
+  lintFiles(program.parse().opts());
 
-const lintDeclarations = (await import(pathToFileURL(path.resolve(__dirname, '../scripts/declarations/lint/index.js')))).default; // load asynchronously to ensure env.js is loaded before
+  mocha.run()
+    .on('fail', () => { hasFailedTests = true; })
+    .on('end', () => {
+      if (hasFailedTests) {
+        process.exit(1);
+      }
 
-lintDeclarations(program.parse().opts());
+      process.exit(0);
+    });
+})();

package/bin/ota-validate.js

@@ -9,7 +9,8 @@ import Mocha from 'mocha';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
-const VALIDATE_PATH = path.resolve(__dirname, '../scripts/declarations/validate/index.mocha.js');
+const VALIDATE_TEST_FILEPATH = '../scripts/declarations/validate/index.mocha.js';
+const VALIDATE_PATH = path.resolve(__dirname, VALIDATE_TEST_FILEPATH);
 
 // Mocha catches unhandled rejection from the user code and re-emits them to the process (see https://github.com/mochajs/mocha/blob/master/lib/runner.js#L198)
 process.on('unhandledRejection', reason => {
@@ -40,7 +41,7 @@ const mocha = new Mocha({
 
   let hasFailedTests = false;
 
-  const generateValidationTestSuite = (await import('../scripts/declarations/validate/index.mocha.js')).default;
+  const generateValidationTestSuite = (await import(VALIDATE_TEST_FILEPATH)).default;
 
   generateValidationTestSuite(program.parse().opts());
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "0.25.1",
+  "version": "0.26.0",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://github.com/ambanum/OpenTermsArchive#readme",
   "bugs": {

package/scripts/declarations/lint/index.mocha.js

@@ -0,0 +1,105 @@
+import fsApi from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+import { expect } from 'chai';
+import config from 'config';
+import { ESLint } from 'eslint';
+
+import * as services from '../../../src/archivist/services/index.js';
+import DeclarationUtils from '../utils/index.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const ROOT_PATH = path.resolve(__dirname, '../../../');
+const ESLINT_CONFIG_PATH = path.join(ROOT_PATH, '.eslintrc.yaml');
+
+const eslint = new ESLint({ overrideConfigFile: ESLINT_CONFIG_PATH, fix: false });
+const eslintWithFix = new ESLint({ overrideConfigFile: ESLINT_CONFIG_PATH, fix: true });
+
+const declarationsPath = path.resolve(process.cwd(), config.get('services.declarationsPath'));
+const instancePath = path.resolve(declarationsPath, '../');
+
+export default async options => {
+  let servicesToValidate = options.services || [];
+
+  const serviceDeclarations = await services.loadWithHistory(servicesToValidate);
+
+  if (!servicesToValidate.length) {
+    servicesToValidate = Object.keys(serviceDeclarations);
+  }
+
+  if (options.modified) {
+    const declarationUtils = new DeclarationUtils(instancePath);
+
+    ({ services: servicesToValidate } = await declarationUtils.getModifiedServiceDocumentTypes());
+  }
+
+  const lintFile = lintAndFixFile(options.fix);
+
+  describe('Service declarations lint validation', async function () {
+    this.timeout(30000);
+
+    servicesToValidate.forEach(serviceId => {
+      const service = serviceDeclarations[serviceId];
+      const filePath = path.join(declarationsPath, `${serviceId}.json`);
+      const historyFilePath = path.join(declarationsPath, `${serviceId}.history.json`);
+      const filtersFilePath = path.join(declarationsPath, `${serviceId}.filters.js`);
+      const filtersHistoryFilePath = path.join(declarationsPath, `${serviceId}.filters.history.js`);
+
+      context(serviceId, async () => {
+        before(async function () {
+          if (!service) {
+            console.log('      (Tests skipped as declaration has been archived)');
+            this.skip();
+          }
+        });
+
+        it('valid declaration file lint', async () => {
+          await lintFile(filePath);
+        });
+
+        if (service && service.hasHistory()) {
+          it('valid history declaration file lint', async () => {
+            await lintFile(historyFilePath);
+          });
+        }
+
+        if (fsApi.existsSync(filtersFilePath)) {
+          it('valid filters file lint', async () => {
+            await lintFile(filtersFilePath);
+          });
+        }
+
+        if (fsApi.existsSync(filtersHistoryFilePath)) {
+          it('valid filters history file lint', async () => {
+            await lintFile(filtersHistoryFilePath);
+          });
+        }
+      });
+    });
+  });
+
+  run();
+};
+
+const lintAndFixFile = fix => async filePath => {
+  // Create a new instance of linter with option `fix` set to true to get a fixed output.
+  // It is not possible to use only a linter with this option enabled because when
+  // this option is set, if it can fix errors, it considers that there are no errors and returns `0` for the `errorCount`.
+  // So use two linters to have access both to `errorCount` and fix `output` variables.
+  const [lintResult] = await eslint.lintFiles(filePath);
+
+  if (!lintResult.errorCount) {
+    return;
+  }
+
+  const [lintResultFixed] = await eslintWithFix.lintFiles(filePath);
+
+  if (fix) {
+    await ESLint.outputFixes([lintResultFixed]);
+
+    return lintAndFixFile(false)(filePath);
+  }
+
+  expect(lintResult.source).to.equal(lintResultFixed.output, `${path.basename(filePath)} is not properly formatted. Use the lint script to format it correctly.\n`);
+};

package/scripts/declarations/validate/index.mocha.js

@@ -1,11 +1,9 @@
 import fsApi from 'fs';
 import path from 'path';
-import { fileURLToPath } from 'url';
 
 import Ajv from 'ajv';
 import { expect } from 'chai';
 import config from 'config';
-import { ESLint } from 'eslint';
 import jsonSourceMap from 'json-source-map';
 
 import fetch, { launchHeadlessBrowser, stopHeadlessBrowser } from '../../../src/archivist/fetcher/index.js';
@@ -18,16 +16,9 @@ import serviceSchema from './service.schema.js';
 
 const fs = fsApi.promises;
 
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-const ROOT_PATH = path.resolve(__dirname, '../../../');
-const ESLINT_CONFIG_PATH = path.join(ROOT_PATH, '.eslintrc.yaml');
-
 const MIN_DOC_LENGTH = 100;
 const SLOW_DOCUMENT_THRESHOLD = 10 * 1000; // number of milliseconds after which a document fetch is considered slow
 
-const eslint = new ESLint({ overrideConfigFile: ESLINT_CONFIG_PATH, fix: false });
-const eslintWithFix = new ESLint({ overrideConfigFile: ESLINT_CONFIG_PATH, fix: true });
-
 const declarationsPath = path.resolve(process.cwd(), config.get('services.declarationsPath'));
 const instancePath = path.resolve(declarationsPath, '../');
 
@@ -76,36 +67,12 @@ export default async options => {
           assertValid(serviceSchema, declaration);
         });
 
-        it('valid declaration file format', async () => {
-          await lintFile(filePath);
-        });
-
         if (service && service.hasHistory()) {
           it('valid history declaration schema', async () => {
             const declarationHistory = JSON.parse(await fs.readFile(historyFilePath));
 
             assertValid(serviceHistorySchema, declarationHistory);
           });
-
-          it('valid history declaration file format', async () => {
-            await lintFile(path.join(declarationsPath, `${serviceId}.history.json`));
-          });
-        }
-
-        const filtersFilePath = path.join(declarationsPath, `${serviceId}.filters.js`);
-
-        if (fsApi.existsSync(filtersFilePath)) {
-          it('valid filters file format', async () => {
-            await lintFile(filtersFilePath);
-          });
-        }
-
-        const filtersHistoryFilePath = path.join(declarationsPath, `${serviceId}.filters.history.js`);
-
-        if (fsApi.existsSync(filtersHistoryFilePath)) {
-          it('valid filters history file format', async () => {
-            await lintFile(filtersHistoryFilePath);
-          });
         }
 
         if (!schemaOnly && service) {
@@ -245,18 +212,3 @@ function assertValid(schema, subject) {
     throw new Error(errorMessage);
   }
 }
-
-async function lintFile(filePath) {
-  const [lintResult] = await eslint.lintFiles(filePath);
-
-  if (!lintResult.errorCount) {
-    return;
-  }
-
-  // Create a new instance of linter with option `fix` set to true to get a fixed output.
-  // It is not possible to use only a linter with this option enabled because when this option is set, if it can fix errors, it considers that there are no errors and returns `0` for the `errorCount`.
-  // So use two linters to have access both to `errorCount` and fix `output` variables.
-  const [lintResultFixed] = await eslintWithFix.lintFiles(filePath);
-
-  expect(lintResult.source).to.equal(lintResultFixed.output, `${path.basename(filePath)} is not properly formatted. Use the lint script to format it correctly.\n`);
-}

package/scripts/declarations/lint/index.js

@@ -1,36 +0,0 @@
-import path from 'path';
-import { fileURLToPath } from 'url';
-
-import config from 'config';
-import { ESLint } from 'eslint';
-
-import DeclarationUtils from '../utils/index.js';
-
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-
-const declarationsPath = config.get('services.declarationsPath');
-const instancePath = path.resolve(declarationsPath, '../');
-const ESLINT_CONFIG_PATH = path.join(__dirname, '../../../.eslintrc.yaml');
-
-const lintDeclarations = async ({ services, modified }) => {
-  console.log(`Linting declaration files in ${instancePath}`);
-  let servicesToValidate = services || ['*'];
-
-  if (modified) {
-    const declarationUtils = new DeclarationUtils(instancePath);
-
-    servicesToValidate = await declarationUtils.getModifiedServices();
-  }
-
-  for (const service of servicesToValidate) {
-    /* eslint-disable no-await-in-loop */
-    const lintResults = await new ESLint({ overrideConfigFile: ESLINT_CONFIG_PATH, fix: true })
-      .lintFiles(path.join(declarationsPath, `${service}.*`));
-
-    await ESLint.outputFixes(lintResults);
-    console.log(lintResults.map(lintResult => `${path.basename(lintResult.filePath)} linted`).join('\n'));
-    /* eslint-enable no-await-in-loop */
-  }
-};
-
-export default lintDeclarations;