@opentermsarchive/engine 4.0.2 → 4.2.0

package/bin/ota-lint.js

@@ -12,7 +12,7 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const LINT_TEST_FILEPATH = '../scripts/declarations/lint/index.mocha.js';
 const LINT_PATH = path.resolve(__dirname, LINT_TEST_FILEPATH);
 
-// Mocha catches unhandled rejection from the user code and re-emits them to the process (see https://github.com/mochajs/mocha/blob/master/lib/runner.js#L198)
+// Mocha catches unhandled rejection from the user code and re-emits them to the process
 process.on('unhandledRejection', reason => {
   // Re-throw them so that the validation command fails in these cases (for example, if there is a syntax error when parsing JSON declaration files)
   throw reason;

package/bin/ota-validate.js

@@ -1,6 +1,5 @@
 #! /usr/bin/env node
 import './env.js';
-
 import path from 'path';
 import { fileURLToPath } from 'url';
 
@@ -9,49 +8,71 @@ import Mocha from 'mocha';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
-const VALIDATE_TEST_FILEPATH = '../scripts/declarations/validate/index.mocha.js';
-const VALIDATE_PATH = path.resolve(__dirname, VALIDATE_TEST_FILEPATH);
+export function createMocha({ delay = false, reporter = 'spec' } = {}) {
+  return new Mocha({
+    delay,
+    failZero: true,
+    reporter,
+  });
+}
+
+export async function runMochaTests(mocha, testPath) {
+  try {
+    mocha.addFile(testPath); // With `delay` option, this statement will not load the file directly, `loadFilesAsync` is required.
+    await mocha.loadFilesAsync(); // Load files previously added to the Mocha cache with `addFile`.
+
+    return new Promise(resolve => {
+      let hasFailedTests = false;
+
+      mocha.run()
+        .on('fail', () => { hasFailedTests = true; })
+        .on('end', () => { resolve(hasFailedTests ? 1 : 0); });
+    });
+  } catch (error) {
+    console.error('Error running tests:', error);
+
+    return 2;
+  }
+}
 
-// Mocha catches unhandled rejection from the user code and re-emits them to the process (see https://github.com/mochajs/mocha/blob/master/lib/runner.js#L198)
-process.on('unhandledRejection', reason => {
-  // Re-throw them so that the validation command fails in these cases (for example, if there is a syntax error when parsing JSON declaration files)
-  throw reason;
+process.on('unhandledRejection', reason => { // Mocha catches unhandled rejection from the user code and re-emits them to the process
+  throw reason; // Re-throw them so that the validation command fails in these cases (for example, if there is a syntax error when parsing JSON declaration files)
 });
 
 program
   .name('ota validate')
+  .description('Validate terms declarations and metadata files');
+
+program.command('declarations')
   .description('Run a series of tests to check the validity of terms declarations')
   .option('-s, --services [serviceId...]', 'service IDs of services to validate')
   .option('-t, --types [termsType...]', 'terms types to validate')
   .option('-m, --modified', 'target only services modified in the current git branch')
-  .option('-o, --schema-only', 'much faster check of declarations, but does not check that the documents are actually accessible');
+  .option('-o, --schema-only', 'much faster check of declarations, but does not check that the documents are actually accessible')
+  .action(async options => {
+    const VALIDATE_TEST_FILEPATH = '../scripts/declarations/validate/index.mocha.js';
+    const VALIDATE_PATH = path.resolve(__dirname, VALIDATE_TEST_FILEPATH);
 
-const mocha = new Mocha({
-  delay: true, // as the validation script performs an asynchronous load before running the tests, the execution of the tests are delayed until run() is called
-  failZero: true, // consider that being called with no service to validate is a failure
-});
+    const mocha = createMocha({ delay: true }); // as the validation script performs an asynchronous load before running the tests, the execution of the tests are delayed until run() is called
+    const generateValidationTestSuite = (await import(VALIDATE_TEST_FILEPATH)).default;
 
-(async () => {
-  mocha.addFile(VALIDATE_PATH); // As `delay` has been called, this statement will not load the file directly, `loadFilesAsync` is required.
-  await mocha.loadFilesAsync() // Load files previously added to the Mocha cache with `addFile`.
-    .catch(error => {
-      console.error(error);
-      process.exit(2);
-    });
+    generateValidationTestSuite(options);
 
-  let hasFailedTests = false;
+    const exitCode = await runMochaTests(mocha, VALIDATE_PATH);
 
-  const generateValidationTestSuite = (await import(VALIDATE_TEST_FILEPATH)).default;
+    process.exit(exitCode);
+  });
 
-  generateValidationTestSuite(program.parse().opts());
+program.command('metadata')
+  .description('Validate the metadata file structure')
+  .action(async () => {
+    const VALIDATE_TEST_FILEPATH = '../scripts/metadata/index.mocha.js';
+    const VALIDATE_PATH = path.resolve(__dirname, VALIDATE_TEST_FILEPATH);
 
-  mocha.run()
-    .on('fail', () => { hasFailedTests = true; })
-    .on('end', () => {
-      if (hasFailedTests) {
-        process.exit(1);
-      }
+    const mocha = createMocha();
+    const exitCode = await runMochaTests(mocha, VALIDATE_PATH);
 
-      process.exit(0);
-    });
-})();
+    process.exit(exitCode);
+  });
+
+program.parse();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "4.0.2",
+  "version": "4.2.0",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {
@@ -36,10 +36,11 @@
     "dataset:release": "node bin/ota.js dataset --publish --remove-local-copy",
     "dataset:scheduler": "npm run dataset:release -- --schedule",
     "declarations:lint": "node bin/ota.js lint",
-    "declarations:validate": "node bin/ota.js validate",
+    "declarations:validate": "node bin/ota.js validate declarations",
     "declarations:validate:schema": "npm run declarations:validate -- --schema-only",
     "lint": "eslint src test scripts bin",
     "lint:fix": "npm run lint -- --fix",
+    "metadata:validate": "node bin/ota.js validate metadata",
     "start": "node -r dotenv/config --max-http-header-size=32768 bin/ota.js track",
     "start:api": "node bin/ota.js serve",
     "start:scheduler": "npm start -- --schedule",
@@ -54,7 +55,8 @@
     "@opentermsarchive/turndown": "^7.1.3",
     "@stylistic/eslint-plugin-js": "^1.4.1",
     "abort-controller": "^3.0.0",
-    "ajv": "^6.12.6",
+    "ajv": "^8.17.1",
+    "ajv-formats": "^3.0.1",
     "archiver": "^5.3.0",
     "async": "^3.2.2",
     "chai": "^4.3.4",

package/scripts/declarations/validate/index.mocha.js

@@ -2,6 +2,7 @@ import fsApi from 'fs';
 import path from 'path';
 
 import Ajv from 'ajv';
+import addFormats from 'ajv-formats';
 import { expect } from 'chai';
 import config from 'config';
 import jsonSourceMap from 'json-source-map';
@@ -178,10 +179,9 @@ export default async options => {
   run();
 };
 
-const validator = new Ajv({
-  allErrors: true,
-  jsonPointers: true,
-});
+const validator = new Ajv({ allErrors: true });
+
+addFormats(validator);
 
 function assertValid(schema, subject) {
   const valid = validator.validate(schema, subject);
@@ -193,7 +193,6 @@ function assertValid(schema, subject) {
     const jsonLines = sourceMap.json.split('\n');
 
     validator.errors.forEach(error => {
-      console.log('error', error);
       errorMessage += `\n\n${validator.errorsText([error])}`;
       const errorPointer = sourceMap.pointers[error.dataPath];
 

package/scripts/declarations/validate/service.schema.js

@@ -67,7 +67,10 @@ const schema = {
     singleSourceDocumentTerms: {
       allOf: [
         { $ref: '#/definitions/sourceDocument' },
-        { required: [ 'fetch', 'select' ] },
+        {
+          type: 'object',
+          required: [ 'fetch', 'select' ],
+        },
       ],
     },
     multipleSourceDocumentsTerms: {

package/scripts/metadata/index.mocha.js

@@ -0,0 +1,85 @@
+import fs from 'fs/promises';
+import path from 'path';
+
+import Ajv from 'ajv';
+import addFormats from 'ajv-formats';
+import config from 'config';
+import Croner from 'croner';
+import yaml from 'js-yaml';
+
+import specsRouter from '../../src/collection-api/routes/docs.js';
+
+describe('Metadata file validation', () => {
+  const formatValidators = {
+    'iso639-1': code => /^[a-z]{2}$/.test(code),
+    'iso3166-2': code => /^[A-Z]{2}(-[A-Z0-9]{1,3})?$/.test(code),
+    'cron-expression': cronExpression => {
+      try {
+        Croner(cronExpression); // eslint-disable-line new-cap
+
+        return true;
+      } catch {
+        return false;
+      }
+    },
+  };
+
+  const formatMessages = {
+    'iso639-1': 'must be a valid ISO 639-1 language code (two lowercase letters, e.g., "en", "fr")',
+    'iso3166-2': 'must be a valid ISO 3166-2 region code (two uppercase letters, e.g., "FR", "US")',
+    'cron-expression': 'must be a valid cron expression (see https://en.wikipedia.org/wiki/Cron#Cron_expression)',
+  };
+
+  let metadata;
+  let validate;
+
+  before(async () => {
+    const { specs } = specsRouter(''); // Extract Metadata OpenAPI specification from JSDoc comments in the collection API router to validate the metadata schema. Can be achieved until API specification and Metadata file schema diverge
+    const metadataSchema = specs.components.schemas.Metadata;
+    const collectionPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.collectionPath'));
+    const metadataContent = await fs.readFile(path.join(collectionPath, 'metadata.yml'), 'utf8');
+
+    metadata = yaml.load(metadataContent, { schema: yaml.CORE_SCHEMA }); // Use CORE_SCHEMA to parse dates as strings rather than JavaScript Date objects
+
+    const ajv = new Ajv({ allErrors: true });
+
+    addFormats(ajv);
+
+    Object.entries(formatValidators).forEach(([ format, validator ]) => {
+      ajv.addFormat(format, { type: 'string', validate: validator });
+    });
+
+    validate = ajv.compile(metadataSchema);
+    validate(metadata);
+  });
+
+  it('is valid', () => {
+    if (!validate.errors) {
+      return;
+    }
+
+    const errors = validate.errors.map(error => {
+      const instancePath = error.instancePath.split('/').slice(1);
+      const actualValue = instancePath.reduce((obj, key) => obj?.[key], metadata);
+      const basePath = error.instancePath || '/root';
+
+      if (error.keyword === 'additionalProperties') {
+        return `- ${basePath}: Found unexpected property "${error.params.additionalProperty}"`;
+      }
+
+      if (error.keyword === 'format' && formatMessages[error.params.format]) {
+        return `- ${basePath}: "${actualValue}" ${formatMessages[error.params.format]}`;
+      }
+
+      let message = `- ${basePath}: "${actualValue}" ${error.message}`;
+
+      if (error.keyword === 'enum') {
+        message += ` "${error.params.allowedValues.join('", "')}"`;
+      }
+
+      return message;
+    });
+
+    throw new Error(`\n${errors.join('\n')}`);
+  });
+});

package/src/collection-api/routes/docs.js

@@ -34,5 +34,7 @@ export default function specsRouter(basePath) {
     return swaggerUi.setup(specs)(req, res);
   });
 
+  router.specs = specs;
+
   return router;
 }

package/src/collection-api/routes/metadata.js

@@ -19,6 +19,7 @@ const PACKAGE_JSON_PATH = '../../../package.json';
  *     Metadata:
  *       type: object
  *       description: Collection metadata
+ *       additionalProperties: false
  *       properties:
  *         id:
  *           type: string
@@ -69,18 +70,21 @@ const PACKAGE_JSON_PATH = '../../../package.json';
  *           description: URL to the collection logo
  *         languages:
  *           type: array
+ *           description: List of ISO 639-1 (two-letter) language codes representing languages allowed by the collection
  *           items:
  *             type: string
- *           description: List of ISO 639 language codes representing languages allowed by the collection
+ *             format: iso639-1
  *         jurisdictions:
  *           type: array
+ *           description: List of ISO 3166-2 country codes representing jurisdictions covered by the collection
  *           items:
  *             type: string
- *           description: List of ISO 3166-2 country codes representing jurisdictions covered by the collection
+ *             format: iso3166-2
  *         trackingPeriods:
  *           type: array
  *           items:
  *             type: object
+ *             additionalProperties: false
  *             properties:
  *               startDate:
  *                 type: string
@@ -88,6 +92,7 @@ const PACKAGE_JSON_PATH = '../../../package.json';
  *                 description: The date when tracking started for this period
  *               schedule:
  *                 type: string
+ *                 format: cron-expression
  *                 description: A cron expression defining when terms are tracked (e.g. "0 0 * * *" for daily at midnight)
  *               serverLocation:
  *                 type: string
@@ -100,6 +105,7 @@ const PACKAGE_JSON_PATH = '../../../package.json';
  *           type: object
  *           additionalProperties:
  *             type: object
+ *             additionalProperties: false
  *             properties:
  *               url:
  *                 type: string
@@ -115,6 +121,11 @@ const PACKAGE_JSON_PATH = '../../../package.json';
  *                   type: string
  *                   enum: [host, administrator, curator, maintainer, sponsor]
  *                   description: Roles of the entity within the governance
+ *         i18n:
+ *           type: object
+ *           description: Internationalization of any of the Metadata properties (except i18n itself) for different language codes
+ *           additionalProperties:
+ *             type: object
  */
 export default async function metadataRouter(collectionPath, services) {
   const router = express.Router();

package/src/reporter/gitlab/index.js

@@ -18,9 +18,7 @@ export default class GitLab {
   static ISSUE_STATE_ALL = 'all';
 
   constructor(repository, baseURL = BASE_URL, apiBaseURL = API_BASE_URL) {
-    const [ owner, repo ] = repository.split('/');
-
-    this.commonParams = { owner, repo };
+    this.repositoryPath = repository;
     this.projectId = null;
     this.baseURL = baseURL;
     console.log('this.baseURL', this.baseURL);
@@ -31,9 +29,8 @@ export default class GitLab {
     const options = GitLab.baseOptionsHttpReq();
 
     try {
-      const repositoryPath = `${this.commonParams.owner}/${this.commonParams.repo}`;
       const response = await nodeFetch(
-        `${this.apiBaseURL}/projects/${encodeURIComponent(repositoryPath)}`,
+        `${this.apiBaseURL}/projects/${encodeURIComponent(this.repositoryPath)}`,
         options,
       );
 
@@ -42,7 +39,7 @@ export default class GitLab {
       if (response.ok) {
         this.projectId = res.id;
       } else {
-        logger.error(`Error while obtaining projectId: ${JSON.strinfigy(res)}`);
+        logger.error(`Error while obtaining projectId: ${JSON.stringify(res)}`);
         this.projectId = null;
       }
     } catch (error) {
@@ -367,15 +364,15 @@ export default class GitLab {
   }
 
   generateDeclarationURL(serviceName) {
-    return `${this.baseURL}/${this.commonParams.owner}/${this.commonParams.repo}/-/blob/main/declarations/${encodeURIComponent(serviceName)}.json`;
+    return `${this.baseURL}/${this.repositoryPath}/-/blob/main/declarations/${encodeURIComponent(serviceName)}.json`;
   }
 
   generateVersionURL(serviceName, termsType) {
-    return `${this.baseURL}/${this.commonParams.owner}/${this.commonParams.repo}/-/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(serviceName, termsType)}.md`;
+    return `${this.baseURL}/${this.repositoryPath}/-/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(serviceName, termsType)}.md`;
   }
 
   generateSnapshotsBaseUrl(serviceName, termsType) {
-    return `${this.baseURL}/${this.commonParams.owner}/${this.commonParams.repo}/-/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}`;
+    return `${this.baseURL}/${this.repositoryPath}/-/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}`;
   }
 
   // GitLab API responses are not cached unlike GitHub, so this method only exists to satisfy the Reporter interface contract