@opentermsarchive/engine 3.0.0 → 4.0.1

package/config/ci.json

@@ -1,7 +1,5 @@
 {
   "@opentermsarchive/engine": {
-    "services": {
-      "declarationsPath": "./demo-declarations/declarations"
-    }
+    "collectionPath": "./demo-declarations/"
   }
 }

package/config/default.json

@@ -1,9 +1,7 @@
 {
   "@opentermsarchive/engine": {
     "trackingSchedule": "30 */12 * * *",
-    "services": {
-      "declarationsPath": "./declarations"
-    },
+    "collectionPath": "./",
     "recorder": {
       "versions": {
         "storage": {

package/config/test.json

@@ -1,8 +1,6 @@
 {
   "@opentermsarchive/engine": {
-    "services": {
-      "declarationsPath": "./test/services"
-    },
+    "collectionPath": "./test/test-declarations",
     "recorder": {
       "versions": {
         "storage": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "3.0.0",
+  "version": "4.0.1",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {
@@ -82,6 +82,7 @@
     "https-proxy-agent": "^5.0.0",
     "iconv-lite": "^0.6.3",
     "joplin-turndown-plugin-gfm": "^1.0.12",
+    "js-yaml": "^4.1.0",
     "jsdom": "^18.1.0",
     "json-source-map": "^0.6.1",
     "lodash": "^4.17.21",

package/scripts/declarations/lint/index.mocha.js

@@ -16,8 +16,8 @@ const ESLINT_CONFIG_PATH = path.join(ROOT_PATH, '.eslintrc.yaml');
 const eslint = new ESLint({ overrideConfigFile: ESLINT_CONFIG_PATH, fix: false });
 const eslintWithFix = new ESLint({ overrideConfigFile: ESLINT_CONFIG_PATH, fix: true });
 
-const declarationsPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.services.declarationsPath'));
-const instancePath = path.resolve(declarationsPath, '../');
+const instancePath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.collectionPath'));
+const declarationsPath = path.resolve(instancePath, services.DECLARATIONS_PATH);
 
 export default async options => {
   let servicesToValidate = options.services || [];

package/scripts/declarations/validate/index.mocha.js

@@ -19,8 +19,8 @@ const fs = fsApi.promises;
 const MIN_DOC_LENGTH = 100;
 const SLOW_DOCUMENT_THRESHOLD = 10 * 1000; // number of milliseconds after which a document fetch is considered slow
 
-const declarationsPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.services.declarationsPath'));
-const instancePath = path.resolve(declarationsPath, '../');
+const instancePath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.collectionPath'));
+const declarationsPath = path.resolve(instancePath, services.DECLARATIONS_PATH);
 
 export default async options => {
   const schemaOnly = options.schemaOnly || false;

package/src/archivist/services/index.js

@@ -1,4 +1,4 @@
-import fsApi from 'fs';
+import fs from 'fs/promises';
 import path from 'path';
 import { pathToFileURL } from 'url';
 
@@ -8,8 +8,8 @@ import Service from './service.js';
 import SourceDocument from './sourceDocument.js';
 import Terms from './terms.js';
 
-const fs = fsApi.promises;
-const declarationsPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.services.declarationsPath'));
+export const DECLARATIONS_PATH = './declarations';
+const declarationsPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.collectionPath'), DECLARATIONS_PATH);
 
 export async function load(servicesIdsToLoad = []) {
   let servicesIds = await getDeclaredServicesIds();

package/src/archivist/services/service.js

@@ -70,6 +70,6 @@ export default class Service {
   }
 
   static getNumberOfTerms(services, servicesIds, termsTypes) {
-    return servicesIds.reduce((acc, serviceId) => acc + services[serviceId].getNumberOfTerms(termsTypes), 0);
+    return (servicesIds || Object.keys(services)).reduce((acc, serviceId) => acc + services[serviceId].getNumberOfTerms(termsTypes), 0);
   }
 }

package/src/collection-api/routes/index.js

@@ -1,11 +1,17 @@
+import path from 'path';
+
+import config from 'config';
 import express from 'express';
 import helmet from 'helmet';
 
+import * as Services from '../../archivist/services/index.js';
+
 import docsRouter from './docs.js';
+import metadataRouter from './metadata.js';
 import servicesRouter from './services.js';
 import versionsRouter from './versions.js';
 
-export default function apiRouter(basePath) {
+export default async function apiRouter(basePath) {
   const router = express.Router();
 
   const defaultDirectives = helmet.contentSecurityPolicy.getDefaultDirectives();
@@ -27,7 +33,11 @@ export default function apiRouter(basePath) {
     res.json({ message: 'Welcome to an instance of the Open Terms Archive API. Documentation is available at /docs. Learn more on Open Terms Archive on https://opentermsarchive.org.' });
   });
 
-  router.use(servicesRouter);
+  const collectionPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.collectionPath'));
+  const services = await Services.load();
+
+  router.use(await metadataRouter(collectionPath, services));
+  router.use(servicesRouter(services));
   router.use(versionsRouter);
 
   return router;

package/src/collection-api/routes/metadata.js

@@ -0,0 +1,151 @@
+import fs from 'fs/promises';
+import path from 'path';
+
+import express from 'express';
+import yaml from 'js-yaml';
+
+import Service from '../../archivist/services/service.js';
+
+const METADATA_FILENAME = 'metadata.yml';
+const PACKAGE_JSON_PATH = '../../../package.json';
+
+/**
+ * @swagger
+ * tags:
+ *   name: Metadata
+ *   description: Collection metadata API
+ * components:
+ *   schemas:
+ *     Metadata:
+ *       type: object
+ *       description: Collection metadata
+ *       properties:
+ *         id:
+ *           type: string
+ *           description: Unique identifier of the collection
+ *         name:
+ *           type: string
+ *           description: Display name of the collection
+ *         tagline:
+ *           type: string
+ *           description: Short description of the collection
+ *         description:
+ *           type: string
+ *           nullable: true
+ *           description: Detailed description of the collection
+ *         totalTerms:
+ *           type: integer
+ *           description: Total number of terms tracked in the collection
+ *         totalServices:
+ *           type: integer
+ *           description: Total number of services tracked in the collection
+ *         engineVersion:
+ *           type: string
+ *           description: Version of the Open Terms Archive engine in SemVer format (MAJOR.MINOR.PATCH)
+ *         dataset:
+ *           type: string
+ *           format: uri
+ *           description: URL to the dataset releases
+ *         declarations:
+ *           type: string
+ *           format: uri
+ *           description: URL to the declarations repository
+ *         versions:
+ *           type: string
+ *           format: uri
+ *           description: URL to the versions repository
+ *         snapshots:
+ *           type: string
+ *           format: uri
+ *           description: URL to the snapshots repository
+ *         donations:
+ *           type: string
+ *           format: uri
+ *           description: URL to the donations page
+ *         logo:
+ *           type: string
+ *           format: uri
+ *           nullable: true
+ *           description: URL to the collection logo
+ *         languages:
+ *           type: array
+ *           items:
+ *             type: string
+ *           description: List of ISO 639 language codes representing languages allowed by the collection
+ *         jurisdictions:
+ *           type: array
+ *           items:
+ *             type: string
+ *           description: List of ISO 3166-2 country codes representing jurisdictions covered by the collection
+ *         trackingPeriods:
+ *           type: array
+ *           items:
+ *             type: object
+ *             properties:
+ *               startDate:
+ *                 type: string
+ *                 format: date
+ *                 description: The date when tracking started for this period
+ *               schedule:
+ *                 type: string
+ *                 description: A cron expression defining when terms are tracked (e.g. "0 0 * * *" for daily at midnight)
+ *               serverLocation:
+ *                 type: string
+ *                 description: The geographic location of the server used for tracking
+ *               endDate:
+ *                 type: string
+ *                 format: date
+ *                 description: The date when tracking ended for this period
+ *         governance:
+ *           type: object
+ *           additionalProperties:
+ *             type: object
+ *             properties:
+ *               url:
+ *                 type: string
+ *                 format: uri
+ *                 description: URL to the entity's website
+ *               logo:
+ *                 type: string
+ *                 format: uri
+ *                 description: URL to the entity's logo
+ *               roles:
+ *                 type: array
+ *                 items:
+ *                   type: string
+ *                   enum: [host, administrator, curator, maintainer, sponsor]
+ *                   description: Roles of the entity within the governance
+ */
+export default async function metadataRouter(collectionPath, services) {
+  const router = express.Router();
+
+  const STATIC_METADATA = yaml.load(await fs.readFile(path.join(collectionPath, METADATA_FILENAME), 'utf8'));
+  const { version: engineVersion } = JSON.parse(await fs.readFile(new URL(PACKAGE_JSON_PATH, import.meta.url)));
+
+  /**
+   * @swagger
+   * /metadata:
+   *   get:
+   *     summary: Get collection metadata
+   *     tags: [Metadata]
+   *     produces:
+   *       - application/json
+   *     responses:
+   *       200:
+   *         description: Collection metadata
+   */
+  router.get('/metadata', (req, res) => {
+    const dynamicMetadata = {
+      totalServices: Object.keys(services).length,
+      totalTerms: Service.getNumberOfTerms(services),
+      engineVersion,
+    };
+
+    res.json({
+      ...STATIC_METADATA,
+      ...dynamicMetadata,
+    });
+  });
+
+  return router;
+}

package/src/collection-api/routes/metadata.test.js

@@ -0,0 +1,89 @@
+import fs from 'fs/promises';
+
+import { expect } from 'chai';
+import config from 'config';
+import request from 'supertest';
+
+import app from '../server.js';
+
+const basePath = config.get('@opentermsarchive/engine.collection-api.basePath');
+const { version: engineVersion } = JSON.parse(await fs.readFile(new URL('../../../package.json', import.meta.url)));
+
+const EXPECTED_RESPONSE = {
+  totalServices: 7,
+  totalTerms: 8,
+  id: 'test',
+  name: 'test',
+  tagline: 'Test collection',
+  description: 'This is a test collection used for testing purposes.',
+  dataset: 'https://github.com/OpenTermsArchive/test-versions/releases',
+  declarations: 'https://github.com/OpenTermsArchive/test-declarations',
+  versions: 'https://github.com/OpenTermsArchive/test-versions',
+  snapshots: 'https://github.com/OpenTermsArchive/test-snapshots',
+  donations: null,
+  logo: 'https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png',
+  languages: [
+    'en',
+  ],
+  jurisdictions: [
+    'EU',
+  ],
+  governance: {
+    hosts: [
+      { name: 'Localhost' },
+    ],
+    administrators: [
+      {
+        name: 'Open Terms Archive',
+        url: 'https://opentermsarchive.org/',
+        logo: 'https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png',
+      },
+    ],
+    curators: [
+      {
+        name: 'Open Terms Archive',
+        url: 'https://opentermsarchive.org/',
+        logo: 'https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png',
+      },
+    ],
+    maintainers: [
+      {
+        name: 'Open Terms Archive',
+        url: 'https://opentermsarchive.org/',
+        logo: 'https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png',
+      },
+    ],
+    sponsors: [
+      {
+        name: 'Open Terms Archive',
+        url: 'https://opentermsarchive.org/',
+        logo: 'https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png',
+      },
+    ],
+  },
+};
+
+describe('Metadata API', () => {
+  describe('GET /metadata', () => {
+    let response;
+
+    before(async () => {
+      response = await request(app).get(`${basePath}/v1/metadata`);
+    });
+
+    it('responds with 200 status code', () => {
+      expect(response.status).to.equal(200);
+    });
+
+    it('responds with Content-Type application/json', () => {
+      expect(response.type).to.equal('application/json');
+    });
+
+    it('returns expected metadata object', () => {
+      expect(response.body).to.deep.equal({
+        ...EXPECTED_RESPONSE,
+        engineVersion,
+      });
+    });
+  });
+});

package/src/collection-api/routes/services.js

@@ -1,9 +1,5 @@
 import express from 'express';
 
-import * as Services from '../../archivist/services/index.js';
-
-const services = await Services.load();
-
 /**
  * @swagger
  * tags:
@@ -54,106 +50,108 @@ const services = await Services.load();
  *                       items:
  *                         type: string
  */
-const router = express.Router();
+export default function servicesRouter(services) {
+  const router = express.Router();
 
-/**
- * @swagger
- * /services:
- *   get:
- *     summary: Enumerate all services.
- *     tags: [Services]
- *     produces:
- *       - application/json
- *     responses:
- *       200:
- *         description: A JSON array of all services.
- *         content:
- *           application/json:
- *             schema:
- *               type: array
- *               items:
- *                 type: object
- *                 properties:
- *                   id:
- *                     type: string
- *                     description: The ID of the service.
- *                   name:
- *                     type: string
- *                     description: The name of the service.
- *                   terms:
- *                     type: array
- *                     description: The declared terms types for this service.
- *                     items:
- *                       type: object
- *                       properties:
- *                         type:
- *                           type: string
- *                           description: The type of terms.
- */
-router.get('/services', (req, res) => {
-  res.status(200).json(Object.values(services).map(service => ({
-    id: service.id,
-    name: service.name,
-    terms: service.getTermsTypes().map(type => ({ type })),
-  })));
-});
+  /**
+   * @swagger
+   * /services:
+   *   get:
+   *     summary: Enumerate all services.
+   *     tags: [Services]
+   *     produces:
+   *       - application/json
+   *     responses:
+   *       200:
+   *         description: A JSON array of all services.
+   *         content:
+   *           application/json:
+   *             schema:
+   *               type: array
+   *               items:
+   *                 type: object
+   *                 properties:
+   *                   id:
+   *                     type: string
+   *                     description: The ID of the service.
+   *                   name:
+   *                     type: string
+   *                     description: The name of the service.
+   *                   terms:
+   *                     type: array
+   *                     description: The declared terms types for this service.
+   *                     items:
+   *                       type: object
+   *                       properties:
+   *                         type:
+   *                           type: string
+   *                           description: The type of terms.
+   */
+  router.get('/services', (req, res) => {
+    res.status(200).json(Object.values(services).map(service => ({
+      id: service.id,
+      name: service.name,
+      terms: service.getTermsTypes().map(type => ({ type })),
+    })));
+  });
 
-/**
- * @swagger
- * /service/{serviceId}:
- *   get:
- *     summary: Retrieve the declaration of a specific service through its ID.
- *     tags: [Services]
- *     produces:
- *       - application/json
- *     parameters:
- *       - in: path
- *         name: serviceId
- *         description: The ID of the service.
- *         schema:
- *           type: string
- *         required: true
- *         examples:
- *             service-1:
- *               value: service-1
- *               summary: Simple service ID
- *             service-2:
- *               value: Service 2!
- *               summary: Service ID with spaces and special characters
- *     responses:
- *       200:
- *         description: The full JSON declaration of the service with the given ID.
- *         content:
- *           application/json:
- *             schema:
- *               $ref: '#/components/schemas/Service'
- *       404:
- *         description: No service matching the provided ID is found.
- */
-router.get('/service/:serviceId', (req, res) => {
-  const matchedServiceID = Object.keys(services).find(key => key.toLowerCase() === req.params.serviceId?.toLowerCase());
-  const service = services[matchedServiceID];
+  /**
+   * @swagger
+   * /service/{serviceId}:
+   *   get:
+   *     summary: Retrieve the declaration of a specific service through its ID.
+   *     tags: [Services]
+   *     produces:
+   *       - application/json
+   *     parameters:
+   *       - in: path
+   *         name: serviceId
+   *         description: The ID of the service.
+   *         schema:
+   *           type: string
+   *         required: true
+   *         examples:
+   *             service-1:
+   *               value: service-1
+   *               summary: Simple service ID
+   *             service-2:
+   *               value: Service 2!
+   *               summary: Service ID with spaces and special characters
+   *     responses:
+   *       200:
+   *         description: The full JSON declaration of the service with the given ID.
+   *         content:
+   *           application/json:
+   *             schema:
+   *               $ref: '#/components/schemas/Service'
+   *       404:
+   *         description: No service matching the provided ID is found.
+   */
+  router.get('/service/:serviceId', (req, res) => {
+    const matchedServiceID = Object.keys(services).find(key => key.toLowerCase() === req.params.serviceId?.toLowerCase());
+    const service = services[matchedServiceID];
 
-  if (!service) {
-    res.status(404).send('Service not found');
+    if (!service) {
+      res.status(404).send('Service not found');
 
-    return;
-  }
+      return;
+    }
 
-  res.status(200).json({
-    id: service.id,
-    name: service.name,
-    terms: service.getTerms().map(terms => ({
-      type: terms.type,
-      sourceDocuments: terms.sourceDocuments.map(({ location, contentSelectors, insignificantContentSelectors, filters, executeClientScripts }) => ({
-        location,
-        contentSelectors,
-        insignificantContentSelectors,
-        executeClientScripts,
-        filters: filters?.map(filter => filter.name),
+    res.status(200).json({
+      id: service.id,
+      name: service.name,
+      terms: service.getTerms().map(terms => ({
+        type: terms.type,
+        sourceDocuments: terms.sourceDocuments.map(({ location, contentSelectors, insignificantContentSelectors, filters, executeClientScripts }) => ({
+          location,
+          contentSelectors,
+          insignificantContentSelectors,
+          executeClientScripts,
+          filters: filters?.map(filter => filter.name),
+        })),
       })),
-    })),
+    });
   });
-});
 
-export default router;
+  return router;
+}

package/src/collection-api/server.js

@@ -15,7 +15,7 @@ if (process.env.NODE_ENV !== 'test') {
 
 const BASE_PATH = `/${config.get('@opentermsarchive/engine.collection-api.basePath')}/v1`.replace(/\/\/+/g, '/'); // ensure there are no double slashes
 
-app.use(BASE_PATH, apiRouter(BASE_PATH));
+app.use(BASE_PATH, await apiRouter(BASE_PATH));
 app.use(errorsMiddleware);
 
 const port = config.get('@opentermsarchive/engine.collection-api.port');