@opentermsarchive/engine 11.0.2 → 12.0.1

package/src/collection-api/routes/index.js

@@ -1,10 +1,13 @@
+import config from 'config';
 import express from 'express';
 import helmet from 'helmet';
 
 import { getCollection } from '../../archivist/collection/index.js';
+import RepositoryFactory from '../../archivist/recorder/repositories/factory.js';
 import * as Services from '../../archivist/services/index.js';
 
 import docsRouter from './docs.js';
+import feedRouter from './feed.js';
 import metadataRouter from './metadata.js';
 import servicesRouter from './services.js';
 import versionsRouter from './versions.js';
@@ -33,10 +36,22 @@ export default async function apiRouter(basePath) {
 
   const services = await Services.load();
   const collection = await getCollection();
+  const versionsStorageConfig = config.get('@opentermsarchive/engine.recorder.versions.storage');
+  const versionsRepository = await RepositoryFactory.create(versionsStorageConfig).initialize();
+  const feedConfig = config.get('@opentermsarchive/engine.collection-api.feed');
+
+  if (!collection.metadata?.id) {
+    throw new Error('Collection metadata "id" is required to expose feed endpoints, as it is used to build the tag URIs that uniquely identify the feed and its entries. Add an "id" field to the collection metadata file.');
+  }
+
+  if (!collection.metadata?.name) {
+    throw new Error('Collection metadata "name" is required to expose feed endpoints, as it is used as the Atom feed title which the Atom 1.0 specification requires to be non-empty. Add a "name" field to the collection metadata file.');
+  }
 
   router.use(await metadataRouter(collection, services));
   router.use(servicesRouter(services));
-  router.use(versionsRouter);
+  router.use(versionsRouter(versionsRepository));
+  router.use(feedRouter(services, versionsRepository, versionsStorageConfig.type, feedConfig.limit, feedConfig.versionUrlTemplate));
 
   return router;
 }

package/src/collection-api/routes/services.js

@@ -130,8 +130,7 @@ export default function servicesRouter(services) {
    *         description: No service matching the provided ID is found.
    */
   router.get('/service/:serviceId', (req, res) => {
-    const matchedServiceID = Object.keys(services).find(key => key.toLowerCase() === req.params.serviceId?.toLowerCase());
-    const service = services[matchedServiceID];
+    const service = Object.hasOwn(services, req.params.serviceId) ? services[req.params.serviceId] : null;
 
     if (!service) {
       res.status(404).send('Service not found');

package/src/collection-api/routes/services.test.js

@@ -56,7 +56,6 @@ describe('Services API', () => {
   describe('GET /service/:serviceId', () => {
     let response;
     const SERVICE_ID = 'Service B!';
-    const CASE_INSENSITIVE_SERVICE_ID = 'service b!';
 
     before(async () => {
       response = await request(app).get(`${basePath}/v1/service/${encodeURI(SERVICE_ID)}`);
@@ -106,49 +105,13 @@ describe('Services API', () => {
       });
     });
 
-    context('with a case-insensitive service ID parameter', () => {
+    context('when the service ID casing does not match', () => {
       before(async () => {
-        response = await request(app).get(`${basePath}/v1/service/${encodeURI(CASE_INSENSITIVE_SERVICE_ID)}`);
+        response = await request(app).get(`${basePath}/v1/service/${encodeURI(SERVICE_ID.toLowerCase())}`);
       });
 
-      it('responds with 200 status code', () => {
-        expect(response.status).to.equal(200);
-      });
-
-      it('returns a service object with id', () => {
-        expect(response.body).to.have.property('id');
-      });
-
-      it('returns the proper service object', () => {
-        expect(response.body.id).to.equal(SERVICE_ID);
-      });
-
-      it('returns a service object with name', () => {
-        expect(response.body).to.have.property('name');
-      });
-
-      it('returns a service object with an array of terms', () => {
-        expect(response.body).to.have.property('terms').that.is.an('array');
-      });
-
-      it('each terms should have a type property', () => {
-        response.body.terms.forEach(terms => {
-          expect(terms).to.have.property('type');
-        });
-      });
-
-      it('each terms should have an array of source documents', () => {
-        response.body.terms.forEach(terms => {
-          expect(terms).to.have.property('sourceDocuments').that.is.an('array');
-        });
-      });
-
-      it('each source document should have a location', () => {
-        response.body.terms.forEach(terms => {
-          terms.sourceDocuments.forEach(sourceDocument => {
-            expect(sourceDocument).to.have.property('location');
-          });
-        });
+      it('responds with 404 status code', () => {
+        expect(response.status).to.equal(404);
       });
     });
 

package/src/collection-api/routes/versions.js

@@ -1,10 +1,10 @@
-import config from 'config';
 import express from 'express';
 
-import RepositoryFactory from '../../archivist/recorder/repositories/factory.js';
 import { toISODateWithoutMilliseconds } from '../../archivist/utils/date.js';
 
 /**
+ * @param   {object}         versionsRepository The versions repository instance
+ * @returns {express.Router}                    The router instance
  * @private
  * @swagger
  * tags:
@@ -27,86 +27,86 @@ import { toISODateWithoutMilliseconds } from '../../archivist/utils/date.js';
  *           type: string
  *           description: The JSON-escaped Markdown content of the version
  */
-const router = express.Router();
+export default function versionsRouter(versionsRepository) {
+  const router = express.Router();
 
-const versionsRepository = await RepositoryFactory.create(config.get('@opentermsarchive/engine.recorder.versions.storage')).initialize();
+  /**
+   * @private
+   * @swagger
+   * /version/{serviceId}/{termsType}/{date}:
+   *   get:
+   *     summary: Get a specific version of some terms at a given date.
+   *     tags: [Versions]
+   *     produces:
+   *       - application/json
+   *     parameters:
+   *       - in: path
+   *         name: serviceId
+   *         description: The ID of the service whose version will be returned.
+   *         schema:
+   *           type: string
+   *         required: true
+   *       - in: path
+   *         name: termsType
+   *         description: The type of terms whose version will be returned.
+   *         schema:
+   *           type: string
+   *         required: true
+   *       - in: path
+   *         name: date
+   *         description: The date and time for which the version is requested, in ISO 8601 format.
+   *         schema:
+   *           type: string
+   *           format: date-time
+   *         required: true
+   *     responses:
+   *       200:
+   *         description: A JSON object containing the version content and metadata.
+   *         content:
+   *           application/json:
+   *             schema:
+   *               $ref: '#/components/schemas/Version'
+   *       404:
+   *         description: No version found for the specified combination of service ID, terms type and date.
+   *         content:
+   *           application/json:
+   *             schema:
+   *               type: object
+   *               properties:
+   *                 error:
+   *                   type: string
+   *                   description: Error message indicating that no version is found.
+   *       416:
+   *         description: The requested date is in the future.
+   *         content:
+   *           application/json:
+   *             schema:
+   *               type: object
+   *               properties:
+   *                 error:
+   *                   type: string
+   *                   description: Error message indicating that the requested date is in the future.
+   */
+  router.get('/version/:serviceId/:termsType/:date', async (req, res) => {
+    const { serviceId, termsType, date } = req.params;
+    const requestedDate = new Date(date);
 
-/**
- * @private
- * @swagger
- * /version/{serviceId}/{termsType}/{date}:
- *   get:
- *     summary: Get a specific version of some terms at a given date.
- *     tags: [Versions]
- *     produces:
- *       - application/json
- *     parameters:
- *       - in: path
- *         name: serviceId
- *         description: The ID of the service whose version will be returned.
- *         schema:
- *           type: string
- *         required: true
- *       - in: path
- *         name: termsType
- *         description: The type of terms whose version will be returned.
- *         schema:
- *           type: string
- *         required: true
- *       - in: path
- *         name: date
- *         description: The date and time for which the version is requested, in ISO 8601 format.
- *         schema:
- *           type: string
- *           format: date-time
- *         required: true
- *     responses:
- *       200:
- *         description: A JSON object containing the version content and metadata.
- *         content:
- *           application/json:
- *             schema:
- *               $ref: '#/components/schemas/Version'
- *       404:
- *         description: No version found for the specified combination of service ID, terms type and date.
- *         content:
- *           application/json:
- *             schema:
- *               type: object
- *               properties:
- *                 error:
- *                   type: string
- *                   description: Error message indicating that no version is found.
- *       416:
- *         description: The requested date is in the future.
- *         content:
- *           application/json:
- *             schema:
- *               type: object
- *               properties:
- *                 error:
- *                   type: string
- *                   description: Error message indicating that the requested date is in the future.
- */
-router.get('/version/:serviceId/:termsType/:date', async (req, res) => {
-  const { serviceId, termsType, date } = req.params;
-  const requestedDate = new Date(date);
-
-  if (requestedDate > new Date()) {
-    return res.status(416).json({ error: 'Requested version is in the future' });
-  }
+    if (requestedDate > new Date()) {
+      return res.status(416).json({ error: 'Requested version is in the future' });
+    }
 
-  const version = await versionsRepository.findByDate(serviceId, termsType, requestedDate);
+    const version = await versionsRepository.findByDate(serviceId, termsType, requestedDate);
 
-  if (!version) {
-    return res.status(404).json({ error: `No version found for date ${date}` });
-  }
+    if (!version) {
+      return res.status(404).json({ error: `No version found for date ${date}` });
+    }
 
-  return res.status(200).json({
-    id: version.id,
-    fetchDate: toISODateWithoutMilliseconds(version.fetchDate),
-    content: version.content,
+    return res.status(200).json({
+      id: version.id,
+      fetchDate: toISODateWithoutMilliseconds(version.fetchDate),
+      content: version.content,
+    });
   });
-});
 
-export default router;
+  return router;
+}

package/src/collection-api/routes/versions.test.js

@@ -17,7 +17,7 @@ describe('Versions API', () => {
     let versionsRepository;
     const FETCH_DATE = new Date('2023-01-01T12:00:00Z');
     const VERSION_COMMON_ATTRIBUTES = {
-      serviceId: 'service-1',
+      serviceId: 'service·A',
       termsType: 'Terms of Service',
       snapshotId: ['snapshot_id'],
     };
@@ -62,7 +62,7 @@ describe('Versions API', () => {
 
     context('when a version is found', () => {
       before(async () => {
-        response = await request.get(`${basePath}/v1/version/service-1/Terms%20of%20Service/${encodeURIComponent(toISODateWithoutMilliseconds(FETCH_DATE))}`);
+        response = await request.get(`${basePath}/v1/version/service·A/Terms%20of%20Service/${encodeURIComponent(toISODateWithoutMilliseconds(FETCH_DATE))}`);
       });
 
       it('responds with 200 status code', () => {
@@ -80,7 +80,7 @@ describe('Versions API', () => {
 
     context('when the requested date is anterior to the first available version', () => {
       before(async () => {
-        response = await request.get(`${basePath}/v1/version/service-1/Terms%20of%20Service/2000-01-01T12:00:00Z`);
+        response = await request.get(`${basePath}/v1/version/service·A/Terms%20of%20Service/2000-01-01T12:00:00Z`);
       });
 
       it('responds with 404 status code', () => {
@@ -100,7 +100,7 @@ describe('Versions API', () => {
       before(async () => {
         const dateInTheFuture = new Date(Date.now() + 60000); // 1 minute in the future
 
-        response = await request.get(`${basePath}/v1/version/service-1/Terms%20of%20Service/${encodeURIComponent(toISODateWithoutMilliseconds(dateInTheFuture))}`);
+        response = await request.get(`${basePath}/v1/version/service·A/Terms%20of%20Service/${encodeURIComponent(toISODateWithoutMilliseconds(dateInTheFuture))}`);
       });
 
       it('responds with 416 status code', () => {

package/src/collection-api/server.js

@@ -8,6 +8,8 @@ import apiRouter from './routes/index.js';
 
 const app = express();
 
+app.set('trust proxy', 'loopback'); // The API binds to 127.0.0.1 and is expected to run behind a reverse proxy. Honour X-Forwarded-* headers only when they come from a local proxy so absolute URLs emitted by routes (notably Atom feed links) reflect the URL seen by clients rather than the internal http://127.0.0.1 hop.
+
 if (process.env.NODE_ENV !== 'test') {
   app.use(loggerMiddleware);
 }

package/src/reporter/gitlab/index.js

@@ -358,7 +358,7 @@ export default class GitLab {
     try {
       let apiUrl = `${this.apiBaseURL}/projects/${this.projectId}/issues?search=${encodeURIComponent(title)}&state=${searchParams.state}&per_page=100`;
 
-      if (searchParams.state == 'all') apiUrl = `${this.apiBaseURL}/projects/${this.projectId}/issues?search=${encodeURIComponent(title)}&per_page=100`;
+      if (searchParams.state == 'all') { apiUrl = `${this.apiBaseURL}/projects/${this.projectId}/issues?search=${encodeURIComponent(title)}&per_page=100`; }
 
       const options = GitLab.baseOptionsHttpReq();
 

package/src/reporter/index.js

@@ -137,7 +137,11 @@ No changes were found in the last run, so no new version has been recorded.`,
     });
     const contributionToolUrl = `${CONTRIBUTION_TOOL_URL}?${contributionToolParams}`;
 
-    const latestDeclarationLink = `[Latest declaration](${this.reporter.generateDeclarationURL(terms.service.name)})`;
+    const declarationFileUrl = this.reporter.generateDeclarationURL(terms.service.name);
+    const updateDeclarationLink = terms.hasMultipleSourceDocuments ? `[on GitHub](${declarationFileUrl})` : `[on the contribution tool](${contributionToolUrl})`;
+    const multiDocumentsUpdateInfo = terms.hasMultipleSourceDocuments ? ' (the contribution tool does not support multi-document)' : '';
+
+    const latestDeclarationLink = `[Latest declaration](${declarationFileUrl})`;
     const latestVersionLink = `[Latest version](${this.reporter.generateVersionURL(terms.service.name, terms.type)})`;
     const snapshotsBaseUrl = this.reporter.generateSnapshotsBaseUrl(terms.service.name, terms.type);
     const latestSnapshotsLink = terms.hasMultipleSourceDocuments
@@ -163,13 +167,13 @@ First of all, check if the source documents are accessible through a web browser
 
 #### If the source documents are accessible through a web browser
 
-[Edit the declaration](${contributionToolUrl}):
+Edit the declaration ${updateDeclarationLink}${multiDocumentsUpdateInfo}:
 - Try updating the selectors.
 - Try switching client scripts on with expert mode.
 
 #### If the source documents are not accessible anymore
 
-- If the source documents have moved, find their new location and [update it](${contributionToolUrl}).
+- If the source documents have moved, find their new location and update it ${updateDeclarationLink}.
 - If these terms have been removed, move them from the declaration to its [history file](${DOC_URL}/terms/explanation/declarations-maintenance/#service-history-reference), using \`${validUntil}\` as the \`validUntil\` value.
 - If the service has closed, move the entire contents of the declaration to its [history file](${DOC_URL}/contributing-terms/#service-history), using \`${validUntil}\` as the \`validUntil\` value.
 

package/src/reporter/index.test.js

@@ -60,4 +60,66 @@ describe('Reporter', () => {
       });
     });
   });
+
+  describe('#generateDescription', () => {
+    const buildReporter = () => new Reporter({
+      type: 'github',
+      repositories: { declarations: 'OpenTermsArchive/test-declarations' },
+    });
+
+    const buildTerms = ({ sourceCount = 1 } = {}) => {
+      const sourceDocuments = Array.from({ length: sourceCount }, (_, index) => ({
+        id: `source-${index}`,
+        location: `https://example.com/source-${index}`,
+        mimeType: 'text/html',
+        snapshotId: `snapshot-${index}`,
+        toPersistence: () => ({ fetch: `https://example.com/source-${index}` }),
+      }));
+
+      return {
+        service: { id: 'TestService', name: 'TestService' },
+        type: 'Terms of Service',
+        sourceDocuments,
+        hasMultipleSourceDocuments: sourceCount > 1,
+        toPersistence: () => ({
+          name: 'TestService',
+          terms: {
+            'Terms of Service': sourceCount > 1
+              ? { combine: sourceDocuments.map(sourceDocument => sourceDocument.toPersistence()) }
+              : sourceDocuments[0].toPersistence(),
+          },
+        }),
+      };
+    };
+
+    const error = { reasons: ['HTTP code 404'] };
+
+    context('when the terms has a single source document', () => {
+      it('deep-links to the contribution tool with the serialized declaration as the edit target', () => {
+        const description = buildReporter().generateDescription({ error, terms: buildTerms({ sourceCount: 1 }) });
+
+        expect(description).to.match(/\(https:\/\/contribute\.opentermsarchive\.org\/[^)]*\bjson=[^)]*\)/);
+      });
+    });
+
+    context('when the terms has multiple source documents (combine)', () => {
+      it('does not deep-link to the contribution tool because it cannot edit multi-source declarations', () => {
+        const description = buildReporter().generateDescription({ error, terms: buildTerms({ sourceCount: 5 }) });
+
+        expect(description).to.not.include('json=');
+      });
+
+      it('links to the declaration file on GitHub as the edit target', () => {
+        const description = buildReporter().generateDescription({ error, terms: buildTerms({ sourceCount: 5 }) });
+
+        expect(description).to.include('github.com/OpenTermsArchive/test-declarations/blob/main/declarations/TestService.json');
+      });
+
+      it('keeps the description below the GitHub 65,536-character issue body limit even with many sources', () => {
+        const description = buildReporter().generateDescription({ error, terms: buildTerms({ sourceCount: 50 }) });
+
+        expect(description.length).to.be.lessThan(65000);
+      });
+    });
+  });
 });