@opentermsarchive/engine 0.33.1 → 0.34.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "0.33.1",
+  "version": "0.34.0",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://github.com/OpenTermsArchive/engine#readme",
   "bugs": {
@@ -71,6 +71,7 @@
     "eslint-plugin-import": "^2.29.0",
     "eslint-plugin-json-format": "^2.0.1",
     "express": "^4.18.2",
+    "express-async-errors": "^3.1.1",
     "fs-extra": "^10.0.0",
     "helmet": "^6.0.1",
     "http-proxy-agent": "^5.0.0",

package/src/api/middlewares/errors.js

@@ -2,5 +2,5 @@ import logger from '../logger.js';
 
 export default function errorsMiddleware(err, req, res, next) {
   logger.error(err.stack);
-  res.status(500).send('Something went wrong!');
+  res.status(500).json({ error: err.message });
 }

package/src/api/routes/docs.js

@@ -13,7 +13,7 @@ export default function specsRouter(basePath) {
     definition: {
       openapi: '3.1.0',
       info: {
-        title: 'Open Terms Archive API',
+        title: 'Open Terms Archive API [Beta]',
         version: '1.0.0',
         license: {
           name: 'EUPL-1.2',

package/src/api/routes/docs.test.js

@@ -10,7 +10,7 @@ describe('Docs API', () => {
   describe('GET /docs', () => {
     let response;
 
-    context('When requested as JSON', () => {
+    context('when requested as JSON', () => {
       before(async () => {
         response = await request(app).get(`${basePath}/v1/docs/`).set('Accept', 'application/json');
       });
@@ -50,11 +50,15 @@ describe('Docs API', () => {
           it('/service/{serviceId}', () => {
             expect(subject).to.have.property('/service/{serviceId}');
           });
+
+          it('/version/{serviceId}/{termsType}/{date}', () => {
+            expect(subject).to.have.property('/version/{serviceId}/{termsType}/{date}');
+          });
         });
       });
     });
 
-    context('When requested as HTML', () => {
+    context('when requested as HTML', () => {
       before(async () => {
         response = await request(app).get(`${basePath}/v1/docs/`);
       });

package/src/api/routes/index.js

@@ -3,6 +3,7 @@ import helmet from 'helmet';
 
 import docsRouter from './docs.js';
 import servicesRouter from './services.js';
+import versionsRouter from './versions.js';
 
 export default function apiRouter(basePath) {
   const router = express.Router();
@@ -27,6 +28,7 @@ export default function apiRouter(basePath) {
   });
 
   router.use(servicesRouter);
+  router.use(versionsRouter);
 
   return router;
 }

package/src/api/routes/services.test.js

@@ -106,7 +106,7 @@ describe('Services API', () => {
       });
     });
 
-    context('With a case-insensitive service ID parameter', () => {
+    context('with a case-insensitive service ID parameter', () => {
       before(async () => {
         response = await request(app).get(`${basePath}/v1/service/${encodeURI(CASE_INSENSITIVE_SERVICE_ID)}`);
       });
@@ -152,7 +152,7 @@ describe('Services API', () => {
       });
     });
 
-    context('When no matching service is found', () => {
+    context('when no matching service is found', () => {
       before(async () => {
         response = await request(app).get(`${basePath}/v1/service/nonExistentService`);
       });

package/src/api/routes/versions.js

@@ -0,0 +1,110 @@
+import config from 'config';
+import express from 'express';
+
+import RepositoryFactory from '../../archivist/recorder/repositories/factory.js';
+import { toISODateWithoutMilliseconds } from '../../archivist/utils/date.js';
+
+/**
+ * @swagger
+ * tags:
+ *   name: Versions
+ *   description: Versions API
+ * components:
+ *   schemas:
+ *     Version:
+ *       type: object
+ *       description: Version content and metadata
+ *       properties:
+ *         fetchDate:
+ *           type: string
+ *           format: date-time
+ *           description: The ISO 8601 datetime string when the version was recorded.
+ *         id:
+ *           type: string
+ *           description: The ID of the version.
+ *         content:
+ *           type: string
+ *           description: The JSON-escaped Markdown content of the version.
+ */
+const router = express.Router();
+
+const versionsRepository = await RepositoryFactory.create(config.get('recorder.versions.storage')).initialize();
+
+/**
+ * @swagger
+ * /version/{serviceId}/{termsType}/{date}:
+ *   get:
+ *     summary: Get a specific version of some terms at a given date.
+ *     tags: [Versions]
+ *     produces:
+ *       - application/json
+ *     parameters:
+ *       - in: path
+ *         name: serviceId
+ *         description: The ID of the service whose version will be returned.
+ *         schema:
+ *           type: string
+ *         required: true
+ *       - in: path
+ *         name: termsType
+ *         description: The type of terms whose version will be returned.
+ *         schema:
+ *           type: string
+ *         required: true
+ *       - in: path
+ *         name: date
+ *         description: The date and time for which the version is requested, in ISO 8601 format.
+ *         schema:
+ *           type: string
+ *           format: date-time
+ *         required: true
+ *     responses:
+ *       200:
+ *         description: A JSON object containing the version content and metadata.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               $ref: '#/components/schemas/Version'
+ *       404:
+ *         description: No version found for the specified combination of service ID, terms type and date.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               type: object
+ *               properties:
+ *                 error:
+ *                   type: string
+ *                   description: Error message indicating that no version is found.
+ *       416:
+ *         description: The requested date is in the future.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               type: object
+ *               properties:
+ *                 error:
+ *                   type: string
+ *                   description: Error message indicating that the requested date is in the future.
+ */
+router.get('/version/:serviceId/:termsType/:date', async (req, res) => {
+  const { serviceId, termsType, date } = req.params;
+  const requestedDate = new Date(date);
+
+  if (requestedDate > new Date()) {
+    return res.status(416).json({ error: 'Requested version is in the future' });
+  }
+
+  const version = await versionsRepository.findByDate(serviceId, termsType, requestedDate);
+
+  if (!version) {
+    return res.status(404).json({ error: `No version found for date ${date}` });
+  }
+
+  return res.status(200).json({
+    id: version.id,
+    fetchDate: toISODateWithoutMilliseconds(version.fetchDate),
+    content: version.content,
+  });
+});
+
+export default router;

package/src/api/routes/versions.test.js

@@ -0,0 +1,120 @@
+import chai from 'chai';
+import config from 'config';
+import supertest from 'supertest';
+
+import RepositoryFactory from '../../archivist/recorder/repositories/factory.js';
+import Version from '../../archivist/recorder/version.js';
+import { toISODateWithoutMilliseconds } from '../../archivist/utils/date.js';
+import app from '../server.js';
+
+const basePath = config.get('api.basePath');
+
+const { expect } = chai;
+const request = supertest(app);
+
+describe('Versions API', () => {
+  describe('GET /version/:serviceId/:termsType/:date', () => {
+    let expectedResult;
+    let versionsRepository;
+    const FETCH_DATE = new Date('2023-01-01T12:00:00Z');
+    const VERSION_COMMON_ATTRIBUTES = {
+      serviceId: 'service-1',
+      termsType: 'Terms of Service',
+      snapshotId: ['snapshot_id'],
+    };
+
+    before(async () => {
+      versionsRepository = RepositoryFactory.create(config.get('recorder.versions.storage'));
+
+      await versionsRepository.initialize();
+
+      const ONE_HOUR = 60 * 60 * 1000;
+
+      await versionsRepository.save(new Version({
+        ...VERSION_COMMON_ATTRIBUTES,
+        content: 'initial content',
+        fetchDate: new Date(new Date(FETCH_DATE).getTime() - ONE_HOUR),
+      }));
+
+      const version = new Version({
+        ...VERSION_COMMON_ATTRIBUTES,
+        content: 'updated content',
+        fetchDate: FETCH_DATE,
+      });
+
+      await versionsRepository.save(version);
+
+      await versionsRepository.save(new Version({
+        ...VERSION_COMMON_ATTRIBUTES,
+        content: 'latest content',
+        fetchDate: new Date(new Date(FETCH_DATE).getTime() + ONE_HOUR),
+      }));
+
+      expectedResult = {
+        id: version.id,
+        fetchDate: toISODateWithoutMilliseconds(version.fetchDate),
+        content: version.content,
+      };
+    });
+
+    after(async () => versionsRepository.removeAll());
+
+    let response;
+
+    context('when a version is found', () => {
+      before(async () => {
+        response = await request.get(`${basePath}/v1/version/service-1/Terms%20of%20Service/${encodeURIComponent(toISODateWithoutMilliseconds(FETCH_DATE))}`);
+      });
+
+      it('responds with 200 status code', () => {
+        expect(response.status).to.equal(200);
+      });
+
+      it('responds with Content-Type application/json', () => {
+        expect(response.type).to.equal('application/json');
+      });
+
+      it('returns the expected version', () => {
+        expect(response.body).to.deep.equal(expectedResult);
+      });
+    });
+
+    context('when the requested date is anterior to the first available version', () => {
+      before(async () => {
+        response = await request.get(`${basePath}/v1/version/service-1/Terms%20of%20Service/2000-01-01T12:00:00Z`);
+      });
+
+      it('responds with 404 status code', () => {
+        expect(response.status).to.equal(404);
+      });
+
+      it('responds with Content-Type application/json', () => {
+        expect(response.type).to.equal('application/json');
+      });
+
+      it('returns an error message', () => {
+        expect(response.body.error).to.contain('No version found').and.to.contain('2000-01-01T12:00:00Z');
+      });
+    });
+
+    context('when the requested date is in the future', () => {
+      before(async () => {
+        const dateInTheFuture = new Date(Date.now() + 60000); // 1 minute in the future
+
+        response = await request.get(`${basePath}/v1/version/service-1/Terms%20of%20Service/${encodeURIComponent(toISODateWithoutMilliseconds(dateInTheFuture))}`);
+      });
+
+      it('responds with 416 status code', () => {
+        expect(response.status).to.equal(416);
+      });
+
+      it('responds with Content-Type application/json', () => {
+        expect(response.type).to.equal('application/json');
+      });
+
+      it('returns an error message', () => {
+        expect(response.body.error).to.equal('Requested version is in the future');
+      });
+    });
+  });
+});

package/src/api/server.js

@@ -1,6 +1,7 @@
 import config from 'config';
 import express from 'express';
 
+import 'express-async-errors';
 import logger from './logger.js';
 import errorsMiddleware from './middlewares/errors.js';
 import loggerMiddleware from './middlewares/logger.js';
@@ -12,12 +13,17 @@ if (process.env.NODE_ENV !== 'test') {
   app.use(loggerMiddleware);
 }
 
-const basePath = `${config.get('api.basePath')}/v1`;
+const BASE_PATH = `${config.get('api.basePath')}/v1`;
 
-app.use(basePath, apiRouter(basePath));
+app.use(BASE_PATH, apiRouter(BASE_PATH));
 app.use(errorsMiddleware);
 
-app.listen(config.get('api.port'));
-logger.info('Start Open Terms Archive API\n');
+const port = config.get('api.port');
+
+app.listen(port);
+
+if (process.env.NODE_ENV !== 'test') {
+  logger.info(`Start Open Terms Archive API on http://localhost:${port}${BASE_PATH}`);
+}
 
 export default app;

package/src/archivist/recorder/repositories/git/index.js

@@ -71,6 +71,13 @@ export default class GitRepository extends RepositoryInterface {
     return this.#toDomain(commit);
   }
 
+  async findByDate(serviceId, termsType, date, documentId) {
+    const filePath = DataMapper.generateFilePath(serviceId, termsType, documentId);
+    const commit = await this.git.getCommit([ `--until=${date?.toISOString()}`, filePath ]);
+
+    return this.#toDomain(commit);
+  }
+
   async findById(recordId) {
     const commit = await this.git.getCommit([recordId]);
 

package/src/archivist/recorder/repositories/git/index.test.js

@@ -374,6 +374,69 @@ describe('GitRepository', () => {
       });
     });
 
+    describe('#findByDate', () => {
+      context('when there are records for the given service', () => {
+        let recordToFindId;
+        let recordFound;
+
+        context('when a record exists for the requested service and date', () => {
+          const UPDATED_FILE_CONTENT = `${CONTENT} (with additional content to trigger a record)`;
+
+          before(async () => {
+            await subject.save(new Version({
+              serviceId: SERVICE_PROVIDER_ID,
+              termsType: TERMS_TYPE,
+              content: CONTENT,
+              fetchDate: FETCH_DATE_EARLIER,
+              snapshotIds: [SNAPSHOT_ID],
+            }));
+
+            ({ id: recordToFindId } = await subject.save(new Version({
+              serviceId: SERVICE_PROVIDER_ID,
+              termsType: TERMS_TYPE,
+              content: UPDATED_FILE_CONTENT,
+              fetchDate: FETCH_DATE,
+              snapshotIds: [SNAPSHOT_ID],
+            })));
+
+            await subject.save(new Version({
+              serviceId: SERVICE_PROVIDER_ID,
+              termsType: TERMS_TYPE,
+              content: `${CONTENT}CONTENT`,
+              fetchDate: FETCH_DATE_LATER,
+              snapshotIds: [SNAPSHOT_ID],
+            }));
+
+            const oneHourBeforeFetchDateLater = new Date(FETCH_DATE_LATER.getTime() - 60 * 60 * 1000);
+
+            recordFound = await subject.findByDate(SERVICE_PROVIDER_ID, TERMS_TYPE, oneHourBeforeFetchDateLater);
+          });
+
+          after(async () => subject.removeAll());
+
+          it('returns a Version object', () => {
+            expect(recordFound).to.be.an.instanceof(Version);
+          });
+
+          it('returns the latest record id', () => {
+            expect(recordFound.id).to.include(recordToFindId);
+          });
+        });
+      });
+
+      context('when there are no records for the given service', () => {
+        let recordFound;
+
+        before(async () => {
+          recordFound = await subject.findByDate(SERVICE_PROVIDER_ID, TERMS_TYPE);
+        });
+
+        it('returns null', async () => {
+          expect(recordFound).to.equal(null);
+        });
+      });
+    });
+
     describe('#findAll', () => {
       let records;
       const expectedIds = [];

package/src/archivist/recorder/repositories/interface.js

@@ -47,6 +47,20 @@ class RepositoryInterface {
     throw new Error(`#findLatest method is not implemented in ${this.constructor.name}`);
   }
 
+  /**
+  * Find the record that was valid on the given date and that matches the given service ID and terms type and optionally the document ID
+  * In case of snapshots, if the record is related to terms extracted from multiple source documents, the document ID is required to find the source snapshot
+  *
+  * @param {string} serviceId - Service ID of record to find
+  * @param {string} termsType - Terms type of record to find
+  * @param {date} date - Datetime on which the record to find was valid
+  * @param {string} [documentId] - Document ID of record to find. Used to identify the source in terms extracted from multiple source documents. Not necessary for terms with a single source document
+  * @returns {Promise<Record>} Promise that will be resolved with the found record or an empty object if none match the given criteria
+  */
+  async findByDate(serviceId, termsType, date, documentId) {
+    throw new Error(`#findByDate method is not implemented in ${this.constructor.name}`);
+  }
+
   /**
   * Find the record that matches the given record ID
   *

package/src/archivist/recorder/repositories/mongo/index.js

@@ -58,6 +58,12 @@ export default class MongoRepository extends RepositoryInterface {
     return this.#toDomain(mongoDocument);
   }
 
+  async findByDate(serviceId, termsType, date) {
+    const [mongoDocument] = await this.collection.find({ serviceId, termsType, fetchDate: { $lte: new Date(date) } }).limit(1).sort({ fetchDate: -1 }).toArray(); // `findOne` doesn't support the `sort` method, so even for only one mongo document use `find`
+
+    return this.#toDomain(mongoDocument);
+  }
+
   async findById(recordId) {
     const mongoDocument = await this.collection.findOne({ _id: new ObjectId(recordId) });
 

package/src/archivist/recorder/repositories/mongo/index.test.js

@@ -391,6 +391,57 @@ describe('MongoRepository', () => {
       });
     });
 
+    describe('#findByDate', () => {
+      context('when there are records for the given service', () => {
+        let recordToFindId;
+        let recordFound;
+
+        context('when a record exists for the requested service and date', () => {
+          const UPDATED_FILE_CONTENT = `${CONTENT} (with additional content to trigger a record)`;
+
+          before(async () => {
+            await subject.save(new Version({
+              serviceId: SERVICE_PROVIDER_ID,
+              termsType: TERMS_TYPE,
+              content: CONTENT,
+              fetchDate: FETCH_DATE_EARLIER,
+              snapshotIds: [SNAPSHOT_ID],
+            }));
+
+            ({ id: recordToFindId } = await subject.save(new Version({
+              serviceId: SERVICE_PROVIDER_ID,
+              termsType: TERMS_TYPE,
+              content: UPDATED_FILE_CONTENT,
+              fetchDate: FETCH_DATE,
+              snapshotIds: [SNAPSHOT_ID],
+            })));
+
+            await subject.save(new Version({
+              serviceId: SERVICE_PROVIDER_ID,
+              termsType: TERMS_TYPE,
+              content: `${CONTENT}CONTENT`,
+              fetchDate: FETCH_DATE_LATER,
+              snapshotIds: [SNAPSHOT_ID],
+            }));
+
+            const oneHourBeforeFetchDateLater = new Date(FETCH_DATE_LATER.getTime() - 60 * 60 * 1000);
+
+            recordFound = await subject.findByDate(SERVICE_PROVIDER_ID, TERMS_TYPE, oneHourBeforeFetchDateLater);
+          });
+
+          after(async () => subject.removeAll());
+
+          it('returns a Version object', () => {
+            expect(recordFound).to.be.an.instanceof(Version);
+          });
+
+          it('returns the latest record id', () => {
+            expect(recordFound.id).to.include(recordToFindId);
+          });
+        });
+      });
+    });
+
     describe('#findAll', () => {
       let records;
       const expectedIds = [];

package/src/archivist/utils/date.js

@@ -0,0 +1,3 @@
+export function toISODateWithoutMilliseconds(date) {
+  return new Date(date).toISOString().replace(/\.\d+/, '');
+}

package/src/archivist/utils/date.test.js

@@ -0,0 +1,28 @@
+import { expect } from 'chai';
+
+import { toISODateWithoutMilliseconds } from './date.js';
+
+describe('toISODateWithoutMilliseconds', () => {
+  const EXPECTED_RESULT = '2023-12-06T12:34:56Z';
+  const INPUTS = {
+    'valid ISO 8601 date string': '2023-12-06T12:34:56.789Z',
+    'parsable date string not in ISO 8601 format': 'Wed, 06 Dec 2023 12:34:56 GMT',
+    'a valid ISO 8601 date string without milliseconds': '2023-12-06T12:34:56Z',
+  };
+
+  Object.entries(INPUTS).forEach(([ description, input ]) => {
+    context(`with ${description}`, () => {
+      it('returns the given date in ISO 8601 format without milliseconds', () => {
+        expect(toISODateWithoutMilliseconds(input)).to.equal(EXPECTED_RESULT);
+      });
+    });
+  });
+
+  context('with an invalid date string', () => {
+    it('throws an error', () => {
+      const inputDate = 'invalidDateString';
+
+      expect(() => toISODateWithoutMilliseconds(inputDate)).to.throw(Error);
+    });
+  });
+});

package/src/reporter/index.js

@@ -1,5 +1,7 @@
 import mime from 'mime';
 
+import { toISODateWithoutMilliseconds } from '../archivist/utils/date.js';
+
 import GitHub from './github.js';
 
 const CONTRIBUTION_TOOL_URL = 'https://contribute.opentermsarchive.org/en/service';
@@ -78,7 +80,7 @@ No changes were found in the last run, so no new version has been recorded.`,
   generateDescription({ error, terms }) {
     const date = new Date();
     const currentFormattedDate = date.toLocaleDateString('en-GB', { year: 'numeric', month: 'long', day: 'numeric', hour: 'numeric', minute: 'numeric', second: 'numeric', timeZoneName: 'short', timeZone: 'UTC' });
-    const validUntil = date.toISOString().replace(/\.\d+/, ''); // ISO date without milliseconds
+    const validUntil = toISODateWithoutMilliseconds(date);
 
     const hasSnapshots = terms.sourceDocuments.every(sourceDocument => sourceDocument.snapshotId);