@opentermsarchive/engine 0.15.0

package/scripts/dataset/export/index.js

@@ -0,0 +1,106 @@
+import fsApi from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+import archiver from 'archiver';
+import config from 'config';
+
+import RepositoryFactory from '../../../src/archivist/recorder/repositories/factory.js';
+import * as renamer from '../../utils/renamer/index.js';
+import readme from '../assets/README.template.js';
+import logger from '../logger/index.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const fs = fsApi.promises;
+
+const ARCHIVE_FORMAT = 'zip'; // for supported formats, see https://www.archiverjs.com/docs/archive-formats
+
+export default async function generate({ archivePath, releaseDate }) {
+  const versionsRepository = await RepositoryFactory.create(config.get('recorder.versions.storage')).initialize();
+
+  const archive = await initializeArchive(archivePath);
+
+  await renamer.loadRules();
+
+  const services = new Set();
+  let firstVersionDate = new Date();
+  let lastVersionDate = new Date(0);
+
+  let index = 1;
+
+  for await (const version of versionsRepository.iterate()) {
+    const { content, fetchDate } = version;
+    const { serviceId, documentType } = renamer.applyRules(version.serviceId, version.documentType);
+
+    if (firstVersionDate > fetchDate) {
+      firstVersionDate = fetchDate;
+    }
+
+    if (fetchDate > lastVersionDate) {
+      lastVersionDate = fetchDate;
+    }
+
+    services.add(serviceId);
+
+    const versionPath = generateVersionPath({ serviceId, documentType, fetchDate });
+
+    logger.info({ message: versionPath, counter: index, hash: version.id });
+
+    archive.stream.append(
+      content,
+      { name: `${archive.basename}/${versionPath}` },
+    );
+    index++;
+  }
+
+  archive.stream.append(
+    readme({
+      servicesCount: services.size,
+      releaseDate,
+      firstVersionDate,
+      lastVersionDate,
+    }),
+    { name: `${archive.basename}/README.md` },
+  );
+  archive.stream.append(
+    fsApi.readFileSync(path.resolve(__dirname, '../assets/LICENSE')),
+    { name: `${archive.basename}/LICENSE` },
+  );
+
+  archive.stream.finalize();
+
+  await archive.done;
+  await versionsRepository.finalize();
+
+  return {
+    servicesCount: services.size,
+    firstVersionDate,
+    lastVersionDate,
+  };
+}
+
+async function initializeArchive(targetPath) {
+  await fs.mkdir(path.dirname(targetPath), { recursive: true });
+
+  const basename = path.basename(targetPath, path.extname(targetPath));
+
+  const output = fsApi.createWriteStream(targetPath);
+  const stream = archiver(ARCHIVE_FORMAT, { zlib: { level: 9 } }); // set compression to max level
+
+  const done = new Promise(resolve => {
+    output.on('close', resolve);
+  });
+
+  stream.pipe(output);
+
+  return { basename, stream, done };
+}
+
+function generateVersionPath({ serviceId, documentType, fetchDate }) {
+  const fsCompliantDate = fetchDate.toISOString()
+    .replace(/\.\d{3}/, '') // remove milliseconds
+    .replace(/:|\./g, '-'); // replace `:` and `.` by `-` to be compliant with the file system
+
+  return `${serviceId}/${documentType}/${fsCompliantDate}.md`;
+}

package/scripts/dataset/export/index.test.js

@@ -0,0 +1,155 @@
+import fs from 'fs/promises';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+import chai from 'chai';
+import config from 'config';
+import dircompare from 'dir-compare';
+import mime from 'mime';
+import StreamZip from 'node-stream-zip';
+
+import Record from '../../../src/archivist/recorder/record.js';
+import GitRepository from '../../../src/archivist/recorder/repositories/git/index.js';
+
+import generateArchive from './index.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const { expect } = chai;
+
+const FIRST_SERVICE_PROVIDER_ID = 'ServiceA';
+const SECOND_SERVICE_PROVIDER_ID = 'ServiceB';
+
+const FIRST_DOCUMENT_TYPE = 'Terms of Service';
+const SECOND_DOCUMENT_TYPE = 'Privacy Policy';
+
+const FIRST_FETCH_DATE = '2021-01-01T11:27:00.000Z';
+const SECOND_FETCH_DATE = '2021-01-11T11:32:47.000Z';
+const THIRD_FETCH_DATE = '2022-01-06T11:32:47.000Z';
+const FOURTH_FETCH_DATE = '2022-01-01T12:12:24.000Z';
+
+const FIRST_CONTENT = 'First Content';
+const SECOND_CONTENT = 'Second Content';
+
+const MIME_TYPE = 'text/markdown';
+
+const SNAPSHOT_ID = '721ce4a63ad399ecbdb548a66d6d327e7bc97876';
+
+const RELEASE_DATE = '2022-01-01T18:21:00.000Z';
+
+describe('Export', () => {
+  describe('#generateArchive', () => {
+    const ARCHIVE_NAME = 'test-dataset';
+    const ARCHIVE_PATH = path.resolve(__dirname, `./tmp/${ARCHIVE_NAME}.zip`);
+    const TMP_PATH = path.resolve(__dirname, './tmp');
+    const EXPECTED_DATASET_PATH = path.resolve(__dirname, './test/fixtures/dataset');
+
+    let repository;
+    let zip;
+
+    before(async function () {
+      this.timeout(10000);
+      repository = new GitRepository({
+        ...config.get('recorder.versions.storage.git'),
+        path: path.resolve(__dirname, '../../../', config.get('recorder.versions.storage.git.path')),
+      });
+
+      await repository.initialize();
+
+      await repository.save(new Record({
+        serviceId: FIRST_SERVICE_PROVIDER_ID,
+        documentType: FIRST_DOCUMENT_TYPE,
+        content: FIRST_CONTENT,
+        mimeType: MIME_TYPE,
+        fetchDate: FIRST_FETCH_DATE,
+        snapshotId: SNAPSHOT_ID,
+      }));
+
+      await repository.save(new Record({
+        serviceId: FIRST_SERVICE_PROVIDER_ID,
+        documentType: FIRST_DOCUMENT_TYPE,
+        content: SECOND_CONTENT,
+        mimeType: MIME_TYPE,
+        fetchDate: SECOND_FETCH_DATE,
+        snapshotId: SNAPSHOT_ID,
+      }));
+
+      await repository.save(new Record({
+        serviceId: SECOND_SERVICE_PROVIDER_ID,
+        documentType: FIRST_DOCUMENT_TYPE,
+        content: FIRST_CONTENT,
+        mimeType: MIME_TYPE,
+        fetchDate: THIRD_FETCH_DATE,
+        snapshotId: SNAPSHOT_ID,
+      }));
+
+      await repository.save(new Record({
+        serviceId: SECOND_SERVICE_PROVIDER_ID,
+        documentType: SECOND_DOCUMENT_TYPE,
+        content: FIRST_CONTENT,
+        mimeType: MIME_TYPE,
+        fetchDate: FOURTH_FETCH_DATE,
+        snapshotId: SNAPSHOT_ID,
+      }));
+
+      await generateArchive({
+        archivePath: ARCHIVE_PATH,
+        releaseDate: new Date(RELEASE_DATE),
+      });
+
+      zip = new StreamZip.async({ file: ARCHIVE_PATH });
+      await zip.extract('', TMP_PATH);
+      await zip.close();
+    });
+
+    after(async () => {
+      await fs.rm(TMP_PATH, { recursive: true });
+      await repository.removeAll();
+    });
+
+    it('is an archive', () => {
+      const mimeType = mime.getType(ARCHIVE_PATH);
+
+      expect(mimeType).to.equal('application/zip');
+    });
+
+    it('has the proper contents', () => {
+      expect(`${TMP_PATH}/${ARCHIVE_NAME}`).to.have.sameContentAs(EXPECTED_DATASET_PATH);
+    });
+  });
+});
+
+chai.use(chai => {
+  const { Assertion } = chai;
+
+  Assertion.addMethod('sameContentAs', function (expectedContentPath) {
+    const givenContentPath = this._obj;
+
+    const result = dircompare.compareSync(givenContentPath, expectedContentPath, {
+      excludeFilter: '.DS_Store',
+      compareContent: true,
+    });
+
+    this.assert(
+      result.same,
+      generateFailureMessage(result),
+      `expected ${givenContentPath} to have a different content as ${expectedContentPath}`,
+    );
+
+    function generateFailureMessage(result) {
+      let message = `expected ${givenContentPath} to have the same content as ${expectedContentPath}
+
+     There are ${result.differences} differences:\n`;
+
+      result.diffSet.forEach(diff => {
+        if (diff.state == 'equal') {
+          return;
+        }
+
+        message += `     ${diff.reason} on file ${diff.name1} | ${diff.name2}\n`;
+      });
+
+      return message;
+    }
+  });
+});