npm - @opentermsarchive/engine - Versions diffs - 0.15.0 - Mend

@opentermsarchive/engine 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (168) hide show

package/.env.example +3 -0
package/.eslintrc.yaml +116 -0
package/.github/workflows/deploy.yml +50 -0
package/.github/workflows/release.yml +64 -0
package/.github/workflows/test.yml +77 -0
package/CHANGELOG.md +14 -0
package/CODE_OF_CONDUCT.md +128 -0
package/CONTRIBUTING.md +143 -0
package/LICENSE +153 -0
package/MIGRATING.md +42 -0
package/README.fr.md +110 -0
package/README.md +438 -0
package/Vagrantfile +38 -0
package/ansible.cfg +13 -0
package/bin/.env.js +1 -0
package/bin/lint-declarations.js +31 -0
package/bin/track.js +26 -0
package/bin/validate-declarations.js +68 -0
package/config/ci.json +5 -0
package/config/contrib.json +35 -0
package/config/dating.json +37 -0
package/config/default.json +71 -0
package/config/france.json +40 -0
package/config/p2b-compliance.json +40 -0
package/config/pga.json +40 -0
package/config/production.json +27 -0
package/config/test.json +49 -0
package/config/vagrant.json +24 -0
package/decision-records/0001-service-name-and-id.md +73 -0
package/decision-records/0002-service-history.md +212 -0
package/decision-records/0003-snapshots-database.md +123 -0
package/ops/README.md +280 -0
package/ops/app.yml +5 -0
package/ops/infra.yml +6 -0
package/ops/inventories/dev.yml +7 -0
package/ops/inventories/production.yml +27 -0
package/ops/roles/infra/defaults/main.yml +2 -0
package/ops/roles/infra/files/.gitconfig +3 -0
package/ops/roles/infra/files/mongod.conf +18 -0
package/ops/roles/infra/files/ota-bot-key.private_key +26 -0
package/ops/roles/infra/tasks/main.yml +78 -0
package/ops/roles/infra/tasks/mongo.yml +40 -0
package/ops/roles/infra/templates/ssh_config.j2 +5 -0
package/ops/roles/ota/defaults/main.yml +14 -0
package/ops/roles/ota/files/.env +21 -0
package/ops/roles/ota/tasks/database.yml +65 -0
package/ops/roles/ota/tasks/main.yml +110 -0
package/ops/site.yml +6 -0
package/package.json +101 -0
package/pm2.config.cjs +20 -0
package/scripts/dataset/README.md +37 -0
package/scripts/dataset/assets/LICENSE +540 -0
package/scripts/dataset/assets/README.template.js +65 -0
package/scripts/dataset/export/index.js +106 -0
package/scripts/dataset/export/index.test.js +155 -0
package/scripts/dataset/export/test/fixtures/dataset/LICENSE +540 -0
package/scripts/dataset/export/test/fixtures/dataset/README.md +40 -0
package/scripts/dataset/export/test/fixtures/dataset/ServiceA/Terms of Service/2021-01-01T11-27-00Z.md +1 -0
package/scripts/dataset/export/test/fixtures/dataset/ServiceA/Terms of Service/2021-01-11T11-32-47Z.md +1 -0
package/scripts/dataset/export/test/fixtures/dataset/ServiceB/Privacy Policy/2022-01-01T12-12-24Z.md +1 -0
package/scripts/dataset/export/test/fixtures/dataset/ServiceB/Terms of Service/2022-01-06T11-32-47Z.md +1 -0
package/scripts/dataset/index.js +40 -0
package/scripts/dataset/logger/index.js +17 -0
package/scripts/dataset/main.js +25 -0
package/scripts/dataset/publish/index.js +39 -0
package/scripts/declarations/lint/index.js +36 -0
package/scripts/declarations/utils/index.js +81 -0
package/scripts/declarations/validate/definitions.js +63 -0
package/scripts/declarations/validate/index.mocha.js +262 -0
package/scripts/declarations/validate/service.history.schema.js +86 -0
package/scripts/declarations/validate/service.schema.js +91 -0
package/scripts/history/logger/index.js +39 -0
package/scripts/history/migrate-services.js +212 -0
package/scripts/history/update-to-full-hash.js +61 -0
package/scripts/history/utils/index.js +23 -0
package/scripts/import/README.md +59 -0
package/scripts/import/config/import.json +12 -0
package/scripts/import/index.js +224 -0
package/scripts/import/loadCommits.js +66 -0
package/scripts/import/logger/index.js +43 -0
package/scripts/rewrite/README.md +131 -0
package/scripts/rewrite/config/rewrite-snapshots.json +32 -0
package/scripts/rewrite/config/rewrite-versions.json +32 -0
package/scripts/rewrite/initializer/files/license +428 -0
package/scripts/rewrite/initializer/files/readme.md +8 -0
package/scripts/rewrite/initializer/index.js +44 -0
package/scripts/rewrite/rewrite-snapshots.js +108 -0
package/scripts/rewrite/rewrite-versions.js +160 -0
package/scripts/rewrite/utils.js +33 -0
package/scripts/utils/renamer/README.md +49 -0
package/scripts/utils/renamer/index.js +45 -0
package/scripts/utils/renamer/rules/documentTypes.json +25 -0
package/scripts/utils/renamer/rules/documentTypesByService.json +170 -0
package/scripts/utils/renamer/rules/serviceNames.json +92 -0
package/src/archivist/errors.js +9 -0
package/src/archivist/fetcher/errors.js +6 -0
package/src/archivist/fetcher/exports.js +18 -0
package/src/archivist/fetcher/fullDomFetcher.js +84 -0
package/src/archivist/fetcher/htmlOnlyFetcher.js +62 -0
package/src/archivist/fetcher/index.js +35 -0
package/src/archivist/fetcher/index.test.js +239 -0
package/src/archivist/filter/exports.js +3 -0
package/src/archivist/filter/index.js +178 -0
package/src/archivist/filter/index.test.js +561 -0
package/src/archivist/index.js +276 -0
package/src/archivist/index.test.js +600 -0
package/src/archivist/recorder/index.js +77 -0
package/src/archivist/recorder/index.test.js +463 -0
package/src/archivist/recorder/record.js +35 -0
package/src/archivist/recorder/record.test.js +91 -0
package/src/archivist/recorder/repositories/factory.js +23 -0
package/src/archivist/recorder/repositories/git/dataMapper.js +83 -0
package/src/archivist/recorder/repositories/git/git.js +122 -0
package/src/archivist/recorder/repositories/git/git.test.js +86 -0
package/src/archivist/recorder/repositories/git/index.js +182 -0
package/src/archivist/recorder/repositories/git/index.test.js +714 -0
package/src/archivist/recorder/repositories/interface.js +108 -0
package/src/archivist/recorder/repositories/mongo/dataMapper.js +32 -0
package/src/archivist/recorder/repositories/mongo/index.js +121 -0
package/src/archivist/recorder/repositories/mongo/index.test.js +721 -0
package/src/archivist/services/documentDeclaration.js +26 -0
package/src/archivist/services/documentDeclaration.test.js +85 -0
package/src/archivist/services/documentTypes.json +386 -0
package/src/archivist/services/index.js +255 -0
package/src/archivist/services/index.test.js +327 -0
package/src/archivist/services/pageDeclaration.js +51 -0
package/src/archivist/services/pageDeclaration.test.js +224 -0
package/src/archivist/services/service.js +60 -0
package/src/archivist/services/service.test.js +164 -0
package/src/exports.js +3 -0
package/src/index.js +59 -0
package/src/logger/README.md +1 -0
package/src/logger/index.js +131 -0
package/src/main.js +18 -0
package/src/notifier/README.md +1 -0
package/src/notifier/index.js +150 -0
package/src/tracker/README.md +1 -0
package/src/tracker/index.js +215 -0
package/test/fixtures/service_A.js +22 -0
package/test/fixtures/service_A_terms.md +10 -0
package/test/fixtures/service_A_terms_snapshot.html +14 -0
package/test/fixtures/service_B.js +22 -0
package/test/fixtures/service_with_declaration_history.js +65 -0
package/test/fixtures/service_with_filters_history.js +155 -0
package/test/fixtures/service_with_history.js +188 -0
package/test/fixtures/service_with_multipage_document.js +100 -0
package/test/fixtures/service_without_history.js +31 -0
package/test/fixtures/services.js +19 -0
package/test/fixtures/terms.pdf +0 -0
package/test/fixtures/termsFromPDF.md +25 -0
package/test/fixtures/termsModified.pdf +0 -0
package/test/services/service_A.json +9 -0
package/test/services/service_B.json +9 -0
package/test/services/service_with_declaration_history.filters.js +7 -0
package/test/services/service_with_declaration_history.history.json +17 -0
package/test/services/service_with_declaration_history.json +13 -0
package/test/services/service_with_filters_history.filters.history.js +29 -0
package/test/services/service_with_filters_history.filters.js +7 -0
package/test/services/service_with_filters_history.json +13 -0
package/test/services/service_with_history.filters.history.js +29 -0
package/test/services/service_with_history.filters.js +7 -0
package/test/services/service_with_history.history.json +26 -0
package/test/services/service_with_history.json +17 -0
package/test/services/service_with_multipage_document.filters.js +7 -0
package/test/services/service_with_multipage_document.history.json +37 -0
package/test/services/service_with_multipage_document.json +28 -0
package/test/services/service_without_history.filters.js +7 -0
package/test/services/service_without_history.json +13 -0

package/src/archivist/recorder/index.test.js ADDED Viewed

@@ -0,0 +1,463 @@
+import chai from 'chai';
+import config from 'config';
+import Recorder from './index.js';
+const { expect } = chai;
+const MIME_TYPE = 'text/html';
+const FETCH_DATE = new Date('2000-01-01T12:00:00.000Z');
+const FETCH_DATE_LATER = new Date('2000-01-02T12:00:00.000Z');
+describe('Recorder', () => {
+  const SERVICE_ID = 'test_service';
+  const TYPE = 'Terms of Service';
+  for (const repositoryType of [ 'git', 'mongo' ]) {
+    describe(repositoryType, () => {
+      let recorder;
+      before(async () => {
+        const options = config.util.cloneDeep(config.recorder);
+        options.versions.storage.type = repositoryType;
+        options.snapshots.storage.type = repositoryType;
+        recorder = new Recorder(options);
+        await recorder.initialize();
+      });
+      after(async () => recorder.finalize());
+      describe('#recordSnapshot', () => {
+        const CONTENT = '<html><h1>ToS fixture data with UTF-8 çhãràčtęrs</h1></html>';
+        let id;
+        let isFirstRecord;
+        let record;
+        context('when a required param is missing', () => {
+          after(async () => recorder.snapshotsRepository.removeAll());
+          const validParams = {
+            serviceId: SERVICE_ID,
+            documentType: TYPE,
+            content: CONTENT,
+            fetchDate: FETCH_DATE,
+            mimeType: MIME_TYPE,
+          };
+          const paramsNameToExpectedTextInError = {
+            serviceId: 'service ID',
+            documentType: 'document type',
+            fetchDate: 'fetch date',
+            content: 'content',
+            mimeType: 'mime type',
+          };
+          Object.entries(validParams).forEach(([testedRequiredParam]) => {
+            context(`when "${testedRequiredParam}" is missing`, () => {
+              it('throws an error', async () => {
+                try {
+                  const validParamsExceptTheOneTested = Object.fromEntries(Object.entries(validParams).filter(([paramName]) => paramName != testedRequiredParam));
+                  await recorder.recordSnapshot(validParamsExceptTheOneTested);
+                } catch (e) {
+                  expect(e).to.be.an('error');
+                  expect(e.message).to.contain(paramsNameToExpectedTextInError[testedRequiredParam]);
+                  return;
+                }
+                expect.fail('No error was thrown');
+              });
+            });
+          });
+        });
+        context('when it is the first record', () => {
+          before(async () => {
+            ({ id, isFirstRecord } = await recorder.recordSnapshot({
+              serviceId: SERVICE_ID,
+              documentType: TYPE,
+              content: CONTENT,
+              mimeType: MIME_TYPE,
+              fetchDate: FETCH_DATE,
+            }));
+            record = await recorder.snapshotsRepository.findLatest(SERVICE_ID, TYPE);
+          });
+          after(async () => recorder.snapshotsRepository.removeAll());
+          it('records the document with the proper content', async () => {
+            expect(await record.content).to.equal(CONTENT);
+          });
+          it('returns the record id', async () => {
+            expect(record.id).to.include(id);
+          });
+          it('returns a boolean to know if it is the first record', async () => {
+            expect(isFirstRecord).to.be.true;
+          });
+        });
+        context('when it is not the first record', () => {
+          const UPDATED_CONTENT = '<html><h1>ToS fixture data with UTF-8 çhãràčtęrs</h1><h2>Updated!</h2></html>';
+          before(async () => {
+            await recorder.recordSnapshot({
+              serviceId: SERVICE_ID,
+              documentType: TYPE,
+              content: CONTENT,
+              mimeType: MIME_TYPE,
+              fetchDate: FETCH_DATE,
+            });
+            ({ id, isFirstRecord } = await recorder.recordSnapshot({
+              serviceId: SERVICE_ID,
+              documentType: TYPE,
+              content: UPDATED_CONTENT,
+              mimeType: MIME_TYPE,
+              fetchDate: FETCH_DATE_LATER,
+            }));
+            record = await recorder.snapshotsRepository.findLatest(SERVICE_ID, TYPE);
+          });
+          after(async () => recorder.snapshotsRepository.removeAll());
+          it('records the document with the proper content', async () => {
+            expect(await record.content).to.equal(UPDATED_CONTENT);
+          });
+          it('returns the record id', async () => {
+            expect(record.id).to.include(id);
+          });
+          it('returns a boolean to know if it is the first record', async () => {
+            expect(isFirstRecord).to.be.false;
+          });
+        });
+        context('when the content has not changed', () => {
+          before(async () => {
+            await recorder.recordSnapshot({
+              serviceId: SERVICE_ID,
+              documentType: TYPE,
+              content: CONTENT,
+              mimeType: MIME_TYPE,
+              fetchDate: FETCH_DATE,
+            });
+            ({ id, isFirstRecord } = await recorder.recordSnapshot({
+              serviceId: SERVICE_ID,
+              documentType: TYPE,
+              content: CONTENT,
+              mimeType: MIME_TYPE,
+              fetchDate: FETCH_DATE_LATER,
+            }));
+            record = await recorder.snapshotsRepository.findLatest(SERVICE_ID, TYPE);
+          });
+          after(async () => recorder.snapshotsRepository.removeAll());
+          it('does not record the document', async () => {
+            expect(id).to.not.be.ok;
+          });
+        });
+      });
+      describe('#recordVersion', () => {
+        const CONTENT = '# ToS fixture data with UTF-8 çhãràčtęrs';
+        const SNAPSHOT_ID = '61af86dc5ff5caa74ae926ad';
+        let id;
+        let isFirstRecord;
+        let record;
+        context('when a required param is missing', () => {
+          after(async () => recorder.versionsRepository.removeAll());
+          const validParams = {
+            serviceId: SERVICE_ID,
+            documentType: TYPE,
+            content: CONTENT,
+            snapshotIds: [SNAPSHOT_ID],
+            fetchDate: FETCH_DATE,
+          };
+          const paramsNameToExpectedTextInError = {
+            serviceId: 'service ID',
+            documentType: 'document type',
+            snapshotIds: 'snapshot ID',
+            fetchDate: 'fetch date',
+            content: 'content',
+          };
+          Object.entries(validParams).forEach(([testedRequiredParam]) => {
+            context(`when "${testedRequiredParam}" is missing`, () => {
+              it('throws an error', async () => {
+                try {
+                  const validParamsExceptTheOneTested = Object.fromEntries(Object.entries(validParams).filter(([paramName]) => paramName != testedRequiredParam));
+                  await recorder.recordVersion(validParamsExceptTheOneTested);
+                } catch (e) {
+                  expect(e).to.be.an('error');
+                  expect(e.message).to.contain(paramsNameToExpectedTextInError[testedRequiredParam]);
+                  return;
+                }
+                expect.fail('No error was thrown');
+              });
+            });
+          });
+        });
+        context('when it is the first record', () => {
+          before(async () => {
+            ({ id, isFirstRecord } = await recorder.recordVersion({
+              serviceId: SERVICE_ID,
+              documentType: TYPE,
+              content: CONTENT,
+              snapshotIds: [SNAPSHOT_ID],
+              fetchDate: FETCH_DATE,
+            }));
+            record = await recorder.versionsRepository.findLatest(SERVICE_ID, TYPE);
+          });
+          after(async () => recorder.versionsRepository.removeAll());
+          it('records the document with the proper content', async () => {
+            expect(await record.content).to.equal(CONTENT);
+          });
+          it('returns the record id', async () => {
+            expect(record.id).to.include(id);
+          });
+          it('returns a boolean to know if it is the first record', async () => {
+            expect(isFirstRecord).to.be.true;
+          });
+        });
+        context('when it is not the first record', () => {
+          const UPDATED_CONTENT = '<html><h1>ToS fixture data with UTF-8 çhãràčtęrs</h1><h2>Updated!</h2></html>';
+          before(async () => {
+            await recorder.recordVersion({
+              serviceId: SERVICE_ID,
+              documentType: TYPE,
+              content: CONTENT,
+              snapshotIds: [SNAPSHOT_ID],
+              fetchDate: FETCH_DATE,
+            });
+            ({ id, isFirstRecord } = await recorder.recordVersion({
+              serviceId: SERVICE_ID,
+              documentType: TYPE,
+              content: UPDATED_CONTENT,
+              snapshotIds: [SNAPSHOT_ID],
+              fetchDate: FETCH_DATE_LATER,
+            }));
+            record = await recorder.versionsRepository.findLatest(SERVICE_ID, TYPE);
+          });
+          after(async () => recorder.versionsRepository.removeAll());
+          it('records the document with the proper content', async () => {
+            expect(await record.content).to.equal(UPDATED_CONTENT);
+          });
+          it('records in the document that it is not a refilter', async () => {
+            expect(record.isRefilter).to.equal(false);
+          });
+          it('returns the record id', async () => {
+            expect(record.id).to.include(id);
+          });
+          it('returns a boolean to know if it is the first record', async () => {
+            expect(isFirstRecord).to.be.false;
+          });
+        });
+        context('when the content has not changed', () => {
+          before(async () => {
+            await recorder.recordVersion({
+              serviceId: SERVICE_ID,
+              documentType: TYPE,
+              content: CONTENT,
+              snapshotIds: [SNAPSHOT_ID],
+              fetchDate: FETCH_DATE,
+            });
+            ({ id, isFirstRecord } = await recorder.recordVersion({
+              serviceId: SERVICE_ID,
+              documentType: TYPE,
+              content: CONTENT,
+              snapshotIds: [SNAPSHOT_ID],
+              fetchDate: FETCH_DATE_LATER,
+            }));
+            record = await recorder.versionsRepository.findLatest(SERVICE_ID, TYPE);
+          });
+          after(async () => recorder.versionsRepository.removeAll());
+          it('does not record the document', async () => {
+            expect(id).to.not.be.ok;
+          });
+        });
+      });
+      describe('#recordRefilter', () => {
+        const CONTENT = '# ToS fixture data with UTF-8 çhãràčtęrs';
+        const SNAPSHOT_ID = '61af86dc5ff5caa74ae926ad';
+        let id;
+        let isFirstRecord;
+        let record;
+        context('when a required param is missing', () => {
+          after(async () => recorder.versionsRepository.removeAll());
+          const validParams = {
+            serviceId: SERVICE_ID,
+            documentType: TYPE,
+            content: CONTENT,
+            snapshotIds: [SNAPSHOT_ID],
+            fetchDate: FETCH_DATE,
+          };
+          const paramsNameToExpectedTextInError = {
+            serviceId: 'service ID',
+            documentType: 'document type',
+            snapshotIds: 'snapshot ID',
+            fetchDate: 'fetch date',
+            content: 'content',
+          };
+          Object.entries(validParams).forEach(([testedRequiredParam]) => {
+            context(`when "${testedRequiredParam}" is missing`, () => {
+              it('throws an error', async () => {
+                try {
+                  const validParamsExceptTheOneTested = Object.fromEntries(Object.entries(validParams).filter(([paramName]) => paramName != testedRequiredParam));
+                  await recorder.recordRefilter(validParamsExceptTheOneTested);
+                } catch (e) {
+                  expect(e).to.be.an('error');
+                  expect(e.message).to.contain(paramsNameToExpectedTextInError[testedRequiredParam]);
+                  return;
+                }
+                expect.fail('No error was thrown');
+              });
+            });
+          });
+        });
+        context('when it is the first record', () => {
+          before(async () => {
+            ({ id, isFirstRecord } = await recorder.recordRefilter({
+              serviceId: SERVICE_ID,
+              documentType: TYPE,
+              content: CONTENT,
+              snapshotIds: [SNAPSHOT_ID],
+              fetchDate: FETCH_DATE,
+            }));
+            record = await recorder.versionsRepository.findLatest(SERVICE_ID, TYPE);
+          });
+          after(async () => recorder.versionsRepository.removeAll()); after(async () => recorder.versionsRepository.removeAll());
+          it('records the document with the proper content', async () => {
+            expect(await record.content).to.equal(CONTENT);
+          });
+          it('returns the record id', async () => {
+            expect(record.id).to.include(id);
+          });
+          it('returns a boolean to know if it is the first record', async () => {
+            expect(isFirstRecord).to.be.true;
+          });
+        });
+        context('when it is not the first record', () => {
+          const UPDATED_CONTENT = '<html><h1>ToS fixture data with UTF-8 çhãràčtęrs</h1><h2>Updated!</h2></html>';
+          before(async () => {
+            await recorder.recordRefilter({
+              serviceId: SERVICE_ID,
+              documentType: TYPE,
+              content: CONTENT,
+              snapshotIds: [SNAPSHOT_ID],
+              fetchDate: FETCH_DATE,
+            });
+            ({ id, isFirstRecord } = await recorder.recordRefilter({
+              serviceId: SERVICE_ID,
+              documentType: TYPE,
+              content: UPDATED_CONTENT,
+              snapshotIds: [SNAPSHOT_ID],
+              fetchDate: FETCH_DATE_LATER,
+            }));
+            record = await recorder.versionsRepository.findLatest(SERVICE_ID, TYPE);
+          });
+          after(async () => recorder.versionsRepository.removeAll());
+          it('records the document with the proper content', async () => {
+            expect(await record.content).to.equal(UPDATED_CONTENT);
+          });
+          it('records in the document that it is a refilter', async () => {
+            expect(record.isRefilter).to.equal(true);
+          });
+          it('returns the record id', async () => {
+            expect(record.id).to.include(id);
+          });
+          it('returns a boolean to know if it is the first record', async () => {
+            expect(isFirstRecord).to.be.false;
+          });
+        });
+        context('when the content has not changed', () => {
+          before(async () => {
+            await recorder.recordRefilter({
+              serviceId: SERVICE_ID,
+              documentType: TYPE,
+              content: CONTENT,
+              snapshotIds: [SNAPSHOT_ID],
+              fetchDate: FETCH_DATE,
+            });
+            ({ id, isFirstRecord } = await recorder.recordRefilter({
+              serviceId: SERVICE_ID,
+              documentType: TYPE,
+              content: CONTENT,
+              snapshotIds: [SNAPSHOT_ID],
+              fetchDate: FETCH_DATE_LATER,
+            }));
+            record = await recorder.versionsRepository.findLatest(SERVICE_ID, TYPE);
+          });
+          after(async () => recorder.versionsRepository.removeAll());
+          it('does not record the document', async () => {
+            expect(id).to.not.be.ok;
+          });
+        });
+      });
+    });
+  }
+});

package/src/archivist/recorder/record.js ADDED Viewed

@@ -0,0 +1,35 @@
+export default class Record {
+  #content;
+  static REQUIRED_PARAMS = Object.freeze([ 'serviceId', 'documentType', 'mimeType', 'fetchDate' ]);
+  constructor(params) {
+    Record.validate(params);
+    Object.assign(this, Object.fromEntries(Object.entries(params)));
+    if (params.content) {
+      this.#content = params.content;
+    }
+  }
+  get content() {
+    if (this.#content === undefined) {
+      throw new Error('Record content not defined, set the content or use Repository#loadRecordContent');
+    }
+    return this.#content;
+  }
+  set content(content) {
+    this.#content = content;
+  }
+  static validate(givenParams) {
+    for (const param of Record.REQUIRED_PARAMS) {
+      if (!Object.prototype.hasOwnProperty.call(givenParams, param) || givenParams[param] == null) {
+        throw new Error(`"${param}" is required`);
+      }
+    }
+  }
+}

package/src/archivist/recorder/record.test.js ADDED Viewed

@@ -0,0 +1,91 @@
+import chai from 'chai';
+import config from 'config';
+import Record from './record.js';
+import RepositoryFactory from './repositories/factory.js';
+const { expect } = chai;
+describe('Record', () => {
+  let repository;
+  let subject;
+  const REQUIRED_PARAMS = [ 'serviceId', 'documentType', 'mimeType', 'fetchDate' ];
+  const recordParams = {
+    serviceId: 'ServiceA',
+    documentType: 'Terms of Service',
+    mimeType: 'text/html',
+    fetchDate: new Date('2000-01-01T12:00:00.000Z'),
+  };
+  describe('Validation', () => {
+    describe('Required paramaters', () => {
+      REQUIRED_PARAMS.forEach(requiredParam => {
+        describe(`"${requiredParam}"`, () => {
+          context('when it is missing', () => {
+            it('throws an error', async () => {
+              try {
+                const params = {};
+                Object.keys(recordParams).filter(param => param != requiredParam).forEach(param => {
+                  params[param] = recordParams[param];
+                });
+                subject = new Record({ ...params });
+              } catch (e) {
+                expect(e).to.be.an('error');
+                expect(e.message).to.have.string(`"${requiredParam}" is required`);
+                return;
+              }
+              expect.fail('No error was thrown');
+            });
+          });
+          context('when it is null', () => {
+            it('throws an error', async () => {
+              try {
+                subject = new Record({ ...recordParams, [requiredParam]: null });
+              } catch (e) {
+                expect(e).to.be.an('error');
+                expect(e.message).to.have.string(`"${requiredParam}" is required`);
+                return;
+              }
+              expect.fail('No error was thrown');
+            });
+          });
+        });
+      });
+    });
+  });
+  describe('Content access', () => {
+    before(async () => {
+      repository = await RepositoryFactory.create(config.get('recorder.versions.storage')).initialize();
+      await repository.save(new Record({
+        ...recordParams,
+        content: 'content',
+      }));
+      ([subject] = await repository.findAll());
+    });
+    after(async () => {
+      await repository.removeAll();
+      await repository.finalize();
+    });
+    context('when it is neither defined nor loaded', () => {
+      it('throws an error explaining how to recover', async () => {
+        try {
+          console.log(subject.content);
+        } catch (e) {
+          expect(e).to.be.an('error');
+          expect(e.message).to.have.string('set the content or use Repository#loadRecordContent');
+          return;
+        }
+        expect.fail('No error was thrown');
+      });
+    });
+  });
+});

package/src/archivist/recorder/repositories/factory.js ADDED Viewed

@@ -0,0 +1,23 @@
+import path from 'path';
+import { fileURLToPath } from 'url';
+import GitRepository from './git/index.js';
+import MongoRepository from './mongo/index.js';
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+export default class RepositoryFactory {
+  static create(params) {
+    switch (params.type) {
+    case 'git':
+      return new GitRepository({
+        ...params.git,
+        path: path.resolve(__dirname, '../../../../', params.git.path),
+      });
+    case 'mongo':
+      return new MongoRepository(params.mongo);
+    default:
+      throw new Error(`Unknown storage repository configuration for type '${params.type}'`);
+    }
+  }
+}

package/src/archivist/recorder/repositories/git/dataMapper.js ADDED Viewed

@@ -0,0 +1,83 @@
+import path from 'path';
+import mime from 'mime';
+import Record from '../../record.js';
+mime.define({ 'text/markdown': ['md'] }, true); // ensure extension for markdown files is `.md` and not `.markdown`
+export const COMMIT_MESSAGE_PREFIX = {
+  startTracking: 'Start tracking',
+  refilter: 'Refilter',
+  update: 'Update',
+};
+export const DOCUMENT_TYPE_AND_PAGE_ID_SEPARATOR = ' #';
+export const SNAPSHOT_ID_MARKER = '%SNAPSHOT_ID';
+const SINGLE_SNAPSHOT_PREFIX = 'This version was recorded after filtering snapshot';
+const MULTIPLE_SNAPSHOT_PREFIX = 'This version was recorded after filtering and assembling the following snapshots from %NUMBER pages:';
+export const COMMIT_MESSAGE_PREFIXES_REGEXP = new RegExp(`^(${COMMIT_MESSAGE_PREFIX.startTracking}|${COMMIT_MESSAGE_PREFIX.refilter}|${COMMIT_MESSAGE_PREFIX.update})`);
+export function toPersistence(record, snapshotIdentiferTemplate) {
+  const { serviceId, documentType, pageId, isRefilter, snapshotIds = [], mimeType, isFirstRecord } = record;
+  let prefix = isRefilter ? COMMIT_MESSAGE_PREFIX.refilter : COMMIT_MESSAGE_PREFIX.update;
+  prefix = isFirstRecord ? COMMIT_MESSAGE_PREFIX.startTracking : prefix;
+  const subject = `${prefix} ${serviceId} ${documentType}`;
+  const pageIdMessage = `${pageId ? `Page ID ${pageId}\n\n` : ''}`;
+  let snapshotIdsMessage;
+  if (snapshotIds.length == 1) {
+    snapshotIdsMessage = `${SINGLE_SNAPSHOT_PREFIX} ${snapshotIdentiferTemplate.replace(SNAPSHOT_ID_MARKER, snapshotIds[0])}`;
+  } else if (snapshotIds.length > 1) {
+    snapshotIdsMessage = `${MULTIPLE_SNAPSHOT_PREFIX.replace('%NUMBER', snapshotIds.length)}\n${snapshotIds.map(snapshotId => `- ${snapshotIdentiferTemplate.replace(SNAPSHOT_ID_MARKER, snapshotId)}`).join('\n')}`;
+  }
+  const filePath = generateFilePath(serviceId, documentType, pageId, mimeType);
+  return {
+    message: `${subject}\n\n${pageIdMessage || ''}\n\n${snapshotIdsMessage || ''}`,
+    content: record.content,
+    filePath,
+  };
+}
+export function toDomain(commit) {
+  const { hash, date, message, body, diff } = commit;
+  const modifiedFilesInCommit = diff.files.map(({ file }) => file);
+  if (modifiedFilesInCommit.length > 1) {
+    throw new Error(`Only one document should have been recorded in ${hash}, but all these documents were recorded: ${modifiedFilesInCommit.join(', ')}`);
+  }
+  const [relativeFilePath] = modifiedFilesInCommit;
+  const snapshotIdsMatch = body.match(/\b[0-9a-f]{5,40}\b/g);
+  const [ documentType, pageId ] = path.basename(relativeFilePath, path.extname(relativeFilePath)).split(DOCUMENT_TYPE_AND_PAGE_ID_SEPARATOR);
+  return new Record({
+    id: hash,
+    serviceId: path.dirname(relativeFilePath),
+    documentType,
+    pageId,
+    mimeType: mime.getType(relativeFilePath),
+    fetchDate: new Date(date),
+    isFirstRecord: message.startsWith(COMMIT_MESSAGE_PREFIX.startTracking),
+    isRefilter: message.startsWith(COMMIT_MESSAGE_PREFIX.refilter),
+    snapshotIds: snapshotIdsMatch || [],
+  });
+}
+function generateFileName(documentType, pageId, extension) {
+  return `${documentType}${pageId ? `${DOCUMENT_TYPE_AND_PAGE_ID_SEPARATOR}${pageId}` : ''}.${extension}`;
+}
+export function generateFilePath(serviceId, documentType, pageId, mimeType) {
+  const extension = mime.getExtension(mimeType) || '*'; // If mime type is undefined, an asterisk is set as an extension. Used to match all files for the given service ID, document type and page ID when mime type is unknown.
+  return `${serviceId}/${generateFileName(documentType, pageId, extension)}`; // Do not use `path.join` as even for Windows, the path should be with `/` and not `\`. See https://github.com/ambanum/OpenTermsArchive/runs/8110230474?check_suite_focus=true#step:7:125
+}