npm - @opentermsarchive/engine - Versions diffs - 0.26.0 → 0.27.0 - Mend

@opentermsarchive/engine 0.26.0 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (64) hide show

package/README.md +1 -469
package/bin/ota-track.js +3 -3
package/bin/ota-validate.js +2 -2
package/bin/ota.js +1 -1
package/config/default.json +1 -1
package/config/test.json +2 -2
package/package.json +6 -7
package/scripts/dataset/export/index.js +4 -4
package/scripts/dataset/export/index.test.js +11 -17
package/scripts/dataset/export/test/fixtures/dataset/README.md +1 -1
package/scripts/declarations/lint/index.mocha.js +1 -1
package/scripts/declarations/utils/index.js +12 -12
package/scripts/declarations/validate/definitions.js +1 -1
package/scripts/declarations/validate/index.mocha.js +30 -34
package/scripts/declarations/validate/service.history.schema.js +11 -11
package/scripts/declarations/validate/service.schema.js +13 -13
package/scripts/history/migrate-services.js +4 -4
package/scripts/history/update-to-full-hash.js +2 -2
package/scripts/import/index.js +14 -14
package/scripts/rewrite/config/rewrite-snapshots.json +1 -1
package/scripts/rewrite/config/rewrite-versions.json +1 -1
package/scripts/rewrite/rewrite-snapshots.js +3 -3
package/scripts/rewrite/rewrite-versions.js +14 -14
package/scripts/utils/renamer/README.md +3 -3
package/scripts/utils/renamer/index.js +13 -13
package/src/archivist/errors.js +1 -1
package/src/archivist/extract/exports.js +3 -0
package/src/archivist/{filter → extract}/index.js +23 -27
package/src/archivist/extract/index.test.js +516 -0
package/src/archivist/index.js +101 -140
package/src/archivist/index.test.js +178 -166
package/src/archivist/recorder/index.js +11 -55
package/src/archivist/recorder/index.test.js +310 -356
package/src/archivist/recorder/record.js +18 -7
package/src/archivist/recorder/repositories/git/dataMapper.js +41 -31
package/src/archivist/recorder/repositories/git/index.js +11 -15
package/src/archivist/recorder/repositories/git/index.test.js +1058 -463
package/src/archivist/recorder/repositories/interface.js +8 -6
package/src/archivist/recorder/repositories/mongo/dataMapper.js +21 -14
package/src/archivist/recorder/repositories/mongo/index.js +8 -8
package/src/archivist/recorder/repositories/mongo/index.test.js +898 -479
package/src/archivist/recorder/snapshot.js +5 -0
package/src/archivist/recorder/snapshot.test.js +65 -0
package/src/archivist/recorder/version.js +14 -0
package/src/archivist/recorder/version.test.js +65 -0
package/src/archivist/services/index.js +60 -51
package/src/archivist/services/index.test.js +63 -83
package/src/archivist/services/service.js +26 -22
package/src/archivist/services/service.test.js +46 -68
package/src/archivist/services/{pageDeclaration.js → sourceDocument.js} +11 -9
package/src/archivist/services/{pageDeclaration.test.js → sourceDocument.test.js} +21 -21
package/src/archivist/services/terms.js +26 -0
package/src/archivist/services/{documentDeclaration.test.js → terms.test.js} +15 -15
package/src/exports.js +2 -2
package/src/index.js +16 -13
package/src/logger/index.js +35 -36
package/src/notifier/index.js +8 -8
package/src/tracker/index.js +6 -6
package/src/archivist/filter/exports.js +0 -3
package/src/archivist/filter/index.test.js +0 -564
package/src/archivist/recorder/record.test.js +0 -91
package/src/archivist/services/documentDeclaration.js +0 -26
/package/scripts/utils/renamer/rules/{documentTypes.json → termsTypes.json} +0 -0
/package/scripts/utils/renamer/rules/{documentTypesByService.json → termsTypesByService.json} +0 -0

package/src/archivist/filter/index.test.js DELETED Viewed

@@ -1,564 +0,0 @@
-import fsApi from 'fs';
-import path from 'path';
-import { fileURLToPath } from 'url';
-import chai from 'chai';
-import jsdom from 'jsdom';
-import { InaccessibleContentError } from '../errors.js';
-import PageDeclaration from '../services/pageDeclaration.js';
-import { convertRelativeURLsToAbsolute, filterHTML, filterPDF } from './index.js';
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-const fs = fsApi.promises;
-const { JSDOM } = jsdom;
-const { expect } = chai;
-const virtualLocation = 'https://exemple.com/main';
-const rawHTML = `
-<!DOCTYPE html>
-<html>
-  <head>
-    <meta charset="UTF-8">
-    <title>TOS</title>
-  </head>
-  <body>
-    <h1>Title</h1>
-    <p><a id="link1" href="/relative/link">link 1</a></p>
-    <p><a id="link2" href="#anchor">link 2</a></p>
-    <p><a id="link3" href="http://absolute.url/link">link 3</a></p>
-    <div id="empty"></div>
-    <div id="whitespaceOnly"> </div>
-  </body>
-</html>`;
-const expectedFiltered = `Title
-=====
-[link 1](https://exemple.com/relative/link)
-[link 2](#anchor)
-[link 3](http://absolute.url/link)`;
-const expectedFilteredWithAdditional = `Title
-=====`;
-const rawHTMLWithCommonChangingItems = `
-<!DOCTYPE html>
-<html>
-  <head>
-    <meta charset="UTF-8">
-    <title>TOS</title>
-    <style>body { background: red }</style>
-    <script>console.log("test")</script>
-  </head>
-  <body>
-    <style>body { background: blue }</style>
-    <script>console.log("test")</script>
-    <h1>Title</h1>
-    <p><a id="link1" href="/relative/link">link 1</a></p>
-    <p><a id="link2" href="#anchor">link 2</a></p>
-    <p><a id="link3" href="http://absolute.url/link">link 3</a></p>
-    <p><a id="link4" href="">link 4</a></p>
-    <a href="/cdn-cgi/l/email-protection#3b4c52555f484f495e5a56154b49524d5a584215484f5a4f5e565e554f7b4c52555f484f495e5a5615585456">[email&#160;protected]</a>
-    <p><a href="/cdn-cgi/l/email-protection#2d4e4243594c4e596d4e4459545e4e424259034858">conta<span>[email&#160;protected]</span></a></p>
-  </body>
-</html>`;
-/* eslint-disable no-irregular-whitespace */
-const expectedFilteredWithCommonChangingItems = `Title
-=====
-[link 1](https://exemple.com/relative/link)
-[link 2](#anchor)
-[link 3](http://absolute.url/link)
-link 4
-[\\[email protected\\]](https://exemple.com/cdn-cgi/l/email-protection)
-[\\[email protected\\]](https://exemple.com/cdn-cgi/l/email-protection)`;
-/* eslint-enable no-irregular-whitespace */
-const additionalFilter = {
-  removeLinks: function removeLinks(document) {
-    const links = document.querySelectorAll('a');
-    links.forEach(link => {
-      link.remove();
-    });
-  },
-  removeLinksAsync: async function removeLinksAsync(document) {
-    return new Promise(resolve => {
-      setTimeout(() => {
-        const links = document.querySelectorAll('a');
-        links.forEach(link => {
-          link.remove();
-        });
-        resolve();
-      }, 300);
-    });
-  },
-};
-describe('Filter', () => {
-  describe('#convertRelativeURLsToAbsolute', () => {
-    let subject;
-    before(() => {
-      const { document: webPageDOM } = new JSDOM(rawHTML).window;
-      convertRelativeURLsToAbsolute(webPageDOM, virtualLocation);
-      subject = Array.from(webPageDOM.querySelectorAll('a[href]')).map(el => el.href);
-    });
-    it('converts relative urls', async () => {
-      expect(subject).to.include('https://exemple.com/relative/link');
-    });
-    it('leaves absolute urls untouched', async () => {
-      expect(subject).to.include('http://absolute.url/link');
-    });
-  });
-  describe('#filterHTML', () => {
-    context('with string selector', () => {
-      it('filters the given HTML content with common changing items', async () => {
-        const result = await filterHTML({
-          content: rawHTMLWithCommonChangingItems,
-          pageDeclaration: new PageDeclaration({
-            location: virtualLocation,
-            contentSelectors: 'body',
-          }),
-        });
-        expect(result).to.equal(expectedFilteredWithCommonChangingItems);
-      });
-      it('filters the given HTML content', async () => {
-        const result = await filterHTML({
-          content: rawHTML,
-          pageDeclaration: new PageDeclaration({
-            location: virtualLocation,
-            contentSelectors: 'body',
-          }),
-        });
-        expect(result).to.equal(expectedFiltered);
-      });
-      context('with no match for the given selector', () => {
-        it('throws an InaccessibleContentError error', async () => {
-          await expect(filterHTML({
-            content: rawHTML,
-            pageDeclaration: new PageDeclaration({
-              location: virtualLocation,
-              contentSelectors: '#thisAnchorDoesNotExist',
-            }),
-          })).to.be.rejectedWith(InaccessibleContentError, /#thisAnchorDoesNotExist/);
-        });
-      });
-      context('with no content for the matching given selector', () => {
-        it('throws an InaccessibleContentError error', async () => {
-          await expect(filterHTML({
-            content: rawHTML,
-            pageDeclaration: new PageDeclaration({
-              location: virtualLocation,
-              contentSelectors: '#empty',
-            }),
-          })).to.be.rejectedWith(InaccessibleContentError, /empty content/);
-        });
-      });
-      context('with a whitespace only content for the corresponding given selector', () => {
-        it('throws an InaccessibleContentError error', async () => {
-          await expect(filterHTML({
-            content: rawHTML,
-            pageDeclaration: new PageDeclaration({
-              location: virtualLocation,
-              contentSelectors: '#whitespaceOnly',
-            }),
-          })).to.be.rejectedWith(InaccessibleContentError, /empty content/);
-        });
-      });
-      context('with an additional filter', () => {
-        it('filters the given HTML content also with given additional filter', async () => {
-          const result = await filterHTML({
-            content: rawHTML,
-            pageDeclaration: new PageDeclaration({
-              location: virtualLocation,
-              contentSelectors: 'body',
-              filters: [additionalFilter.removeLinks],
-            }),
-          });
-          expect(result).to.equal(expectedFilteredWithAdditional);
-        });
-      });
-      context('with an additional async filter', () => {
-        it('filters the given HTML content also with given additional filter', async () => {
-          const result = await filterHTML({
-            content: rawHTML,
-            pageDeclaration: new PageDeclaration({
-              location: virtualLocation,
-              contentSelectors: 'body',
-              filters: [additionalFilter.removeLinksAsync],
-            }),
-          });
-          expect(result).to.equal(expectedFilteredWithAdditional);
-        });
-      });
-      context('with multiple selectors in one string', () => {
-        it('filters the given HTML content', async () => {
-          const result = await filterHTML({
-            content: rawHTML,
-            pageDeclaration: new PageDeclaration({
-              location: virtualLocation,
-              contentSelectors: 'h1, #link2',
-            }),
-          });
-          expect(result).to.equal('Title\n=====\n\n[link 2](#anchor)');
-        });
-      });
-    });
-    context('with an array of selectors', () => {
-      it('filters the given HTML content', async () => {
-        const result = await filterHTML({
-          content: rawHTML,
-          pageDeclaration: new PageDeclaration({
-            location: virtualLocation,
-            contentSelectors: [ 'h1', '#link2' ],
-          }),
-        });
-        expect(result).to.equal('Title\n=====\n\n[link 2](#anchor)');
-      });
-      context('when one selector is dependent on another', () => {
-        it('filters the given HTML content', async () => {
-          const result = await filterHTML({
-            content: rawHTML,
-            pageDeclaration: new PageDeclaration({
-              location: virtualLocation,
-              contentSelectors: [ 'h1', 'h1 ~ p' ],
-            }),
-          });
-          expect(result).to.equal('Title\n=====\n\n[link 1](https://exemple.com/relative/link)\n\n[link 2](#anchor)\n\n[link 3](http://absolute.url/link)');
-        });
-      });
-    });
-    context('with range selector', () => {
-      context('with startBefore and endBefore', () => {
-        it('filters the given HTML content', async () => {
-          const result = await filterHTML({
-            content: rawHTML,
-            pageDeclaration: new PageDeclaration({
-              location: virtualLocation,
-              contentSelectors: {
-                startBefore: '#link1',
-                endBefore: '#link2',
-              },
-            }),
-          });
-          expect(result).to.equal('[link 1](https://exemple.com/relative/link)');
-        });
-      });
-      context('with startBefore and endAfter', () => {
-        it('filters the given HTML content', async () => {
-          const result = await filterHTML({
-            content: rawHTML,
-            pageDeclaration: new PageDeclaration({
-              location: virtualLocation,
-              contentSelectors: {
-                startBefore: '#link2',
-                endAfter: '#link2',
-              },
-            }),
-          });
-          expect(result).to.equal('[link 2](#anchor)');
-        });
-      });
-      context('with startAfter and endBefore', () => {
-        it('filters the given HTML content', async () => {
-          const result = await filterHTML({
-            content: rawHTML,
-            pageDeclaration: new PageDeclaration({
-              location: virtualLocation,
-              contentSelectors: {
-                startAfter: '#link1',
-                endBefore: '#link3',
-              },
-            }),
-          });
-          expect(result).to.equal('[link 2](#anchor)');
-        });
-      });
-      context('with startAfter and endAfter', () => {
-        it('filters the given HTML content', async () => {
-          const result = await filterHTML({
-            content: rawHTML,
-            pageDeclaration: new PageDeclaration({
-              location: virtualLocation,
-              contentSelectors: {
-                startAfter: '#link2',
-                endAfter: '#link3',
-              },
-            }),
-          });
-          expect(result).to.equal('[link 3](http://absolute.url/link)');
-        });
-      });
-      context('with a "start" selector that has no match', () => {
-        it('throws an InaccessibleContentError error', async () => {
-          await expect(filterHTML({
-            content: rawHTML,
-            pageDeclaration: new PageDeclaration({
-              location: virtualLocation,
-              contentSelectors: {
-                startAfter: '#paragraph1',
-                endAfter: '#link2',
-              },
-            }),
-          })).to.be.rejectedWith(InaccessibleContentError, /"start" selector has no match/);
-        });
-      });
-      context('with an "end" selector that has no match', () => {
-        it('throws an InaccessibleContentError error', async () => {
-          await expect(filterHTML({
-            content: rawHTML,
-            pageDeclaration: new PageDeclaration({
-              location: virtualLocation,
-              contentSelectors: {
-                startAfter: '#link2',
-                endAfter: '#paragraph1',
-              },
-            }),
-          })).to.be.rejectedWith(InaccessibleContentError, /"end" selector has no match/);
-        });
-      });
-    });
-    context('with an array of range selectors', () => {
-      it('filters the given HTML content', async () => {
-        const result = await filterHTML({
-          content: rawHTML,
-          pageDeclaration: new PageDeclaration({
-            location: virtualLocation,
-            contentSelectors: [
-              {
-                startAfter: '#link1',
-                endAfter: '#link2',
-              },
-              {
-                startAfter: '#link2',
-                endAfter: '#link3',
-              },
-            ],
-          }),
-        });
-        expect(result).to.equal('[link 2](#anchor)\n\n[link 3](http://absolute.url/link)');
-      });
-    });
-    context('with an array of mixed string selectors and range selectors', () => {
-      it('filters the given HTML content', async () => {
-        const result = await filterHTML({
-          content: rawHTML,
-          pageDeclaration: new PageDeclaration({
-            location: virtualLocation,
-            contentSelectors: [
-              'h1',
-              {
-                startAfter: '#link2',
-                endAfter: '#link3',
-              },
-            ],
-          }),
-        });
-        expect(result).to.equal('Title\n=====\n\n[link 3](http://absolute.url/link)');
-      });
-    });
-    describe('Remove elements', () => {
-      context('with a simple selector', () => {
-        it('removes the specified elements', async () => {
-          const result = await filterHTML({
-            content: rawHTML,
-            pageDeclaration: new PageDeclaration({
-              location: virtualLocation,
-              contentSelectors: 'body',
-              noiseSelectors: 'h1',
-            }),
-          });
-          expect(result).to.equal('[link 1](https://exemple.com/relative/link)\n\n[link 2](#anchor)\n\n[link 3](http://absolute.url/link)');
-        });
-      });
-      context('with an array of string selectors', () => {
-        it('removes the specified elements', async () => {
-          const result = await filterHTML({
-            content: rawHTML,
-            pageDeclaration: new PageDeclaration({
-              location: virtualLocation,
-              contentSelectors: 'body',
-              noiseSelectors: [ 'h1', '#link3' ],
-            }),
-          });
-          expect(result).to.equal('[link 1](https://exemple.com/relative/link)\n\n[link 2](#anchor)');
-        });
-      });
-      context('with a simple range selector', () => {
-        it('removes the specified elements', async () => {
-          const result = await filterHTML({
-            content: rawHTML,
-            pageDeclaration: new PageDeclaration({
-              location: virtualLocation,
-              contentSelectors: 'body',
-              noiseSelectors: {
-                startBefore: '#link1',
-                endAfter: '#link3',
-              },
-            }),
-          });
-          expect(result).to.equal('Title\n=====');
-        });
-        context('with a "start" selector that has no match', () => {
-          it('throws an InaccessibleContentError error', async () => {
-            await expect(filterHTML({
-              content: rawHTML,
-              pageDeclaration: new PageDeclaration({
-                location: virtualLocation,
-                contentSelectors: 'body',
-                noiseSelectors: {
-                  startAfter: '#paragraph1',
-                  endAfter: '#link2',
-                },
-              }),
-            })).to.be.rejectedWith(InaccessibleContentError, /"start" selector has no match/);
-          });
-        });
-        context('with an "end" selector that has no match', () => {
-          it('throws an InaccessibleContentError error', async () => {
-            await expect(filterHTML({
-              content: rawHTML,
-              pageDeclaration: new PageDeclaration({
-                location: virtualLocation,
-                contentSelectors: 'body',
-                noiseSelectors: {
-                  startAfter: '#link2',
-                  endAfter: '#paragraph1',
-                },
-              }),
-            })).to.be.rejectedWith(InaccessibleContentError, /"end" selector has no match/);
-          });
-        });
-      });
-      context('with an array of range selectors', () => {
-        it('removes all the selections', async () => {
-          const result = await filterHTML({
-            content: rawHTML,
-            pageDeclaration: new PageDeclaration({
-              location: virtualLocation,
-              contentSelectors: 'body',
-              noiseSelectors: [
-                {
-                  startBefore: 'h1',
-                  endBefore: '#link1',
-                },
-                {
-                  startBefore: '#link3',
-                  endAfter: '#link3',
-                },
-              ],
-            }),
-          });
-          expect(result).to.equal('[link 1](https://exemple.com/relative/link)\n\n[link 2](#anchor)');
-        });
-      });
-      context('with an array of mixed selectors and range selectors', () => {
-        it('removes all the selections', async () => {
-          const result = await filterHTML({
-            content: rawHTML,
-            pageDeclaration: new PageDeclaration({
-              location: virtualLocation,
-              contentSelectors: 'body',
-              noiseSelectors: [
-                'h1',
-                {
-                  startBefore: '#link3',
-                  endAfter: '#link3',
-                },
-              ],
-            }),
-          });
-          expect(result).to.equal('[link 1](https://exemple.com/relative/link)\n\n[link 2](#anchor)');
-        });
-        context('where one selector is dependent on another', () => {
-          it('removes all the selections', async () => {
-            const result = await filterHTML({
-              content: rawHTML,
-              pageDeclaration: new PageDeclaration({
-                location: virtualLocation,
-                contentSelectors: 'body',
-                noiseSelectors: [
-                  'h1',
-                  {
-                    startAfter: 'h1',
-                    endBefore: '#link2',
-                  },
-                ],
-              }),
-            });
-            expect(result).to.equal('[link 2](#anchor)\n\n[link 3](http://absolute.url/link)');
-          });
-        });
-      });
-    });
-  });
-  describe('#filterPDF', () => {
-    let pdfContent;
-    let expectedFilteredContent;
-    before(async () => {
-      pdfContent = await fs.readFile(path.resolve(__dirname, '../../../test/fixtures/terms.pdf'));
-      expectedFilteredContent = await fs.readFile(
-        path.resolve(__dirname, '../../../test/fixtures/termsFromPDF.md'),
-        { encoding: 'utf8' },
-      );
-    });
-    it('filters the given PDF', async () => {
-      expect(await filterPDF({ content: pdfContent })).to.equal(expectedFilteredContent);
-    });
-  });
-});

package/src/archivist/recorder/record.test.js DELETED Viewed

@@ -1,91 +0,0 @@
-import chai from 'chai';
-import config from 'config';
-import Record from './record.js';
-import RepositoryFactory from './repositories/factory.js';
-const { expect } = chai;
-describe('Record', () => {
-  let repository;
-  let subject;
-  const REQUIRED_PARAMS = [ 'serviceId', 'documentType', 'mimeType', 'fetchDate' ];
-  const recordParams = {
-    serviceId: 'ServiceA',
-    documentType: 'Terms of Service',
-    mimeType: 'text/html',
-    fetchDate: new Date('2000-01-01T12:00:00.000Z'),
-  };
-  describe('Validation', () => {
-    describe('Required paramaters', () => {
-      REQUIRED_PARAMS.forEach(requiredParam => {
-        describe(`"${requiredParam}"`, () => {
-          context('when it is missing', () => {
-            it('throws an error', async () => {
-              try {
-                const params = {};
-                Object.keys(recordParams).filter(param => param != requiredParam).forEach(param => {
-                  params[param] = recordParams[param];
-                });
-                subject = new Record({ ...params });
-              } catch (e) {
-                expect(e).to.be.an('error');
-                expect(e.message).to.have.string(`"${requiredParam}" is required`);
-                return;
-              }
-              expect.fail('No error was thrown');
-            });
-          });
-          context('when it is null', () => {
-            it('throws an error', async () => {
-              try {
-                subject = new Record({ ...recordParams, [requiredParam]: null });
-              } catch (e) {
-                expect(e).to.be.an('error');
-                expect(e.message).to.have.string(`"${requiredParam}" is required`);
-                return;
-              }
-              expect.fail('No error was thrown');
-            });
-          });
-        });
-      });
-    });
-  });
-  describe('Content access', () => {
-    before(async () => {
-      repository = await RepositoryFactory.create(config.get('recorder.versions.storage')).initialize();
-      await repository.save(new Record({
-        ...recordParams,
-        content: 'content',
-      }));
-      ([subject] = await repository.findAll());
-    });
-    after(async () => {
-      await repository.removeAll();
-      await repository.finalize();
-    });
-    context('when it is neither defined nor loaded', () => {
-      it('throws an error explaining how to recover', async () => {
-        try {
-          console.log(subject.content);
-        } catch (e) {
-          expect(e).to.be.an('error');
-          expect(e.message).to.have.string('set the content or use Repository#loadRecordContent');
-          return;
-        }
-        expect.fail('No error was thrown');
-      });
-    });
-  });
-});

package/src/archivist/services/documentDeclaration.js DELETED Viewed

@@ -1,26 +0,0 @@
-export default class DocumentDeclaration {
-  constructor({ service, type, pages, validUntil }) {
-    this.service = service;
-    this.type = type;
-    this.pages = pages;
-    if (validUntil) {
-      this.validUntil = validUntil;
-    }
-  }
-  get isMultiPage() {
-    return this.pages.length > 1;
-  }
-  toPersistence() {
-    return {
-      name: this.service.name,
-      documents: {
-        [this.type]: this.isMultiPage
-          ? { combine: this.pages.map(page => page.toPersistence()) }
-          : this.pages[0].toPersistence(),
-      },
-    };
-  }
-}

/package/scripts/utils/renamer/rules/{documentTypes.json → termsTypes.json} RENAMED Viewed

File without changes

/package/scripts/utils/renamer/rules/{documentTypesByService.json → termsTypesByService.json} RENAMED Viewed

File without changes