@opentermsarchive/engine 11.0.1 → 12.0.0

package/src/archivist/recorder/repositories/mongo/index.js

@@ -88,13 +88,76 @@ export default class MongoRepository extends RepositoryInterface {
     return this.#toDomain(mongoDocument);
   }
 
-  async findAll() {
-    return Promise.all((await this.collection.find().project({ content: 0 }).sort({ fetchDate: 1 }).toArray())
+  async findAll({ limit, offset, includeTechnicalUpgrades = true } = {}) {
+    const filter = includeTechnicalUpgrades ? {} : { isTechnicalUpgrade: { $ne: true } };
+    let query = this.collection.find(filter).project({ content: 0 }).sort({ fetchDate: -1 });
+
+    if (offset !== undefined) {
+      query = query.skip(offset);
+    }
+
+    if (limit !== undefined) {
+      query = query.limit(limit);
+    }
+
+    return Promise.all((await query.toArray())
+      .map(mongoDocument => this.#toDomain(mongoDocument, { deferContentLoading: true })));
+  }
+
+  async findByServiceAndTermsType(serviceId, termsType, { limit, offset, includeTechnicalUpgrades = true } = {}) {
+    const filter = { serviceId, termsType };
+
+    if (!includeTechnicalUpgrades) {
+      filter.isTechnicalUpgrade = { $ne: true };
+    }
+
+    let query = this.collection.find(filter).project({ content: 0 }).sort({ fetchDate: -1 });
+
+    if (offset !== undefined) {
+      query = query.skip(offset);
+    }
+
+    if (limit !== undefined) {
+      query = query.limit(limit);
+    }
+
+    return Promise.all((await query.toArray())
       .map(mongoDocument => this.#toDomain(mongoDocument, { deferContentLoading: true })));
   }
 
-  count() {
-    return this.collection.countDocuments();
+  async findByService(serviceId, { limit, offset, includeTechnicalUpgrades = true } = {}) {
+    const filter = { serviceId };
+
+    if (!includeTechnicalUpgrades) {
+      filter.isTechnicalUpgrade = { $ne: true };
+    }
+
+    let query = this.collection.find(filter).project({ content: 0 }).sort({ fetchDate: -1 });
+
+    if (offset !== undefined) {
+      query = query.skip(offset);
+    }
+
+    if (limit !== undefined) {
+      query = query.limit(limit);
+    }
+
+    return Promise.all((await query.toArray())
+      .map(mongoDocument => this.#toDomain(mongoDocument, { deferContentLoading: true })));
+  }
+
+  count(serviceId, termsType) {
+    const filter = {};
+
+    if (serviceId) {
+      filter.serviceId = serviceId;
+    }
+
+    if (termsType) {
+      filter.termsType = termsType;
+    }
+
+    return this.collection.countDocuments(filter);
   }
 
   async* iterate() {

package/src/archivist/recorder/repositories/mongo/index.test.js

@@ -629,45 +629,362 @@ describe('MongoRepository', () => {
         }
       });
 
-      it('returns records in ascending order', () => {
-        expect(records.map(record => record.fetchDate)).to.deep.equal([ FETCH_DATE_EARLIER, FETCH_DATE, FETCH_DATE_LATER ]);
+      it('returns records in descending order', () => {
+        expect(records.map(record => record.fetchDate)).to.deep.equal([ FETCH_DATE_LATER, FETCH_DATE, FETCH_DATE_EARLIER ]);
+      });
+
+      context('with includeTechnicalUpgrades: false', () => {
+        let filteredRecords;
+
+        before(async () => {
+          filteredRecords = await subject.findAll({ includeTechnicalUpgrades: false });
+        });
+
+        it('excludes technical upgrade records', () => {
+          expect(filteredRecords.length).to.equal(2);
+        });
+
+        it('only returns records that represent actual content changes', () => {
+          for (const record of filteredRecords) {
+            expect(record.isTechnicalUpgrade).to.not.be.true;
+          }
+        });
+
+        it('returns the expected records in descending order', () => {
+          expect(filteredRecords.map(record => record.fetchDate)).to.deep.equal([ FETCH_DATE_LATER, FETCH_DATE ]);
+        });
       });
     });
 
-    describe('#count', () => {
-      let count;
+    describe('#findByServiceAndTermsType', () => {
+      const expectedIds = [];
+      let records;
 
       before(async () => {
-        await subject.save(new Version({
+        const { id: id1 } = await subject.save(new Version({
           serviceId: SERVICE_PROVIDER_ID,
           termsType: TERMS_TYPE,
           content: CONTENT,
           fetchDate: FETCH_DATE,
           snapshotIds: [SNAPSHOT_ID],
         }));
-        await subject.save(new Version({
+
+        expectedIds.push(id1);
+
+        const { id: id2 } = await subject.save(new Version({
           serviceId: SERVICE_PROVIDER_ID,
           termsType: TERMS_TYPE,
           content: `${CONTENT} - updated`,
           fetchDate: FETCH_DATE_LATER,
           snapshotIds: [SNAPSHOT_ID],
         }));
+
+        expectedIds.push(id2);
+
         await subject.save(new Version({
+          serviceId: 'other_service',
+          termsType: 'Privacy Policy',
+          content: `${CONTENT} - other`,
+          fetchDate: FETCH_DATE,
+          snapshotIds: [SNAPSHOT_ID],
+        }));
+
+        (records = await subject.findByServiceAndTermsType(SERVICE_PROVIDER_ID, TERMS_TYPE));
+      });
+
+      after(() => subject.removeAll());
+
+      it('returns only matching records', () => {
+        expect(records.length).to.equal(2);
+      });
+
+      it('returns Version objects', () => {
+        for (const record of records) {
+          expect(record).to.be.an.instanceof(Version);
+        }
+      });
+
+      it('returns records with matching service ID', () => {
+        for (const record of records) {
+          expect(record.serviceId).to.equal(SERVICE_PROVIDER_ID);
+        }
+      });
+
+      it('returns records with matching terms type', () => {
+        for (const record of records) {
+          expect(record.termsType).to.equal(TERMS_TYPE);
+        }
+      });
+
+      it('returns records in descending order', () => {
+        expect(records.map(record => record.fetchDate)).to.deep.equal([ FETCH_DATE_LATER, FETCH_DATE ]);
+      });
+
+      it('returns records with correct IDs', () => {
+        expect(records.map(record => record.id)).to.have.members(expectedIds);
+      });
+
+      context('when no matching records exist', () => {
+        it('returns an empty array', async () => {
+          const result = await subject.findByServiceAndTermsType('non_existent_service', 'Non Existent Terms');
+
+          expect(result).to.be.an('array').that.is.empty;
+        });
+      });
+
+      context('with includeTechnicalUpgrades: false', () => {
+        let filteredRecords;
+        let technicalUpgradeId;
+
+        before(async () => {
+          ({ id: technicalUpgradeId } = await subject.save(new Version({
+            serviceId: SERVICE_PROVIDER_ID,
+            termsType: TERMS_TYPE,
+            content: `${CONTENT} - technical upgrade`,
+            fetchDate: FETCH_DATE_EARLIER,
+            snapshotIds: [SNAPSHOT_ID],
+            isTechnicalUpgrade: true,
+          })));
+
+          filteredRecords = await subject.findByServiceAndTermsType(SERVICE_PROVIDER_ID, TERMS_TYPE, { includeTechnicalUpgrades: false });
+        });
+
+        it('excludes technical upgrade records', () => {
+          expect(filteredRecords.map(record => record.id)).to.not.include(technicalUpgradeId);
+        });
+
+        it('only returns records that represent actual content changes', () => {
+          for (const record of filteredRecords) {
+            expect(record.isTechnicalUpgrade).to.not.be.true;
+          }
+        });
+      });
+    });
+
+    describe('#findByService', () => {
+      const OTHER_TERMS_TYPE = 'Privacy Policy';
+      const expectedIds = [];
+      let records;
+
+      before(async () => {
+        const { id: id1 } = await subject.save(new Version({
           serviceId: SERVICE_PROVIDER_ID,
           termsType: TERMS_TYPE,
-          content: `${CONTENT} - updated 2`,
-          isTechnicalUpgrade: true,
+          content: CONTENT,
+          fetchDate: FETCH_DATE,
+          snapshotIds: [SNAPSHOT_ID],
+        }));
+
+        expectedIds.push(id1);
+
+        const { id: id2 } = await subject.save(new Version({
+          serviceId: SERVICE_PROVIDER_ID,
+          termsType: TERMS_TYPE,
+          content: `${CONTENT} - updated`,
+          fetchDate: FETCH_DATE_LATER,
+          snapshotIds: [SNAPSHOT_ID],
+        }));
+
+        expectedIds.push(id2);
+
+        const { id: id3 } = await subject.save(new Version({
+          serviceId: SERVICE_PROVIDER_ID,
+          termsType: OTHER_TERMS_TYPE,
+          content: `${CONTENT} - other terms type`,
           fetchDate: FETCH_DATE_EARLIER,
           snapshotIds: [SNAPSHOT_ID],
         }));
 
-        (count = await subject.count());
+        expectedIds.push(id3);
+
+        await subject.save(new Version({
+          serviceId: 'other_service',
+          termsType: TERMS_TYPE,
+          content: `${CONTENT} - other service`,
+          fetchDate: FETCH_DATE,
+          snapshotIds: [SNAPSHOT_ID],
+        }));
+
+        (records = await subject.findByService(SERVICE_PROVIDER_ID));
       });
 
       after(() => subject.removeAll());
 
-      it('returns the proper count', () => {
-        expect(count).to.equal(3);
+      it('returns only matching records', () => {
+        expect(records.length).to.equal(3);
+      });
+
+      it('returns Version objects', () => {
+        for (const record of records) {
+          expect(record).to.be.an.instanceof(Version);
+        }
+      });
+
+      it('returns records with matching service ID', () => {
+        for (const record of records) {
+          expect(record.serviceId).to.equal(SERVICE_PROVIDER_ID);
+        }
+      });
+
+      it('returns records across all terms types of the service', () => {
+        expect(new Set(records.map(record => record.termsType))).to.deep.equal(new Set([ TERMS_TYPE, OTHER_TERMS_TYPE ]));
+      });
+
+      it('returns records in descending order', () => {
+        expect(records.map(record => record.fetchDate)).to.deep.equal([ FETCH_DATE_LATER, FETCH_DATE, FETCH_DATE_EARLIER ]);
+      });
+
+      it('returns records with correct IDs', () => {
+        expect(records.map(record => record.id)).to.have.members(expectedIds);
+      });
+
+      context('when no matching records exist', () => {
+        it('returns an empty array', async () => {
+          const result = await subject.findByService('non_existent_service');
+
+          expect(result).to.be.an('array').that.is.empty;
+        });
+      });
+
+      context('with includeTechnicalUpgrades: false', () => {
+        let filteredRecords;
+        let technicalUpgradeId;
+
+        before(async () => {
+          ({ id: technicalUpgradeId } = await subject.save(new Version({
+            serviceId: SERVICE_PROVIDER_ID,
+            termsType: TERMS_TYPE,
+            content: `${CONTENT} - technical upgrade`,
+            fetchDate: new Date('2000-01-03T12:00:00.000Z'),
+            snapshotIds: [SNAPSHOT_ID],
+            isTechnicalUpgrade: true,
+          })));
+
+          filteredRecords = await subject.findByService(SERVICE_PROVIDER_ID, { includeTechnicalUpgrades: false });
+        });
+
+        it('excludes technical upgrade records', () => {
+          expect(filteredRecords.map(record => record.id)).to.not.include(technicalUpgradeId);
+        });
+
+        it('only returns records that represent actual content changes', () => {
+          for (const record of filteredRecords) {
+            expect(record.isTechnicalUpgrade).to.not.be.true;
+          }
+        });
+      });
+    });
+
+    describe('#count', () => {
+      context('without filters', () => {
+        let count;
+
+        before(async () => {
+          await subject.save(new Version({
+            serviceId: SERVICE_PROVIDER_ID,
+            termsType: TERMS_TYPE,
+            content: CONTENT,
+            fetchDate: FETCH_DATE,
+            snapshotIds: [SNAPSHOT_ID],
+          }));
+          await subject.save(new Version({
+            serviceId: SERVICE_PROVIDER_ID,
+            termsType: TERMS_TYPE,
+            content: `${CONTENT} - updated`,
+            fetchDate: FETCH_DATE_LATER,
+            snapshotIds: [SNAPSHOT_ID],
+          }));
+          await subject.save(new Version({
+            serviceId: SERVICE_PROVIDER_ID,
+            termsType: TERMS_TYPE,
+            content: `${CONTENT} - updated 2`,
+            isTechnicalUpgrade: true,
+            fetchDate: FETCH_DATE_EARLIER,
+            snapshotIds: [SNAPSHOT_ID],
+          }));
+
+          (count = await subject.count());
+        });
+
+        after(() => subject.removeAll());
+
+        it('returns the proper count', () => {
+          expect(count).to.equal(3);
+        });
+      });
+
+      context('with serviceId and termsType filters', () => {
+        before(async () => {
+          await subject.save(new Version({
+            serviceId: SERVICE_PROVIDER_ID,
+            termsType: TERMS_TYPE,
+            content: CONTENT,
+            fetchDate: FETCH_DATE,
+            snapshotIds: [SNAPSHOT_ID],
+          }));
+          await subject.save(new Version({
+            serviceId: SERVICE_PROVIDER_ID,
+            termsType: TERMS_TYPE,
+            content: `${CONTENT} - updated`,
+            fetchDate: FETCH_DATE_LATER,
+            snapshotIds: [SNAPSHOT_ID],
+          }));
+          await subject.save(new Version({
+            serviceId: 'other_service',
+            termsType: 'Privacy Policy',
+            content: 'Other content',
+            fetchDate: FETCH_DATE,
+            snapshotIds: [SNAPSHOT_ID],
+          }));
+        });
+
+        after(() => subject.removeAll());
+
+        it('returns count for specific service and terms type', async () => {
+          const filteredCount = await subject.count(SERVICE_PROVIDER_ID, TERMS_TYPE);
+
+          expect(filteredCount).to.equal(2);
+        });
+
+        it('returns zero for non-existent service', async () => {
+          const filteredCount = await subject.count('non-existent-service', TERMS_TYPE);
+
+          expect(filteredCount).to.equal(0);
+        });
+      });
+
+      context('with only serviceId filter', () => {
+        before(async () => {
+          await subject.save(new Version({
+            serviceId: SERVICE_PROVIDER_ID,
+            termsType: TERMS_TYPE,
+            content: CONTENT,
+            fetchDate: FETCH_DATE,
+            snapshotIds: [SNAPSHOT_ID],
+          }));
+          await subject.save(new Version({
+            serviceId: SERVICE_PROVIDER_ID,
+            termsType: 'Different Terms',
+            content: 'Different content',
+            fetchDate: FETCH_DATE_LATER,
+            snapshotIds: [SNAPSHOT_ID],
+          }));
+          await subject.save(new Version({
+            serviceId: 'other_service',
+            termsType: 'Privacy Policy',
+            content: 'Other content',
+            fetchDate: FETCH_DATE,
+            snapshotIds: [SNAPSHOT_ID],
+          }));
+        });
+
+        after(() => subject.removeAll());
+
+        it('returns count for all terms types of a service', async () => {
+          const filteredCount = await subject.count(SERVICE_PROVIDER_ID);
+
+          expect(filteredCount).to.equal(2);
+        });
       });
     });
 
@@ -1197,8 +1514,8 @@ describe('MongoRepository', () => {
         }
       });
 
-      it('returns records in ascending order', () => {
-        expect(records.map(record => record.fetchDate)).to.deep.equal([ FETCH_DATE_EARLIER, FETCH_DATE, FETCH_DATE_LATER ]);
+      it('returns records in descending order', () => {
+        expect(records.map(record => record.fetchDate)).to.deep.equal([ FETCH_DATE_LATER, FETCH_DATE, FETCH_DATE_EARLIER ]);
       });
     });
 

package/src/archivist/recorder/version.test.js

@@ -60,4 +60,42 @@ describe('Version', () => {
       });
     });
   });
+
+  describe('#displayTitle', () => {
+    const baseParams = {
+      serviceId: 'service-A',
+      termsType: 'Terms of Service',
+      fetchDate: new Date('2000-01-01T12:00:00.000Z'),
+      content: 'some content',
+      snapshotIds: ['dd263f270b3065e1c18201b49ab898474b357566'],
+    };
+
+    context('when the record is the first one for its service and terms type', () => {
+      it('starts with the first-record prefix', () => {
+        subject = new Version({ ...baseParams, isFirstRecord: true });
+        expect(subject.displayTitle).to.equal('First record of service-A Terms of Service');
+      });
+    });
+
+    context('when the record is a technical upgrade', () => {
+      it('starts with the technical-upgrade prefix', () => {
+        subject = new Version({ ...baseParams, isTechnicalUpgrade: true });
+        expect(subject.displayTitle).to.equal('Apply technical or declaration upgrade on service-A Terms of Service');
+      });
+    });
+
+    context('when the record is a regular content change', () => {
+      it('starts with the update prefix', () => {
+        subject = new Version(baseParams);
+        expect(subject.displayTitle).to.equal('Record new changes of service-A Terms of Service');
+      });
+    });
+
+    context('when the record is both a first record and a technical upgrade', () => {
+      it('prioritises the first-record prefix', () => {
+        subject = new Version({ ...baseParams, isFirstRecord: true, isTechnicalUpgrade: true });
+        expect(subject.displayTitle).to.equal('First record of service-A Terms of Service');
+      });
+    });
+  });
 });

package/src/archivist/services/index.js

@@ -281,7 +281,7 @@ function getHistoryFilePaths(serviceId) {
 }
 
 async function loadServiceHistory(historyFilePath) {
-  if (!(await fileExists(historyFilePath))) return {};
+  if (!(await fileExists(historyFilePath))) { return {}; }
 
   try {
     return JSON.parse(await fs.readFile(historyFilePath));

package/src/archivist/services/sourceDocument.js

@@ -1,3 +1,7 @@
+import path from 'path';
+
+import mime from 'mime';
+
 export default class SourceDocument {
   /**
    * Represents a source document containing web content and metadata for extraction.
@@ -21,7 +25,7 @@ export default class SourceDocument {
     this.filters = filters;
     this.content = content;
     this.mimeType = mimeType;
-    this.id = new URL(location).pathname.split('/').filter(Boolean).join('-');
+    this.id = SourceDocument.generateId(location);
   }
 
   get cssSelectors() {
@@ -60,6 +64,20 @@ export default class SourceDocument {
     return [selector];
   }
 
+  static generateId(location) {
+    const ILLEGAL_CHARACTERS = /[\\:"<>|*?]/g; // Characters forbidden in filenames for cross-platform compatibility; see https://github.com/actions/toolkit/blob/main/packages/artifact/src/internal/upload/path-and-artifact-name-validation.ts
+
+    const pathname = decodeURIComponent(new URL(location).pathname);
+    const extension = path.extname(pathname);
+    const pathnameWithoutExtension = mime.getType(extension) ? pathname.slice(0, -extension.length) : pathname; // Remove file extension when it corresponds to a known MIME type, as the extension is not part of the document's identity but a web server implementation detail
+
+    return pathnameWithoutExtension
+      .split('/')
+      .filter(Boolean)
+      .join('-')
+      .replace(ILLEGAL_CHARACTERS, '_');
+  }
+
   toPersistence() {
     return {
       fetch: this.location,

package/src/archivist/services/sourceDocument.test.js

@@ -157,6 +157,63 @@ describe('SourceDocument', () => {
     });
   });
 
+  describe('#generateId', () => {
+    it('generates ID from URL pathname', () => {
+      expect(new SourceDocument({ location: 'https://example.com/legal/terms' }).id).to.equal('legal-terms');
+    });
+
+    it('returns empty string for root URL', () => {
+      expect(new SourceDocument({ location: 'https://example.com/' }).id).to.equal('');
+    });
+
+    it('decodes URL-encoded characters', () => {
+      expect(new SourceDocument({ location: 'https://example.com/terms%20of%20service' }).id).to.equal('terms of service');
+    });
+
+    it('removes known file extension from URL pathname', () => {
+      expect(new SourceDocument({ location: 'https://example.com/en.html' }).id).to.equal('en');
+    });
+
+    it('removes only the last file extension', () => {
+      expect(new SourceDocument({ location: 'https://example.com/terms.backup.html' }).id).to.equal('terms.backup');
+    });
+
+    it('keeps unknown extension in URL pathname', () => {
+      expect(new SourceDocument({ location: 'https://example.com/terms.of.service' }).id).to.equal('terms.of.service');
+    });
+    it('decodes URL-encoded characters before replacing illegal ones', () => {
+      expect(new SourceDocument({ location: 'https://example.com/terms%3Aof%3Aservice' }).id).to.equal('terms_of_service');
+    });
+
+    context('replaces characters that are illegal in filenames for cross-platform compatibility', () => {
+      const ILLEGAL_CHARACTERS_IN_URL_PATHNAME = {
+        ':': 'colon',
+        '"': 'double quote',
+        '<': 'less than',
+        '>': 'greater than',
+        '|': 'vertical bar',
+        '*': 'asterisk',
+      };
+
+      for (const [ character, name ] of Object.entries(ILLEGAL_CHARACTERS_IN_URL_PATHNAME)) {
+        it(`replaces ${name} "${character}"`, () => {
+          expect(new SourceDocument({ location: `https://example.com/before${character}after` }).id).to.equal('before_after');
+        });
+      }
+
+      const ILLEGAL_CHARACTERS_ENCODED_IN_URL = {
+        '%5C': 'backslash',
+        '%3F': 'question mark',
+      };
+
+      for (const [ encoded, name ] of Object.entries(ILLEGAL_CHARACTERS_ENCODED_IN_URL)) {
+        it(`replaces ${name} decoded from "${encoded}"`, () => {
+          expect(new SourceDocument({ location: `https://example.com/before${encoded}after` }).id).to.equal('before_after');
+        });
+      }
+    });
+  });
+
   describe('#toPersistence', () => {
     it('converts basic source document declarations into JSON representation', () => {
       const result = new SourceDocument({