@opentermsarchive/engine 9.0.0 → 9.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "9.0.0",
+  "version": "9.1.0",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {

package/src/archivist/index.js

@@ -158,46 +158,30 @@ export default class Archivist extends events.EventEmitter {
 
   async trackTermsChanges({ terms, extractOnly = false }) {
     if (!extractOnly) {
-      await this.fetchSourceDocuments(terms);
-      await this.recordSnapshots(terms);
+      await this.fetchAndRecordSnapshots(terms);
     }
 
-    await this.loadSourceDocumentsFromSnapshots(terms);
+    const contents = await this.extractContentsFromSnapshots(terms);
 
-    if (terms.sourceDocuments.filter(sourceDocument => !sourceDocument.content).length) {
-      // If some source documents do not have associated snapshots, it is not possible to generate a fully valid version
+    if (contents.filter(Boolean).length !== terms.sourceDocuments.length) { // If there is not content for all source documents, it is not possible to generate a fully valid version
       return;
     }
 
-    await this.recordVersion(terms, extractOnly);
-
-    terms.sourceDocuments.forEach(sourceDocument => {
-      sourceDocument.content = null; // Reduce memory usage by clearing no longer needed large content strings
-      sourceDocument.mimeType = null; // …and associated MIME type
-      sourceDocument.snapshotId = null; // …and associated snapshot ID for consistency
-    });
+    await this.recordVersion(terms, contents.join(Version.SOURCE_DOCUMENTS_SEPARATOR), extractOnly);
   }
 
-  async fetchSourceDocuments(terms) {
+  async fetchAndRecordSnapshots(terms) {
     terms.fetchDate = new Date();
-
     const fetchDocumentErrors = [];
 
     for (const sourceDocument of terms.sourceDocuments) {
-      const { location: url, executeClientScripts, cssSelectors } = sourceDocument;
-
-      try {
-        const { mimeType, content, fetcher } = await this.fetch({ url, executeClientScripts, cssSelectors });
-
-        sourceDocument.content = content;
-        sourceDocument.mimeType = mimeType;
-        sourceDocument.fetcher = fetcher;
-      } catch (error) {
-        if (!(error instanceof FetchDocumentError)) {
-          throw error;
-        }
+      const error = await this.fetchSourceDocument(sourceDocument);
 
+      if (error) {
         fetchDocumentErrors.push(error);
+      } else {
+        await this.recordSnapshot(terms, sourceDocument);
+        sourceDocument.clearContent(); // Reduce memory usage by clearing no longer needed large content strings
       }
     }
 
@@ -206,27 +190,49 @@ export default class Archivist extends events.EventEmitter {
     }
   }
 
-  loadSourceDocumentsFromSnapshots(terms) {
-    return Promise.all(terms.sourceDocuments.map(async sourceDocument => {
-      const snapshot = await this.recorder.getLatestSnapshot(terms, sourceDocument.id);
+  async fetchSourceDocument(sourceDocument) {
+    const { location: url, executeClientScripts, cssSelectors } = sourceDocument;
 
-      if (!snapshot) { // This can happen if one of the source documents for a terms has not yet been fetched
-        return;
+    try {
+      const { mimeType, content, fetcher } = await this.fetch({ url, executeClientScripts, cssSelectors });
+
+      sourceDocument.content = content;
+      sourceDocument.mimeType = mimeType;
+      sourceDocument.fetcher = fetcher;
+    } catch (error) {
+      if (!(error instanceof FetchDocumentError)) {
+        throw error;
       }
 
-      sourceDocument.content = snapshot.content;
-      sourceDocument.mimeType = snapshot.mimeType;
-      sourceDocument.snapshotId = snapshot.id;
-      terms.fetchDate = snapshot.fetchDate;
-    }));
+      return error;
+    }
   }
 
-  async extractVersionContent(sourceDocuments) {
+  async extractContentsFromSnapshots(terms) {
     const extractDocumentErrors = [];
 
-    const result = await Promise.all(sourceDocuments.map(async sourceDocument => {
+    const contents = await Promise.all(terms.sourceDocuments.map(async sourceDocument => {
+      const snapshot = await this.recorder.getLatestSnapshot(terms, sourceDocument.id);
+
       try {
-        return await this.extract(sourceDocument);
+        if (!snapshot) { // This can happen if one of the source documents for a terms has not yet been fetched
+          return;
+        }
+
+        sourceDocument.content = snapshot.content;
+        sourceDocument.mimeType = snapshot.mimeType;
+        sourceDocument.snapshotId = snapshot.id;
+        terms.fetchDate = snapshot.fetchDate;
+
+        if (!sourceDocument.content) {
+          throw new ExtractDocumentError(`Empty content for source document ${sourceDocument.location} in snapshot ${snapshot.id}`);
+        }
+
+        const content = await this.extract(sourceDocument);
+
+        sourceDocument.clearContent(); // Reduce memory usage by clearing no longer needed large content strings
+
+        return content;
       } catch (error) {
         if (!(error instanceof ExtractDocumentError)) {
           throw error;
@@ -240,12 +246,10 @@ export default class Archivist extends events.EventEmitter {
       throw new InaccessibleContentError(extractDocumentErrors);
     }
 
-    return result.join(Version.SOURCE_DOCUMENTS_SEPARATOR);
+    return contents;
   }
 
-  async recordVersion(terms, extractOnly) {
-    const content = await this.extractVersionContent(terms.sourceDocuments);
-
+  async recordVersion(terms, content, extractOnly) {
     const record = new Version({
       content,
       snapshotIds: terms.sourceDocuments.map(sourceDocuments => sourceDocuments.snapshotId),
@@ -269,35 +273,33 @@ export default class Archivist extends events.EventEmitter {
     return record;
   }
 
-  recordSnapshots(terms) {
-    return Promise.all(terms.sourceDocuments.map(async sourceDocument => {
-      const record = new Snapshot({
-        serviceId: terms.service.id,
-        termsType: terms.type,
-        documentId: terms.hasMultipleSourceDocuments && sourceDocument.id,
-        fetchDate: terms.fetchDate,
-        content: sourceDocument.content,
-        mimeType: sourceDocument.mimeType,
-        metadata: {
-          'x-engine-version': PACKAGE_VERSION,
-          'x-fetcher': sourceDocument.fetcher,
-          'x-source-document-location': sourceDocument.location,
-        },
-      });
+  async recordSnapshot(terms, sourceDocument) {
+    const record = new Snapshot({
+      serviceId: terms.service.id,
+      termsType: terms.type,
+      documentId: terms.hasMultipleSourceDocuments && sourceDocument.id,
+      fetchDate: terms.fetchDate,
+      content: sourceDocument.content,
+      mimeType: sourceDocument.mimeType,
+      metadata: {
+        'x-engine-version': PACKAGE_VERSION,
+        'x-fetcher': sourceDocument.fetcher,
+        'x-source-document-location': sourceDocument.location,
+      },
+    });
 
-      await this.recorder.record(record);
+    await this.recorder.record(record);
 
-      if (!record.id) {
-        this.emit('snapshotNotChanged', record);
+    if (!record.id) {
+      this.emit('snapshotNotChanged', record);
 
-        return record;
-      }
+      return record;
+    }
 
-      sourceDocument.snapshotId = record.id;
+    sourceDocument.snapshotId = record.id;
 
-      this.emit(record.isFirstRecord ? 'firstSnapshotRecorded' : 'snapshotRecorded', record);
+    this.emit(record.isFirstRecord ? 'firstSnapshotRecorded' : 'snapshotRecorded', record);
 
-      return record;
-    }));
+    return record;
   }
 }

package/src/archivist/index.test.js

@@ -276,8 +276,8 @@ describe('Archivist', function () {
         service: { id: 'test-service' },
         type: 'test-type',
         sourceDocuments: [
-          { location: 'https://example.com/doc1' },
-          { location: 'https://example.com/doc2' },
+          { location: 'https://example.com/doc1', content: 'test', mimeType: 'text/html' },
+          { location: 'https://example.com/doc2', content: 'test', mimeType: 'text/html' },
         ],
       };
     });
@@ -446,7 +446,7 @@ describe('Archivist', function () {
       });
     });
 
-    describe('#recordSnapshots', () => {
+    describe('#recordSnapshot', () => {
       let terms;
       let snapshot;
 
@@ -462,7 +462,7 @@ describe('Archivist', function () {
 
       context('when it is the first record', () => {
         before(async () => {
-          [snapshot] = await app.recordSnapshots(terms);
+          snapshot = await app.recordSnapshot(terms, terms.sourceDocuments[0]);
         });
 
         after(() => {
@@ -483,12 +483,12 @@ describe('Archivist', function () {
           let changedSnapshot;
 
           before(async () => {
-            await app.recordSnapshots(terms);
+            await app.recordSnapshot(terms, terms.sourceDocuments[0]);
             resetSpiesHistory();
             terms.sourceDocuments.forEach(sourceDocument => {
               sourceDocument.content = serviceBSnapshotExpectedContent;
             });
-            [changedSnapshot] = await app.recordSnapshots(terms);
+            changedSnapshot = await app.recordSnapshot(terms, terms.sourceDocuments[0]);
           });
 
           after(() => {
@@ -508,9 +508,9 @@ describe('Archivist', function () {
           let snapshot;
 
           before(async () => {
-            await app.recordSnapshots(terms);
+            await app.recordSnapshot(terms, terms.sourceDocuments[0]);
             resetSpiesHistory();
-            [snapshot] = await app.recordSnapshots(terms);
+            snapshot = await app.recordSnapshot(terms, terms.sourceDocuments[0]);
           });
 
           after(() => {
@@ -544,7 +544,7 @@ describe('Archivist', function () {
 
       context('when it is the first record', () => {
         before(async () => {
-          version = await app.recordVersion(terms);
+          version = await app.recordVersion(terms, 'content');
         });
 
         after(() => {
@@ -565,12 +565,12 @@ describe('Archivist', function () {
           let changedVersion;
 
           before(async () => {
-            await app.recordVersion(terms);
+            await app.recordVersion(terms, 'content');
             resetSpiesHistory();
             terms.sourceDocuments.forEach(sourceDocument => {
               sourceDocument.content = serviceBSnapshotExpectedContent;
             });
-            changedVersion = await app.recordVersion(terms);
+            changedVersion = await app.recordVersion(terms, 'content updated');
           });
 
           after(() => {
@@ -590,9 +590,9 @@ describe('Archivist', function () {
           let version;
 
           before(async () => {
-            await app.recordVersion(terms);
+            await app.recordVersion(terms, 'content');
             resetSpiesHistory();
-            version = await app.recordVersion(terms);
+            version = await app.recordVersion(terms, 'content');
           });
 
           after(() => {

package/src/archivist/services/sourceDocument.js

@@ -35,6 +35,11 @@ export default class SourceDocument {
     return result.filter(selector => selector);
   }
 
+  clearContent() {
+    this.content = null;
+    this.mimeType = null;
+  }
+
   static extractCssSelectorsFromProperty(property) {
     if (Array.isArray(property)) {
       return []