@opentermsarchive/engine 16.0.2 → 16.1.0

package/config/default.json

@@ -36,6 +36,19 @@
         }
       }
     },
+    "tracking-results": {
+      "storage": {
+        "type": "git",
+        "git": {
+          "path": "./data/tracking-results",
+          "publish": false,
+          "author": {
+            "name": "Open Terms Archive Bot",
+            "email": "bot@opentermsarchive.org"
+          }
+        }
+      }
+    },
     "fetcher": {
       "waitForElementsTimeout": 10000,
       "navigationTimeout": 30000,

package/config/test.json

@@ -38,6 +38,18 @@
         }
       }
     },
+    "tracking-results": {
+      "storage": {
+        "git": {
+          "path": "./test/data/tracking-results",
+          "publish": false,
+          "author": {
+            "name": "Open Terms Archive Testing Bot",
+            "email": "bot@opentermsarchive.org"
+          }
+        }
+      }
+    },
     "fetcher": {
       "waitForElementsTimeout": 1000
     },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "16.0.2",
+  "version": "16.1.0",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {

package/scripts/declarations/validate/index.mocha.js

@@ -11,6 +11,7 @@ import * as exposedFilters from '../../../src/archivist/extract/exposedFilters.j
 import extract from '../../../src/archivist/extract/index.js';
 import fetch, { launchHeadlessBrowser, stopHeadlessBrowser } from '../../../src/archivist/fetcher/index.js';
 import * as services from '../../../src/archivist/services/index.js';
+import { isPortableFileName } from '../../../src/git/pathSegment.js';
 import DeclarationUtils from '../utils/index.js';
 
 import serviceHistorySchema from './service.history.schema.js';
@@ -65,6 +66,12 @@ export default async options => {
           }
         });
 
+        it('service ID is supported as a cross-platform file name', () => {
+          if (!isPortableFileName(serviceId)) {
+            throw new Error(`Service ID "${serviceId}" contains unsupported characters: a service ID must be usable as a file name on every platform, so it cannot contain control characters nor \`/ \\ : " < > | * ?\`, nor be "." or "..". See https://docs.opentermsarchive.org/terms/how-to/track-terms/#service-id`);
+          }
+        });
+
         it('valid declaration schema', async () => {
           const declaration = JSON.parse(await fs.readFile(filePath));
 

package/scripts/import/index.js

@@ -8,7 +8,7 @@ import mime from 'mime';
 import { MongoClient } from 'mongodb';
 import nodeFetch from 'node-fetch';
 
-import Git from '../../src/archivist/recorder/repositories/git/git.js';
+import Git from '../../src/git/index.js';
 import * as renamer from '../utils/renamer/index.js';
 
 import logger from './logger/index.js';

package/scripts/import/loadCommits.js

@@ -5,7 +5,7 @@ import { fileURLToPath } from 'url';
 import config from 'config';
 import { MongoClient } from 'mongodb';
 
-import Git from '../../src/archivist/recorder/repositories/git/git.js';
+import Git from '../../src/git/index.js';
 
 import logger from './logger/index.js';
 

package/scripts/rewrite/initializer/index.js

@@ -4,7 +4,7 @@ import { fileURLToPath } from 'url';
 
 import config from 'config';
 
-import Git from '../../../src/archivist/recorder/repositories/git/git.js';
+import Git from '../../../src/git/index.js';
 import { fileExists } from '../utils.js';
 
 const fs = fsApi.promises;

package/scripts/rewrite/rewrite-snapshots.js

@@ -4,8 +4,8 @@ import { fileURLToPath } from 'url';
 import config from 'config';
 
 import Recorder from '../../src/archivist/recorder/index.js';
-import Git from '../../src/archivist/recorder/repositories/git/git.js';
 import GitRepository from '../../src/archivist/recorder/repositories/git/index.js';
+import Git from '../../src/git/index.js';
 import * as renamer from '../utils/renamer/index.js';
 
 import * as initializer from './initializer/index.js';

package/scripts/rewrite/rewrite-versions.js

@@ -6,9 +6,9 @@ import config from 'config';
 import { InaccessibleContentError } from '../../src/archivist/errors.js';
 import extract from '../../src/archivist/extract/index.js';
 import Recorder from '../../src/archivist/recorder/index.js';
-import Git from '../../src/archivist/recorder/repositories/git/git.js';
 import GitRepository from '../../src/archivist/recorder/repositories/git/index.js';
 import * as services from '../../src/archivist/services/index.js';
+import Git from '../../src/git/index.js';
 import * as renamer from '../utils/renamer/index.js';
 
 import * as initializer from './initializer/index.js';

package/src/archivist/fetcher/htmlOnlyFetcher.js

@@ -57,6 +57,14 @@ export default async function fetch(url, config) {
       throw new Error(`Network system error ${error.code} occurred when trying to fetch '${url}'`);
     }
 
+    if (error.type == 'max-redirect') { // Node-fetch reports the last redirect target, which may carry per-request tokens and would make the message differ at each attempt
+      throw new Error(`maximum redirect reached when trying to fetch '${url}'`);
+    }
+
+    if (error.type == 'invalid-redirect') { // Same as above, for the invalid redirect target
+      throw new Error(`invalid redirect URL received when trying to fetch '${url}'`);
+    }
+
     if (error instanceof AbortError) {
       throw new Error(`Timed out after ${config.navigationTimeout / 1000} seconds when trying to fetch '${url}'`);
     }

package/src/archivist/fetcher/index.test.js

@@ -31,6 +31,7 @@ describe('Fetcher', function () {
 
     before(done => {
       let blockCount = 0;
+      let redirectCount = 0;
 
       temporaryServer = http.createServer((request, response) => {
         if (request.url === '/') {
@@ -57,6 +58,9 @@ describe('Fetcher', function () {
             response.writeHead(200, { 'Content-Type': 'text/html' }).write(termsHTML);
           }
         }
+        if (request.url.startsWith('/redirect-loop')) {
+          response.writeHead(302, { Location: `/redirect-loop?token=${redirectCount++}` });
+        }
         if (request.url === '/always-block') {
           response.writeHead(403, { 'Content-Type': 'text/html' }).write('<!DOCTYPE html><html><body>Access Denied - Bot Detected</body></html>');
         }
@@ -226,6 +230,14 @@ describe('Fetcher', function () {
         });
       });
 
+      context('when server redirects endlessly to URLs that differ at each request', () => {
+        const redirectLoopUrl = `http://127.0.0.1:${SERVER_PORT}/redirect-loop`;
+
+        it('throws a FetchDocumentError error that only mentions the requested URL', async () => {
+          await expect(fetch({ url: redirectLoopUrl })).to.be.rejectedWith(FetchDocumentError, `maximum redirect reached when trying to fetch '${redirectLoopUrl}'`);
+        });
+      });
+
       context('when server is not resolved', () => {
         const notAvailableUrl = 'https://not.available.example';
 

package/src/archivist/index.js

@@ -11,6 +11,7 @@ import Snapshot from './recorder/snapshot.js';
 import Version from './recorder/version.js';
 import * as services from './services/index.js';
 import Service from './services/service.js';
+import TrackingResults, { MissingCollectionIdError, RUN_ID_TRAILER_KEY } from './tracking-results/index.js';
 
 const require = createRequire(import.meta.url);
 const { version: PACKAGE_VERSION } = require('../../package.json');
@@ -43,12 +44,13 @@ export default class Archivist extends events.EventEmitter {
     return Object.keys(this.services).sort((a, b) => a.localeCompare(b)); // Sort service IDs by lowercase name to have more intuitive logs;
   }
 
-  constructor({ recorderConfig, fetcherConfig }) {
+  constructor({ recorderConfig, fetcherConfig, trackingResultsConfig }) {
     super();
     this.fetcherConfig = fetcherConfig;
     this.recorder = new Recorder(recorderConfig);
     this.fetch = params => fetch({ ...params, config: fetcherConfig });
     this.extract = extract;
+    this.trackingResultsConfig = trackingResultsConfig; // Stored for use in initialize, where the tracking-results module is created asynchronously
   }
 
   async initialize() {
@@ -58,13 +60,52 @@ export default class Archivist extends events.EventEmitter {
     }
 
     await this.recorder.initialize();
+
+    if (this.trackingResultsConfig) {
+      try {
+        this.trackingResults = await TrackingResults.create(this.trackingResultsConfig);
+      } catch (error) {
+        if (!(error instanceof MissingCollectionIdError)) {
+          throw error;
+        }
+
+        this.emit('warn', { message: `${error.message} Tracking-results is disabled.` }); // An auxiliary audit trail must not prevent tracking itself
+      }
+
+      if (this.trackingResults) {
+        this.trackingResults.on('warn', (...args) => this.emit('warn', ...args)); // Relay the module's warnings onto the engine's public event surface
+        await this.trackingResults.initialize();
+      }
+    }
+
     this.initQueue();
+    this.declarationsCommit = await this.trackingResults?.getDeclarationsCommit(); // Captured right before loading the declarations, so that it identifies the declarations applied by every run of this process, even if their repository is updated in the meantime
     this.services = await services.load();
 
-    this.on('error', async () => {
+    this.on('error', () => this.shutdownOnFatalError());
+
+    this.emit('info', 'Initialization completed');
+
+    return this;
+  }
+
+  initQueue() {
+    this.trackingQueue = async.queue(async item => {
+      try {
+        await this.trackTermsChanges(item);
+      } catch (error) {
+        await this.handleTrackingError(error, item); // Handled inside the worker so drain() also waits for the failure handling, including the tracking-results write; async.queue's error callback is fire-and-forget and would let completeRun race the recording
+      }
+    }, MAX_PARALLEL_TRACKING);
+  }
+
+  fatalShutdownPromise = null;
+
+  shutdownOnFatalError() {
+    this.fatalShutdownPromise ||= (async () => { // Memoised so a second fatal error while the sequence is in flight awaits the same promise, instead of running a concurrent cleanup that would race the finalize pushes and process.exit
       console.log('Abort and clean up operations before exiting…');
 
-      setTimeout(() => {
+      const forceExitTimeout = setTimeout(() => {
         console.log('Cleaning timed out, force process to exit');
         process.exit(2);
       }, 60 * 1000);
@@ -72,21 +113,27 @@ export default class Archivist extends events.EventEmitter {
       this.trackingQueue.kill();
       await stopHeadlessBrowser().then(() => console.log('Headless browser stopped'));
       await this.recorder.finalize().then(() => console.log('Recorder finalized'));
-      process.exit(1);
-    });
 
-    this.emit('info', 'Initialization completed');
+      if (this.trackingResults) {
+        await this.trackingResults.finalize().then(() => console.log('Tracking-results finalized'));
+      }
 
-    return this;
-  }
+      clearTimeout(forceExitTimeout); // The guard is only needed while the cleanup above may hang; leaving it armed would fire a stray forced exit when process.exit is stubbed in tests
 
-  initQueue() {
-    this.trackingQueue = async.queue(this.trackTermsChanges.bind(this), MAX_PARALLEL_TRACKING);
-    this.trackingQueue.error(this.handleTrackingError.bind(this));
+      process.exit(1);
+    })();
+
+    return this.fatalShutdownPromise;
   }
 
-  handleTrackingError(error, { terms, isRetry }) {
+  async handleTrackingError(error, { terms, isRetry, technicalUpgradeOnly }) {
     if (!(error instanceof InaccessibleContentError)) {
+      try {
+        await this.trackingResults?.recordFailure(terms, [error]); // Recorded before emitting the fatal error: the shutdown sequence finalizes the repositories and nothing may write to them once it started
+      } catch (recordError) {
+        this.emit('warn', { message: `Could not record the tracking outcome before shutdown: ${recordError.message}`, serviceId: terms.service.id, termsType: terms.type });
+      }
+
       this.emit('error', {
         message: error.stack,
         serviceId: terms.service.id,
@@ -105,12 +152,18 @@ export default class Archivist extends events.EventEmitter {
         termsType: terms.type,
       });
 
-      this.trackingQueue.push({ terms, isRetry: true });
+      this.trackingQueue.push({ terms, isRetry: true, technicalUpgradeOnly, transientErrors: error.errors }); // Propagate the transient errors across the retry boundary so a successful retry can persist them on the tracking-result
 
       return;
     }
 
     this.emit('inaccessibleContent', error, terms);
+
+    try {
+      await this.trackingResults?.recordFailure(terms, error.errors);
+    } catch (recordError) {
+      this.emit('error', { message: recordError.stack, serviceId: terms.service.id, termsType: terms.type });
+    }
   }
 
   attach(listener) {
@@ -154,6 +207,10 @@ export default class Archivist extends events.EventEmitter {
 
     await Promise.all([ launchHeadlessBrowser(this.fetcherConfig.language), this.recorder.initialize() ]);
 
+    if (!technicalUpgradeOnly) { // Technical upgrades intentionally skip the tracking-results lifecycle: they reprocess existing snapshots and do not represent a substantive tracking attempt. As no run is started for them, the recordings below have no effect
+      await this.trackingResults?.startRun({ services: this.services, declarationsCommit: this.declarationsCommit, selectedServicesIds: servicesIds, selectedTermsTypes: termsTypes });
+    }
+
     this.trackingQueue.concurrency = concurrency;
 
     servicesIds.forEach(serviceId => {
@@ -170,12 +227,22 @@ export default class Archivist extends events.EventEmitter {
       await this.trackingQueue.drain();
     }
 
-    await Promise.all([ stopHeadlessBrowser(), this.recorder.finalize() ]);
+    if (this.trackingResults?.hasRunInProgress) {
+      try {
+        await this.trackingResults.completeRun();
+      } catch (error) {
+        this.emit('error', { message: `Failed to complete tracking-results run: ${error.stack}` }); // Fatal: the idempotent shutdown stops the browser, finalizes and pushes both recorders, then exits
+
+        return this.fatalShutdownPromise; // Prevent the finalizations below from racing the shutdown sequence, and keep the tracking pending until the process exits so that the scheduler does not start a new run in the meantime; run.json stays in_progress and the next boot closes the run additively, as after a crash
+      }
+    }
+
+    await Promise.all([ stopHeadlessBrowser(), this.recorder.finalize(), this.trackingResults?.finalize() ]);
 
     this.emit('trackingCompleted', servicesIds.length, numberOfTerms, technicalUpgradeOnly);
   }
 
-  async trackTermsChanges({ terms, technicalUpgradeOnly = false }) {
+  async trackTermsChanges({ terms, technicalUpgradeOnly = false, transientErrors }) {
     if (!technicalUpgradeOnly) {
       await this.fetchAndRecordSnapshots(terms);
     } else {
@@ -189,6 +256,12 @@ export default class Archivist extends events.EventEmitter {
     }
 
     await this.recordVersion(terms, contents.join(Version.SOURCE_DOCUMENTS_SEPARATOR), technicalUpgradeOnly);
+
+    try {
+      await this.trackingResults?.recordSuccess(terms, { transientErrors });
+    } catch (error) {
+      this.emit('error', { message: `Could not record the tracking-results outcome: ${error.stack}`, serviceId: terms.service.id, termsType: terms.type }); // Emitted here rather than thrown to the tracking error handling, which would record as failed a terms that was successfully tracked, in contradiction with its snapshots and version
+    }
   }
 
   async fetchAndRecordSnapshots(terms) {
@@ -258,6 +331,8 @@ export default class Archivist extends events.EventEmitter {
   async fetchSourceDocument(sourceDocument) {
     const { location: url, executeClientScripts, cssSelectors } = sourceDocument;
 
+    sourceDocument.resetObservations(); // A failed fetch must be recorded with the observations of this attempt, not with the previous run's values
+
     try {
       const { mimeType, content, fetcher } = await this.fetch({ url, executeClientScripts, cssSelectors });
 
@@ -274,14 +349,13 @@ export default class Archivist extends events.EventEmitter {
   }
 
   async extractContentsFromSnapshots(terms) {
-    const extractDocumentErrors = [];
-
-    const contents = await Promise.all(terms.sourceDocuments.map(async sourceDocument => {
+    // Each callback returns { content } or { error } instead of pushing into a shared array: Promise.all preserves input order while side-effect pushes from concurrent callbacks would order errors by completion time, making event.reasons non-deterministic and triggering spurious "Update failure reasons" tracking-results commits
+    const results = await Promise.all(terms.sourceDocuments.map(async sourceDocument => {
       const snapshot = await this.recorder.getLatestSnapshot(terms, sourceDocument.id);
 
       try {
         if (!snapshot) { // This can happen if one of the source documents for a terms has not yet been fetched
-          return;
+          return {};
         }
 
         sourceDocument.content = snapshot.content;
@@ -297,21 +371,29 @@ export default class Archivist extends events.EventEmitter {
 
         sourceDocument.clearContent(); // Reduce memory usage by clearing no longer needed large content strings
 
-        return content;
+        return { content };
       } catch (error) {
         if (!(error instanceof ExtractDocumentError)) {
           throw error;
         }
 
-        extractDocumentErrors.push(error);
+        return { error };
       }
     }));
 
+    const extractDocumentErrors = results.map(({ error }) => error).filter(Boolean);
+
     if (extractDocumentErrors.length) {
       throw new InaccessibleContentError(extractDocumentErrors);
     }
 
-    return contents;
+    return results.map(({ content }) => content);
+  }
+
+  get runIdMetadata() { // Ties snapshots and versions to the tracking-results run that records them; empty outside an active run (module disabled, failed run start, technical upgrades) so the trailer is simply absent
+    const runId = this.trackingResults?.currentRunId;
+
+    return runId ? { [RUN_ID_TRAILER_KEY]: runId } : {};
   }
 
   async recordVersion(terms, content, technicalUpgradeOnly) {
@@ -322,7 +404,7 @@ export default class Archivist extends events.EventEmitter {
       termsType: terms.type,
       fetchDate: terms.fetchDate,
       isTechnicalUpgrade: technicalUpgradeOnly,
-      metadata: { 'x-engine-version': PACKAGE_VERSION },
+      metadata: { 'x-engine-version': PACKAGE_VERSION, ...this.runIdMetadata },
     });
 
     await this.recorder.record(record);
@@ -350,6 +432,7 @@ export default class Archivist extends events.EventEmitter {
         'x-engine-version': PACKAGE_VERSION,
         'x-fetcher': sourceDocument.fetcher,
         'x-source-document-location': sourceDocument.location,
+        ...this.runIdMetadata,
       },
     });