@opentermsarchive/engine 16.0.2 → 16.1.1

package/src/archivist/tracking-results/index.test.js

@@ -0,0 +1,406 @@
+import { expect, use } from 'chai';
+import sinon from 'sinon';
+import sinonChai from 'sinon-chai';
+
+import Git from '../../git/index.js';
+import { ExtractDocumentError } from '../extract/index.js';
+import { FetchDocumentError } from '../fetcher/index.js';
+import { getDeclaredTerms } from '../services/index.js';
+import SourceDocument from '../services/sourceDocument.js';
+
+import { UnreadableRunError } from './errors.js';
+
+import TrackingResults from './index.js';
+
+use(sinonChai);
+
+const DECLARATIONS_COMMIT = 'c0ffee1234567890c0ffee1234567890c0ffee12';
+
+function makeRecorder(overrides = {}) {
+  return {
+    currentRun: null,
+    repository: { path: '/data/tracking-results' },
+    initialize: sinon.stub().resolves(),
+    finalize: sinon.stub().resolves(),
+    startRun: sinon.stub().resolves(),
+    recordTermsOutcome: sinon.stub().resolves(),
+    completeRun: sinon.stub().resolves(),
+    recoverCrashedRunIfAny: sinon.stub().resolves(null),
+    ...overrides,
+  };
+}
+
+function makeServices() { // Minimal stand-ins for the loaded services map, exposing the getTermsTypes(filter) and getNumberOfTerms(filter) contracts used by the run start
+  const service = termsTypes => {
+    const getTermsTypes = filter => (filter?.length ? termsTypes.filter(type => filter.includes(type)) : termsTypes);
+
+    return { getTermsTypes, getNumberOfTerms: filter => getTermsTypes(filter).length };
+  };
+
+  return {
+    'Service A': service([ 'Terms of Service', 'Privacy Policy' ]),
+    'Service B': service(['Terms of Service']),
+  };
+}
+
+function makeTerms() {
+  const sourceDocument = new SourceDocument({ location: 'https://example.com/terms', contentSelectors: 'body' });
+
+  sourceDocument.mimeType = 'text/html';
+  sourceDocument.snapshotId = 'abc123';
+
+  return { service: { id: 'Service A', name: 'Service A' }, type: 'Terms of Service', sourceDocuments: [sourceDocument] };
+}
+
+describe('TrackingResults', () => {
+  let recorder;
+  let subject;
+  let warnings;
+
+  beforeEach(() => {
+    recorder = makeRecorder();
+    subject = new TrackingResults({ recorder });
+    warnings = [];
+    subject.on('warn', ({ message }) => warnings.push(message));
+  });
+
+  describe('.create', () => {
+    context('when the configured storage type is not git', () => {
+      it('throws an error stating the supported type', async () => {
+        const error = await TrackingResults.create({ storage: { type: 'mongo', mongo: {} } }).catch(error => error);
+
+        expect(error.message).to.include('only "git" is supported');
+      });
+    });
+  });
+
+  describe('#initialize', () => {
+    it('initialises the recorder and attempts crash recovery', async () => {
+      await subject.initialize();
+
+      expect(recorder.initialize).to.have.been.calledOnce;
+      expect(recorder.recoverCrashedRunIfAny).to.have.been.calledOnce;
+    });
+
+    context('when a crashed run has been recovered', () => {
+      beforeEach(() => {
+        recorder.recoverCrashedRunIfAny.resolves({ shortRunId: 'ota-run-f47ac10b', coverage: { processed: 2, skipped: [{ serviceId: 'Service B', termsType: 'Terms of Service', reason: 'no outcome recorded before the engine crashed' }] } });
+      });
+
+      it('announces it', async () => {
+        await subject.initialize();
+
+        expect(warnings.join('\n')).to.include('Recovered crashed run ota-run-f47ac10b: persisted 2 processed terms and 1 skipped terms');
+      });
+    });
+
+    context('when the declarations commit of the crashed run is not reachable', () => {
+      let declaredTerms;
+
+      beforeEach(() => {
+        recorder.recoverCrashedRunIfAny.callsFake(async ({ getDeclaredTermsAtCommit }) => {
+          declaredTerms = await getDeclaredTermsAtCommit('deadbeef'.repeat(5)); // Well-formed but unknown commit
+
+          return null;
+        });
+      });
+
+      it('falls back to the currently declared terms and reports the approximation', async () => {
+        await subject.initialize();
+
+        expect(declaredTerms).to.have.deep.members(await getDeclaredTerms());
+        expect(warnings.join('\n')).to.include('is not reachable');
+      });
+    });
+
+    context('when the repository cannot be initialized', () => {
+      beforeEach(() => {
+        recorder.initialize.onFirstCall().rejects(new Error('Unable to create index.lock'));
+      });
+
+      it('degrades with a warning instead of failing the engine initialization', async () => {
+        await subject.initialize();
+
+        expect(recorder.recoverCrashedRunIfAny).to.not.have.been.called;
+        expect(warnings.join('\n')).to.include('Unable to create index.lock');
+      });
+
+      it('points operators to the repository to inspect if the warning persists', async () => {
+        await subject.initialize();
+
+        expect(warnings.join('\n')).to.include('inspect or delete the repository at "/data/tracking-results"');
+      });
+
+      it('retries the initialization at the next run', async () => {
+        await subject.initialize();
+        await subject.startRun({ services: makeServices(), declarationsCommit: DECLARATIONS_COMMIT, selectedServicesIds: ['Service A'], selectedTermsTypes: [] });
+
+        expect(recorder.initialize).to.have.been.calledTwice;
+        expect(recorder.startRun).to.have.been.calledOnce;
+      });
+    });
+  });
+
+  describe('#getDeclarationsCommit', () => {
+    afterEach(() => Git.getHeadSha.restore());
+
+    it('resolves to the commit of the declarations repository', async () => {
+      sinon.stub(Git, 'getHeadSha').resolves(DECLARATIONS_COMMIT);
+
+      expect(await subject.getDeclarationsCommit()).to.equal(DECLARATIONS_COMMIT);
+      expect(warnings).to.be.empty;
+    });
+
+    context('when the declarations are not versioned', () => {
+      it('resolves to null and reports it', async () => {
+        sinon.stub(Git, 'getHeadSha').resolves(null);
+
+        expect(await subject.getDeclarationsCommit()).to.be.null;
+        expect(warnings.join('\n')).to.include('is not a Git repository');
+      });
+    });
+
+    context('when the commit cannot be read', () => {
+      it('resolves to null and reports it', async () => {
+        sinon.stub(Git, 'getHeadSha').rejects(new Error('git failure'));
+
+        expect(await subject.getDeclarationsCommit()).to.be.null;
+        expect(warnings.join('\n')).to.include('Could not read the declarations commit: git failure');
+      });
+    });
+  });
+
+  describe('#startRun', () => {
+    it('starts a recorder run with the full declared counts and the unselected terms as skips', async () => {
+      const services = makeServices();
+
+      await subject.startRun({ services, declarationsCommit: DECLARATIONS_COMMIT, selectedServicesIds: ['Service A'], selectedTermsTypes: [] });
+
+      expect(recorder.startRun).to.have.been.calledOnce;
+
+      const { declarationsCommit, servicesCount, termsCount, skippedTerms } = recorder.startRun.firstCall.args[0];
+
+      expect(declarationsCommit).to.equal(DECLARATIONS_COMMIT);
+      expect(servicesCount).to.equal(2);
+      expect(termsCount).to.equal(3);
+      expect(skippedTerms).to.deep.equal([{ serviceId: 'Service B', termsType: 'Terms of Service', reason: 'not selected for this run' }]);
+    });
+
+    it('narrows the skips to the selected terms types', async () => {
+      const services = makeServices();
+
+      await subject.startRun({ services, declarationsCommit: DECLARATIONS_COMMIT, selectedServicesIds: [ 'Service A', 'Service B' ], selectedTermsTypes: ['Privacy Policy'] });
+
+      const { skippedTerms } = recorder.startRun.firstCall.args[0];
+
+      expect(skippedTerms).to.deep.equal([
+        { serviceId: 'Service A', termsType: 'Terms of Service', reason: 'not selected for this run' },
+        { serviceId: 'Service B', termsType: 'Terms of Service', reason: 'not selected for this run' },
+      ]);
+    });
+
+    context('when the declarations commit is unknown', () => {
+      it('does not start a recorder run', async () => {
+        await subject.startRun({ services: makeServices(), declarationsCommit: null, selectedServicesIds: ['Service A'], selectedTermsTypes: [] });
+
+        expect(recorder.startRun).to.not.have.been.called;
+      });
+    });
+
+    context('when the recorder cannot start the run', () => {
+      beforeEach(() => {
+        recorder.startRun.rejects(new Error('Run start commit failed'));
+      });
+
+      it('degrades with a warning instead of failing the tracking run', async () => {
+        await subject.startRun({ services: makeServices(), declarationsCommit: DECLARATIONS_COMMIT, selectedServicesIds: ['Service A'], selectedTermsTypes: [] });
+
+        expect(subject.hasRunInProgress).to.be.false;
+        expect(warnings.join('\n')).to.include('tracking-results is disabled for this run');
+      });
+    });
+
+    context('when crash recovery keeps failing', () => {
+      beforeEach(() => {
+        recorder.recoverCrashedRunIfAny.rejects(new Error('transient git failure'));
+      });
+
+      it('does not start a recorder run', async () => { // A new Start run commit would hide the crashed run's reference SHA and make its recovery impossible forever
+        await subject.startRun({ services: makeServices(), declarationsCommit: DECLARATIONS_COMMIT, selectedServicesIds: ['Service A'], selectedTermsTypes: [] });
+
+        expect(recorder.startRun).to.not.have.been.called;
+        expect(warnings.join('\n')).to.include('will be retried at the next one');
+      });
+
+      it('retries the recovery at the next run', async () => {
+        await subject.startRun({ services: makeServices(), declarationsCommit: DECLARATIONS_COMMIT, selectedServicesIds: ['Service A'], selectedTermsTypes: [] });
+        await subject.startRun({ services: makeServices(), declarationsCommit: DECLARATIONS_COMMIT, selectedServicesIds: ['Service A'], selectedTermsTypes: [] });
+
+        expect(recorder.recoverCrashedRunIfAny).to.have.been.calledTwice;
+      });
+    });
+
+    context('when the previous run.json is unreadable', () => {
+      beforeEach(() => {
+        recorder.recoverCrashedRunIfAny.rejects(new UnreadableRunError('Could not parse JSON'));
+      });
+
+      it('proceeds and announces the file will be overwritten', async () => {
+        await subject.startRun({ services: makeServices(), declarationsCommit: DECLARATIONS_COMMIT, selectedServicesIds: ['Service A'], selectedTermsTypes: [] });
+
+        expect(recorder.startRun).to.have.been.calledOnce;
+        expect(warnings.join('\n')).to.include('it will be overwritten by the next run');
+      });
+    });
+
+    context('when recovery already succeeded', () => {
+      it('does not attempt it again', async () => {
+        await subject.initialize();
+        await subject.startRun({ services: makeServices(), declarationsCommit: DECLARATIONS_COMMIT, selectedServicesIds: ['Service A'], selectedTermsTypes: [] });
+
+        expect(recorder.recoverCrashedRunIfAny).to.have.been.calledOnce;
+      });
+    });
+  });
+
+  describe('#recordSuccess', () => {
+    context('when a run is in progress', () => {
+      beforeEach(() => {
+        recorder.currentRun = {};
+      });
+
+      it('maps the terms to the recorder payload', async () => {
+        await subject.recordSuccess(makeTerms());
+
+        expect(recorder.recordTermsOutcome).to.have.been.calledOnce;
+        expect(recorder.recordTermsOutcome.firstCall.args[0]).to.deep.equal({
+          serviceId: 'Service A',
+          termsType: 'Terms of Service',
+          serviceName: 'Service A',
+          sourceDocuments: [{
+            id: 'terms',
+            fetch: 'https://example.com/terms',
+            select: 'body',
+            remove: undefined,
+            filter: undefined,
+            executeClientScripts: undefined,
+            mimeType: 'text/html',
+            snapshotId: 'abc123',
+          }],
+          status: 'ok',
+          reasons: undefined,
+          transientErrorReasons: undefined,
+        });
+      });
+
+      it('normalises the MIME type reported by the fetcher', async () => {
+        const terms = makeTerms();
+
+        terms.sourceDocuments[0].mimeType = 'text/html; charset=utf-8';
+        await subject.recordSuccess(terms);
+
+        expect(recorder.recordTermsOutcome.firstCall.args[0].sourceDocuments[0].mimeType).to.equal('text/html');
+      });
+
+      it('records a missing MIME type as null', async () => {
+        const terms = makeTerms();
+
+        terms.sourceDocuments[0].mimeType = undefined;
+        await subject.recordSuccess(terms);
+
+        expect(recorder.recordTermsOutcome.firstCall.args[0].sourceDocuments[0].mimeType).to.be.null;
+      });
+
+      it('categorises the transient errors', async () => {
+        await subject.recordSuccess(makeTerms(), { transientErrors: [new FetchDocumentError('HTTP code 503')] });
+
+        expect(recorder.recordTermsOutcome.firstCall.args[0].transientErrorReasons).to.deep.equal(['[fetch] Fetch failed: HTTP code 503']);
+      });
+    });
+
+    context('when no run is in progress', () => {
+      it('records nothing', async () => {
+        await subject.recordSuccess(makeTerms());
+
+        expect(recorder.recordTermsOutcome).to.not.have.been.called;
+      });
+    });
+  });
+
+  describe('#recordFailure', () => {
+    beforeEach(() => {
+      recorder.currentRun = {};
+    });
+
+    it('categorises each error by origin', async () => {
+      await subject.recordFailure(makeTerms(), [
+        new FetchDocumentError('HTTP code 500'),
+        new ExtractDocumentError('CSS selector has no match'),
+        new Error('Could not commit /home/ota/data/snapshots/Service/Terms.html'),
+      ]);
+
+      expect(recorder.recordTermsOutcome.firstCall.args[0].status).to.equal('failed');
+      expect(recorder.recordTermsOutcome.firstCall.args[0].reasons).to.deep.equal([
+        '[fetch] Fetch failed: HTTP code 500',
+        '[extraction] Extract failed: CSS selector has no match',
+        '[internal] Unexpected engine error',
+      ]);
+    });
+  });
+
+  describe('#completeRun and #finalize', () => {
+    it('delegate to the recorder', async () => {
+      await subject.initialize();
+      await subject.completeRun();
+      await subject.finalize();
+
+      expect(recorder.completeRun).to.have.been.calledOnce;
+      expect(recorder.finalize).to.have.been.calledOnce;
+    });
+
+    context('when the repository could not be initialized', () => {
+      beforeEach(() => {
+        recorder.initialize.rejects(new Error('Unable to create index.lock'));
+      });
+
+      it('does not finalize it', async () => {
+        await subject.initialize();
+        await subject.finalize();
+
+        expect(recorder.finalize).to.not.have.been.called;
+      });
+    });
+
+    context('when the push fails at finalize', () => {
+      beforeEach(() => {
+        recorder.finalize.rejects(new Error('remote rejected'));
+      });
+
+      it('degrades with a warning, as the commits remain local', async () => {
+        await subject.initialize();
+        await subject.finalize();
+
+        expect(warnings.join('\n')).to.include('recorded commits are kept locally');
+      });
+    });
+  });
+
+  describe('#hasRunInProgress', () => {
+    it('reflects the recorder state', () => {
+      expect(subject.hasRunInProgress).to.be.false;
+      recorder.currentRun = {};
+      expect(subject.hasRunInProgress).to.be.true;
+    });
+  });
+
+  describe('#currentRunId', () => {
+    it('returns null when no run is in progress', () => {
+      expect(subject.currentRunId).to.be.null;
+    });
+
+    it('returns the id of the run in progress', () => {
+      recorder.currentRun = { runId: 'ota-run-f47ac10b-58cc-4372-a567-0e02b2c3d479' };
+      expect(subject.currentRunId).to.equal('ota-run-f47ac10b-58cc-4372-a567-0e02b2c3d479');
+    });
+  });
+});

package/src/archivist/tracking-results/recorder.js

@@ -0,0 +1,156 @@
+import { UnreadableRunError } from './errors.js';
+import Run, { RUN_STATUSES } from './run/index.js';
+import { TRANSITIONS_BY_EVENT_TYPE, termsKey } from './terms-result/dataMapper.js';
+import TermsResult, { STATUSES } from './terms-result/index.js';
+
+export const RUN_ID_TRAILER_KEY = 'x-run-id'; // Git trailer tying every commit of a run to its runId, so run membership stays greppable without relying on commit ranges between two run.json commits; parseTrailers lowercases keys, hence the casing
+
+export default class TrackingResultsRecorder {
+  constructor({ repository, collectionId, schedule, engineVersion }) {
+    this.repository = repository;
+    this.collectionId = collectionId;
+    this.schedule = schedule;
+    this.engineVersion = engineVersion;
+    this.currentRun = null;
+  }
+
+  async initialize() {
+    await this.repository.initialize();
+  }
+
+  finalize() {
+    return this.repository.finalize();
+  }
+
+  async startRun({ declarationsCommit, servicesCount, termsCount, skippedTerms = [] }) {
+    const run = new Run({
+      runId: Run.generateId(),
+      collectionId: this.collectionId,
+      schedule: this.schedule,
+      startDate: new Date().toISOString(),
+      engineVersion: this.engineVersion,
+      declarationsCommit,
+      servicesCount,
+      termsCount,
+    });
+
+    skippedTerms.forEach(skipped => run.addSkipped(skipped)); // Skips known at run start (e.g. terms not selected by a partial run) are persisted in the run-start commit: a crash during the run must not let recovery attribute them to the crash
+
+    await this.repository.saveRun(run, { trailers: runTrailers(run) });
+
+    this.currentRun = run; // Assigned only once the run-start commit landed: a failed start must not leave a half-open run that later recordings would attach to
+
+    return run;
+  }
+
+  async recordTermsOutcome({ serviceId, termsType, serviceName, sourceDocuments, status, reasons, transientErrorReasons }) {
+    const run = this.activeRun;
+
+    const event = {
+      date: new Date().toISOString(),
+      serviceName,
+      sourceDocuments,
+    };
+
+    if (status === STATUSES.failed) {
+      event.reasons = reasons;
+    }
+
+    if (transientErrorReasons?.length) {
+      event.transientError = { reasons: transientErrorReasons };
+    }
+
+    const newResult = new TermsResult({ serviceId, termsType, status, event });
+    const { eventType } = await this.repository.saveTermsResult(newResult, { trailers: runTrailers(run) });
+
+    run.coverage.processed++; // Counted per call: every terms of a run is either recorded once or persisted as skipped at run start; the Archivist is responsible for that invariant
+    run.tracked[status]++;
+
+    if (event.transientError) {
+      run.transientErrors++;
+    }
+
+    const transitionType = TRANSITIONS_BY_EVENT_TYPE[eventType];
+
+    if (transitionType) {
+      run.recordTransition(transitionType, { serviceId, termsType });
+    }
+  }
+
+  async completeRun() {
+    const run = this.activeRun;
+
+    run.markCompleted(new Date().toISOString());
+
+    await this.repository.saveRun(run, { trailers: runTrailers(run) });
+
+    this.currentRun = null;
+  }
+
+  async recoverCrashedRunIfAny({ getDeclaredTermsAtCommit }) { // Must be invoked before startRun: a new run-start commit would hide the crashed run's reference SHA from findLatestRunCommitSha. The callback returns the terms declared at a given declarations commit, since the recorder has no knowledge of the declarations module
+    let run;
+
+    try {
+      run = await this.repository.findLatestRun();
+    } catch (error) {
+      throw error.code ? error : new UnreadableRunError(error.message, { cause: error }); // System errors carry a code and are worth a retry, unlike a content that cannot be parsed or validated
+    }
+
+    if (!run || run.lastRun.status !== RUN_STATUSES.inProgress) {
+      return null;
+    }
+
+    const runStartSha = await this.repository.findLatestRunCommitSha();
+    const committedTerms = await this.repository.findCommittedTermsResultsSince(runStartSha);
+    const persistedSkipped = run.coverage.skipped; // Skips persisted by the run-start commit (e.g. terms not selected by a partial run) are preserved with their original reason, not re-attributed to the crash
+    const accountedKeys = new Set([ ...committedTerms, ...persistedSkipped ].map(({ serviceId, termsType }) => termsKey(serviceId, termsType)));
+    const declaredTerms = await getDeclaredTermsAtCommit(run.declarations.commit);
+    const crashSkipped = declaredTerms
+      .filter(({ serviceId, termsType }) => !accountedKeys.has(termsKey(serviceId, termsType)))
+      .map(({ serviceId, termsType }) => ({ serviceId, termsType, reason: 'no outcome recorded before the engine crashed' })); // Terms processed without a substantive change leave no commit, so they cannot be told apart from unprocessed ones and end up here, understating `processed`: the reason states the absence of a record rather than claiming the terms was not processed
+
+    run.coverage = { processed: committedTerms.length, skipped: [ ...persistedSkipped, ...crashSkipped ] };
+    ({ tracked: run.tracked, transientErrors: run.transientErrors } = await this.deriveCounts(committedTerms));
+    run.markCrashed(new Date().toISOString()); // run.transitions is left as persisted by the run-start commit (empty): deriving it back would mean parsing commit subjects, and the transitions of a crashed run remain derivable by consumers from its per-terms commits
+    await this.repository.saveRun(run, { trailers: runTrailers(run) }); // Carries the crashed run's id, so the finalization commit is greppable alongside the run it closes
+
+    return run;
+  }
+
+  async deriveCounts(committedTerms) { // The committed terms files still hold the crashed run's last written state, since recovery runs before any new startRun
+    const tracked = { ok: 0, failed: 0 };
+    let transientErrors = 0;
+
+    for (const { serviceId, termsType } of committedTerms) {
+      const result = await this.repository.findLatestTermsResult(serviceId, termsType);
+
+      if (!result) { // Defensive: a file removed by hand since the crash leaves the counts unchanged rather than failing the recovery
+        continue;
+      }
+
+      tracked[result.status]++;
+
+      if (result.event.transientError) {
+        transientErrors++;
+      }
+    }
+
+    return { tracked, transientErrors };
+  }
+
+  discardCurrentRun() { // Test entry point to drop the run in progress without persisting anything
+    this.currentRun = null;
+  }
+
+  get activeRun() { // Single guarded access path to the run in progress: dereferencing it before startRun throws, so no mutating method needs a separate assertion. External readers keep using the nullable currentRun property
+    if (!this.currentRun) {
+      throw new Error(`${this.constructor.name}: no tracking run in progress; call startRun first`);
+    }
+
+    return this.currentRun;
+  }
+}
+
+function runTrailers(run) {
+  return { [RUN_ID_TRAILER_KEY]: run.runId };
+}