@opentermsarchive/engine 16.0.2 → 16.1.1

package/src/archivist/tracking-results/run/dataMapper.js

@@ -0,0 +1,51 @@
+import Run, { RUN_STATUSES } from './index.js';
+
+export const FILE_NAME = 'run.json';
+
+export function toPersistence(run) {
+  return {
+    message: formatMessage(run),
+    content: `${JSON.stringify(toJSON(run), null, 2)}\n`,
+    filePath: FILE_NAME,
+    date: run.lastRun.endDate || run.lastRun.startDate,
+  };
+}
+
+export function toDomain(data) {
+  try {
+    const run = Object.assign(new Run(), data); // The persisted shape is the in-memory shape, so every field accumulated through the run's lifecycle is restored as is
+
+    run.validate();
+
+    return run;
+  } catch (error) {
+    throw new Error(`Invalid run content: ${error.message}`);
+  }
+}
+
+export function formatMessage(run) { // The status of the run tells which step of its lifecycle is being committed
+  switch (run.lastRun.status) {
+  case RUN_STATUSES.inProgress:
+    return `Start run ${run.shortRunId}`;
+  case RUN_STATUSES.completed:
+    return `Complete run ${run.shortRunId} (${run.tracked.ok} ok, ${run.tracked.failed} failed)`;
+  case RUN_STATUSES.crashed:
+    return `Finalize crashed run ${run.shortRunId}`;
+  default:
+    throw new Error(`Unknown run status: "${run.lastRun.status}"`);
+  }
+}
+
+function toJSON(run) {
+  return {
+    runId: run.runId,
+    collectionId: run.collectionId,
+    schedule: run.schedule,
+    lastRun: run.lastRun,
+    declarations: run.declarations,
+    tracked: run.tracked,
+    coverage: run.coverage,
+    transitions: run.transitions,
+    transientErrors: run.transientErrors,
+  };
+}

package/src/archivist/tracking-results/run/dataMapper.test.js

@@ -0,0 +1,168 @@
+import { expect } from 'chai';
+
+import { FILE_NAME, formatMessage, toDomain, toPersistence } from './dataMapper.js';
+
+import Run from './index.js';
+
+const VALID_PARAMS = {
+  runId: 'ota-run-f47ac10b-58cc-4372-a567-0e02b2c3d479',
+  collectionId: 'france',
+  schedule: '30 */12 * * *',
+  startDate: '2026-04-06T10:30:00Z',
+  engineVersion: '12.0.0',
+  declarationsCommit: 'abc123def456',
+  servicesCount: 1,
+  termsCount: 1,
+};
+
+describe('run/dataMapper', () => {
+  describe('FILE_NAME', () => {
+    it('is run.json at the repository root', () => {
+      expect(FILE_NAME).to.equal('run.json');
+    });
+  });
+
+  describe('#formatMessage', () => {
+    it('formats an in-progress run', () => {
+      const run = new Run(VALID_PARAMS);
+
+      expect(formatMessage(run)).to.equal('Start run ota-run-f47ac10b');
+    });
+
+    it('formats a completed run with counts read from run.tracked', () => {
+      const run = new Run(VALID_PARAMS);
+
+      run.markCompleted('2026-04-06T10:42:34Z');
+      run.tracked = { ok: 1518, failed: 5 };
+
+      expect(formatMessage(run)).to.equal('Complete run ota-run-f47ac10b (1518 ok, 5 failed)');
+    });
+
+    it('formats a crashed run', () => {
+      const run = new Run(VALID_PARAMS);
+
+      run.markCrashed('2026-04-06T10:42:34Z');
+
+      expect(formatMessage(run)).to.equal('Finalize crashed run ota-run-f47ac10b');
+    });
+
+    it('throws for unknown status', () => {
+      const run = new Run(VALID_PARAMS);
+
+      run.lastRun.status = 'nonsense';
+
+      expect(() => formatMessage(run)).to.throw(/Unknown run status/);
+    });
+  });
+
+  describe('#toPersistence', () => {
+    context('with an in-progress run', () => {
+      let persistence;
+
+      before(() => {
+        persistence = toPersistence(new Run(VALID_PARAMS));
+      });
+
+      it('uses the Start run subject', () => {
+        expect(persistence.message).to.equal('Start run ota-run-f47ac10b');
+      });
+
+      it('writes to run.json at the repository root', () => {
+        expect(persistence.filePath).to.equal('run.json');
+      });
+
+      it('serialises the run payload structure as pretty JSON terminated by a newline', () => {
+        const parsed = JSON.parse(persistence.content);
+
+        expect(parsed).to.have.all.keys([ 'runId', 'collectionId', 'schedule', 'lastRun', 'declarations', 'tracked', 'coverage', 'transitions', 'transientErrors' ]);
+        expect(parsed.runId).to.equal('ota-run-f47ac10b-58cc-4372-a567-0e02b2c3d479');
+        expect(persistence.content.endsWith('\n')).to.be.true;
+      });
+
+      it('uses the start date when no end date is set', () => {
+        expect(persistence.date).to.equal('2026-04-06T10:30:00Z');
+      });
+    });
+
+    context('with a completed run', () => {
+      let persistence;
+
+      before(() => {
+        const run = new Run(VALID_PARAMS);
+
+        run.markCompleted('2026-04-06T10:42:34Z');
+        run.tracked = { ok: 1518, failed: 5 };
+        persistence = toPersistence(run);
+      });
+
+      it('uses the Complete run subject with counts', () => {
+        expect(persistence.message).to.equal('Complete run ota-run-f47ac10b (1518 ok, 5 failed)');
+      });
+
+      it('uses the end date once it is set', () => {
+        expect(persistence.date).to.equal('2026-04-06T10:42:34Z');
+      });
+    });
+
+    context('with a crashed run', () => {
+      it('uses the Finalize crashed run subject', () => {
+        const run = new Run(VALID_PARAMS);
+
+        run.markCrashed('2026-04-06T10:42:34Z');
+        expect(toPersistence(run).message).to.equal('Finalize crashed run ota-run-f47ac10b');
+      });
+    });
+  });
+
+  describe('#toDomain', () => {
+    it('rebuilds a Run instance from persisted data', () => {
+      const persistence = toPersistence(new Run(VALID_PARAMS));
+      const data = JSON.parse(persistence.content);
+      const run = toDomain(data);
+
+      expect(run).to.be.instanceOf(Run);
+      expect(run.runId).to.equal(VALID_PARAMS.runId);
+      expect(run.lastRun.status).to.equal('in_progress');
+      expect(run.shortRunId).to.equal('ota-run-f47ac10b');
+    });
+
+    it('restores lifecycle state (status, endDate, tracked, coverage, transitions, transientErrors)', () => {
+      const original = new Run(VALID_PARAMS);
+
+      original.markCompleted('2026-04-06T10:42:34Z');
+      original.tracked = { ok: 1518, failed: 5 };
+      original.coverage = { processed: 1523, skipped: [{ serviceId: 'X', termsType: 'Y', reason: 'declaration validation failed' }] };
+      original.recordTransition('newFailures', { serviceId: 'A', termsType: 'B' });
+      original.transientErrors = 23;
+
+      const persistence = toPersistence(original);
+      const rebuilt = toDomain(JSON.parse(persistence.content));
+
+      expect(rebuilt.lastRun.endDate).to.equal('2026-04-06T10:42:34Z');
+      expect(rebuilt.lastRun.status).to.equal('completed');
+      expect(rebuilt.tracked).to.deep.equal({ ok: 1518, failed: 5 });
+      expect(rebuilt.coverage).to.deep.equal({ processed: 1523, skipped: [{ serviceId: 'X', termsType: 'Y', reason: 'declaration validation failed' }] });
+      expect(rebuilt.transitions.newFailures).to.deep.equal([{ serviceId: 'A', termsType: 'B' }]);
+      expect(rebuilt.transientErrors).to.equal(23);
+      expect(toPersistence(rebuilt).content).to.equal(persistence.content);
+    });
+
+    context('when persisted data is invalid', () => {
+      it('throws an error contextualised with the run domain', () => {
+        const persistence = toPersistence(new Run(VALID_PARAMS));
+        const data = JSON.parse(persistence.content);
+
+        data.runId = 'bare-uuid-without-prefix';
+        expect(() => toDomain(data)).to.throw(/Invalid run content/);
+      });
+
+      it('throws when a required nested field is missing', () => {
+        const persistence = toPersistence(new Run(VALID_PARAMS));
+        const data = JSON.parse(persistence.content);
+
+        delete data.lastRun.startDate;
+        expect(() => toDomain(data)).to.throw(/Invalid run content/);
+      });
+    });
+  });
+});

package/src/archivist/tracking-results/run/index.js

@@ -0,0 +1,115 @@
+import { randomUUID } from 'crypto';
+
+export const RUN_STATUSES = Object.freeze({
+  inProgress: 'in_progress',
+  completed: 'completed',
+  crashed: 'crashed',
+});
+
+export const RUN_ID_PREFIX = 'ota-run-'; // Disambiguates from Git short SHAs and tags run IDs uniquely across logs and Git trailers
+const RUN_ID_REGEXP = /^ota-run-([0-9a-f]{8})-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
+
+const VALID_STATUSES = Object.freeze(Object.values(RUN_STATUSES));
+const VALID_TRANSITION_TYPES = Object.freeze([ 'newFailures', 'recoveries', 'reasonChanges' ]);
+
+export default class Run {
+  static generateId() {
+    return `${RUN_ID_PREFIX}${randomUUID()}`;
+  }
+
+  constructor({
+    runId,
+    collectionId,
+    schedule,
+    startDate,
+    engineVersion,
+    declarationsCommit,
+    servicesCount,
+    termsCount,
+  } = {}) {
+    this.runId = runId;
+    this.collectionId = collectionId;
+    this.schedule = schedule ?? null;
+    this.lastRun = {
+      startDate,
+      endDate: null,
+      engineVersion,
+      status: RUN_STATUSES.inProgress,
+    };
+    this.declarations = {
+      commit: declarationsCommit,
+      services: servicesCount,
+      terms: termsCount,
+    };
+    this.tracked = { ok: 0, failed: 0 };
+    this.coverage = { processed: 0, skipped: [] };
+    this.transitions = { newFailures: [], recoveries: [], reasonChanges: [] };
+    this.transientErrors = 0;
+  }
+
+  get shortRunId() {
+    return `${RUN_ID_PREFIX}${runIdMatch(this.runId)[1]}`; // Keeps the `ota-run-` prefix so the short form remains unmistakable against a Git short SHA
+  }
+
+  markCompleted(endDate) {
+    this.lastRun.endDate = endDate;
+    this.lastRun.status = RUN_STATUSES.completed;
+  }
+
+  markCrashed(endDate) {
+    this.lastRun.endDate = endDate;
+    this.lastRun.status = RUN_STATUSES.crashed;
+  }
+
+  addSkipped({ serviceId, termsType, reason }) {
+    this.coverage.skipped.push({ serviceId, termsType, reason });
+  }
+
+  recordTransition(type, { serviceId, termsType }) {
+    if (!VALID_TRANSITION_TYPES.includes(type)) {
+      throw new Error(`${this.constructor.name}: invalid transition type "${type}"; must be one of "${VALID_TRANSITION_TYPES.join('", "')}"`);
+    }
+
+    this.transitions[type].push({ serviceId, termsType });
+  }
+
+  validate() {
+    const requiredParams = { // Keyed by constructor param names, as callers provide them, even for the values stored under sub-objects; optional chaining so a missing sub-object reports its field as missing instead of crashing the validation
+      runId: this.runId,
+      collectionId: this.collectionId,
+      startDate: this.lastRun?.startDate,
+      engineVersion: this.lastRun?.engineVersion,
+      declarationsCommit: this.declarations?.commit,
+    };
+
+    for (const [ requiredParam, value ] of Object.entries(requiredParams)) {
+      if (value == null) {
+        throw new Error(`${this.constructor.name} is not valid; "${requiredParam}" is missing`);
+      }
+    }
+
+    if (!VALID_STATUSES.includes(this.lastRun.status)) {
+      throw new Error(`${this.constructor.name} is not valid; "lastRun.status" must be one of "${VALID_STATUSES.join('", "')}", got "${this.lastRun.status}"`);
+    }
+
+    if (!Array.isArray(this.coverage?.skipped)) { // The only accumulated field that crash recovery reads structurally
+      throw new Error(`${this.constructor.name} is not valid; "coverage.skipped" must be an array`);
+    }
+
+    try {
+      runIdMatch(this.runId); // Single source of truth for the runId format check, reused by shortRunId
+    } catch (error) {
+      throw new Error(`${this.constructor.name} is not valid; ${error.message}`);
+    }
+  }
+}
+
+function runIdMatch(runId) {
+  const match = RUN_ID_REGEXP.exec(runId);
+
+  if (!match) {
+    throw new Error(`Invalid runId "${runId}"; expected format "ota-run-<uuid>"`);
+  }
+
+  return match;
+}

package/src/archivist/tracking-results/run/index.test.js

@@ -0,0 +1,221 @@
+import { expect } from 'chai';
+
+import Run from './index.js';
+
+const VALID_PARAMS = {
+  runId: 'ota-run-f47ac10b-58cc-4372-a567-0e02b2c3d479',
+  collectionId: 'france',
+  schedule: '30 */12 * * *',
+  startDate: '2026-04-06T10:30:00Z',
+  engineVersion: '12.0.0',
+  declarationsCommit: 'abc123def456',
+  servicesCount: 523,
+  termsCount: 1523,
+};
+
+describe('Run', () => {
+  let subject;
+
+  describe('.generateId', () => {
+    it('returns an id with the "ota-run-" prefix and a UUID payload', () => {
+      const id = Run.generateId();
+
+      expect(id).to.match(/^ota-run-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/);
+    });
+
+    it('produces distinct ids on successive calls', () => {
+      expect(Run.generateId()).to.not.equal(Run.generateId());
+    });
+
+    it('produces ids accepted by Run.validate', () => {
+      const run = new Run({ ...VALID_PARAMS, runId: Run.generateId() });
+
+      expect(() => run.validate()).to.not.throw();
+    });
+  });
+
+  describe('constructor', () => {
+    context('with flat params (initial construction)', () => {
+      before(() => {
+        subject = new Run(VALID_PARAMS);
+      });
+
+      it('exposes runId, collectionId and schedule at the root', () => {
+        expect(subject.runId).to.equal('ota-run-f47ac10b-58cc-4372-a567-0e02b2c3d479');
+        expect(subject.collectionId).to.equal('france');
+        expect(subject.schedule).to.equal('30 */12 * * *');
+      });
+
+      it('initializes lastRun with status "in_progress" and no endDate', () => {
+        expect(subject.lastRun).to.deep.equal({
+          startDate: '2026-04-06T10:30:00Z',
+          endDate: null,
+          engineVersion: '12.0.0',
+          status: 'in_progress',
+        });
+      });
+
+      it('initializes declarations with the commit and counts', () => {
+        expect(subject.declarations).to.deep.equal({
+          commit: 'abc123def456',
+          services: 523,
+          terms: 1523,
+        });
+      });
+
+      it('initializes tracked counters to zero', () => {
+        expect(subject.tracked).to.deep.equal({ ok: 0, failed: 0 });
+      });
+
+      it('initializes coverage with empty skipped array', () => {
+        expect(subject.coverage).to.deep.equal({ processed: 0, skipped: [] });
+      });
+
+      it('initializes transitions with empty arrays', () => {
+        expect(subject.transitions).to.deep.equal({ newFailures: [], recoveries: [], reasonChanges: [] });
+      });
+
+      it('initializes transientErrors to zero', () => {
+        expect(subject.transientErrors).to.equal(0);
+      });
+
+      context('when no schedule is provided', () => {
+        it('defaults schedule to null', () => {
+          const run = new Run({ ...VALID_PARAMS, schedule: undefined });
+
+          expect(run.schedule).to.be.null;
+        });
+      });
+    });
+  });
+
+  describe('#shortRunId', () => {
+    it('returns the prefixed first segment of the UUID', () => {
+      subject = new Run(VALID_PARAMS);
+      expect(subject.shortRunId).to.equal('ota-run-f47ac10b');
+    });
+
+    it('throws when the runId does not match the expected format', () => {
+      subject = new Run({ ...VALID_PARAMS, runId: 'not-a-valid-id' });
+      expect(() => subject.shortRunId).to.throw(/Invalid runId/);
+    });
+  });
+
+  describe('#markCompleted', () => {
+    it('sets status to "completed" and records the endDate', () => {
+      subject = new Run(VALID_PARAMS);
+      subject.markCompleted('2026-04-06T10:42:34Z');
+      expect(subject.lastRun.status).to.equal('completed');
+      expect(subject.lastRun.endDate).to.equal('2026-04-06T10:42:34Z');
+    });
+  });
+
+  describe('#markCrashed', () => {
+    it('sets status to "crashed" and records the endDate', () => {
+      subject = new Run(VALID_PARAMS);
+      subject.markCrashed('2026-04-06T10:42:34Z');
+      expect(subject.lastRun.status).to.equal('crashed');
+      expect(subject.lastRun.endDate).to.equal('2026-04-06T10:42:34Z');
+    });
+  });
+
+  describe('#addSkipped', () => {
+    it('appends an entry to coverage.skipped', () => {
+      subject = new Run(VALID_PARAMS);
+      subject.addSkipped({ serviceId: 'Facebook', termsType: 'Terms of Service', reason: 'declaration validation failed' });
+      expect(subject.coverage.skipped).to.deep.equal([
+        { serviceId: 'Facebook', termsType: 'Terms of Service', reason: 'declaration validation failed' },
+      ]);
+    });
+
+    it('accumulates multiple entries in order', () => {
+      subject = new Run(VALID_PARAMS);
+      subject.addSkipped({ serviceId: 'A', termsType: 'ToS', reason: 'bad A' });
+      subject.addSkipped({ serviceId: 'B', termsType: 'PP', reason: 'bad B' });
+      expect(subject.coverage.skipped).to.deep.equal([
+        { serviceId: 'A', termsType: 'ToS', reason: 'bad A' },
+        { serviceId: 'B', termsType: 'PP', reason: 'bad B' },
+      ]);
+    });
+  });
+
+  describe('#recordTransition', () => {
+    [ 'newFailures', 'recoveries', 'reasonChanges' ].forEach(type => {
+      context(`with type "${type}"`, () => {
+        it('appends to the matching transitions list', () => {
+          subject = new Run(VALID_PARAMS);
+          subject.recordTransition(type, { serviceId: 'Facebook', termsType: 'Terms of Service' });
+          expect(subject.transitions[type]).to.deep.equal([
+            { serviceId: 'Facebook', termsType: 'Terms of Service' },
+          ]);
+        });
+      });
+    });
+
+    context('with an unknown type', () => {
+      it('throws an error', () => {
+        subject = new Run(VALID_PARAMS);
+        expect(() => subject.recordTransition('unknown', { serviceId: 'X', termsType: 'Y' })).to.throw(/transition type/);
+      });
+    });
+  });
+
+  describe('#validate', () => {
+    context('when all required params are present', () => {
+      it('does not throw', () => {
+        subject = new Run(VALID_PARAMS);
+        expect(subject.validate.bind(subject)).to.not.throw();
+      });
+    });
+
+    [ 'runId', 'collectionId', 'startDate', 'engineVersion', 'declarationsCommit' ].forEach(requiredParam => {
+      describe(`"${requiredParam}"`, () => {
+        context('when missing', () => {
+          it('throws an error', () => {
+            subject = new Run({ ...VALID_PARAMS, [requiredParam]: undefined });
+            expect(subject.validate.bind(subject)).to.throw(RegExp(requiredParam));
+          });
+        });
+      });
+    });
+
+    context('when lastRun.status is not a valid value', () => {
+      it('throws an error', () => {
+        subject = new Run(VALID_PARAMS);
+        subject.lastRun.status = 'unknown';
+        expect(subject.validate.bind(subject)).to.throw(/status/);
+      });
+    });
+
+    context('when coverage.skipped is not an array', () => {
+      it('throws an error', () => {
+        subject = new Run(VALID_PARAMS);
+        subject.coverage = { processed: 0, skipped: {} };
+        expect(subject.validate.bind(subject)).to.throw(/coverage\.skipped/);
+      });
+    });
+
+    context('when runId does not match the expected format', () => {
+      it('throws an error', () => {
+        subject = new Run({ ...VALID_PARAMS, runId: 'f47ac10b-58cc-4372-a567-0e02b2c3d479' });
+        expect(subject.validate.bind(subject)).to.throw(/runId/);
+      });
+    });
+
+    context('when lastRun is replaced with null after construction', () => {
+      it('throws on the first required field resolved through lastRun', () => {
+        subject = new Run(VALID_PARAMS);
+        subject.lastRun = null;
+        expect(subject.validate.bind(subject)).to.throw(/startDate/);
+      });
+    });
+
+    context('when declarations is replaced with null after construction', () => {
+      it('throws on the first required field resolved through declarations', () => {
+        subject = new Run(VALID_PARAMS);
+        subject.declarations = null;
+        expect(subject.validate.bind(subject)).to.throw(/declarationsCommit/);
+      });
+    });
+  });
+});