@opentermsarchive/engine 16.0.2 → 16.1.0

package/src/archivist/recorder/repositories/git/dataMapper.js

@@ -67,7 +67,8 @@ export function toDomain(commit) {
   }
 
   const [relativeFilePath] = modifiedFilesInCommit;
-  const snapshotIdsMatch = body.match(/\b[0-9a-f]{5,40}\b/g);
+  const bodyWithoutTrailers = Object.keys(trailers).length ? body.split(/\n\n+/).slice(0, -1).join('\n\n') : body; // Trailers, when present, are the last section of the body; their values, such as the hexadecimal segments of a run ID, must not be read as snapshot IDs
+  const snapshotIdsMatch = bodyWithoutTrailers.match(/\b[0-9a-f]{5,40}\b/g);
 
   const [ termsType, documentId ] = path.basename(relativeFilePath, path.extname(relativeFilePath)).split(TERMS_TYPE_AND_DOCUMENT_ID_SEPARATOR);
 

package/src/archivist/recorder/repositories/git/index.js

@@ -8,27 +8,16 @@ import path from 'path';
 
 import mime from 'mime';
 
+import Git from '../../../../git/index.js';
+import { isPlainPathSegment } from '../../../../git/pathSegment.js';
 import RepositoryInterface from '../interface.js';
 
 import * as DataMapper from './dataMapper.js';
-import Git from './git.js';
 
 const fs = fsApi.promises;
 
 const RECORD_ID_REGEXP = /^[0-9a-f]{7,40}$/i; // Git commit SHA 7 (abbreviated) to 40 (full) hexadecimal characters. Prevent value such as `--output=…` to be parsed as a command-line option
 
-const CONTROL_CHARACTERS_REGEXP = /\p{Cc}/u; // Matches any Unicode "control" character: the C0 range (U+0000 to U+001F), DEL (U+007F) and the C1 range (U+0080 to U+009F), i.e. 65 non-printable characters including NUL. The `u` flag is required for the `\p{...}` property escape to be recognised, otherwise the pattern would match the literal text `p{Cc}`. Legitimate service IDs, terms types and document IDs never contain these, and NUL in particular can truncate a value once it reaches git or the filesystem, so any segment holding one is rejected.
-
-// Keeps hostile values from reaching git, where a pathspec that resolves outside the repository (such as `../foo/*`) aborts with an error that exposes the repository location.
-function isPlainPathSegment(segment) {
-  return segment.length > 0
-    && segment !== '.'
-    && segment !== '..'
-    && !segment.includes('/')
-    && !segment.includes('\\')
-    && !CONTROL_CHARACTERS_REGEXP.test(segment);
-}
-
 function canMatchRecordFilePath(...pathSegments) {
   // A non-string segment means "not provided" (`undefined`, or `false` for an absent document ID) and constrains nothing
   return pathSegments.every(segment => typeof segment !== 'string' || isPlainPathSegment(segment));

package/src/archivist/recorder/repositories/git/index.test.js

@@ -7,11 +7,11 @@ import chaiAsPromised from 'chai-as-promised';
 import config from 'config';
 import mime from 'mime';
 
+import Git from '../../../../git/index.js';
 import Snapshot from '../../snapshot.js';
 import Version from '../../version.js';
 
 import { TERMS_TYPE_AND_DOCUMENT_ID_SEPARATOR, SNAPSHOT_ID_MARKER, COMMIT_MESSAGE_PREFIXES } from './dataMapper.js';
-import Git from './git.js';
 
 import GitRepository from './index.js';
 
@@ -398,6 +398,28 @@ describe('GitRepository', () => {
         expect(record.metadata).to.deep.equal(METADATA);
       });
 
+      context('when a metadata value contains hexadecimal segments', () => {
+        let recordWithRunId;
+
+        before(async () => {
+          const { id: recordId } = await subject.save(new Version({
+            serviceId: SERVICE_PROVIDER_ID,
+            termsType: TERMS_TYPE,
+            content: `${CONTENT} (updated)`,
+            fetchDate: FETCH_DATE_LATER,
+            snapshotIds: [SNAPSHOT_ID],
+            mimeType: HTML_MIME_TYPE,
+            metadata: { ...METADATA, 'x-run-id': 'ota-run-189e7be3-60ef-40c7-ac28-81a44288e105' },
+          }));
+
+          recordWithRunId = await subject.findById(recordId);
+        });
+
+        it('returns only the snapshot ID', () => {
+          expect(recordWithRunId.snapshotIds).to.deep.equal([SNAPSHOT_ID]);
+        });
+      });
+
       context('when requested record does not exist', () => {
         it('returns null', async () => {
           expect(await subject.findById('inexistantID')).to.equal(null);

package/src/archivist/services/index.js

@@ -2,8 +2,10 @@ import fs from 'fs/promises';
 import path from 'path';
 import { pathToFileURL } from 'url';
 
+import async from 'async';
 import config from 'config';
 
+import Git from '../../git/index.js';
 import * as exposedFilters from '../extract/exposedFilters.js';
 
 import Service from './service.js';
@@ -13,6 +15,8 @@ import Terms from './terms.js';
 export const DECLARATIONS_PATH = './declarations';
 const declarationsPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.collectionPath'), DECLARATIONS_PATH);
 
+const MAX_PARALLEL_DECLARATIONS_READS = 5; // Reading declarations at a commit spawns one git process per service; left unbounded, a large collection would exhaust the file descriptors or processes allowed to the engine
+
 const JSON_EXT = '.json';
 const JS_EXT = '.js';
 const HISTORY_SUFFIX = '.history';
@@ -106,10 +110,11 @@ function createWrappedFilter(baseFunction, filterName, filterParams) {
     return;
   }
 
-  if (filterParams || exposedFilters[filterName]) { // Built-in filters always receive their parameters before the context, even when none are declared
+  if (filterParams !== undefined || exposedFilters[filterName]) { // Built-in filters always receive their parameters before the context, even when none are declared
     const wrappedFilter = (webPageDOM, context) => baseFunction(webPageDOM, filterParams, context);
 
     Object.defineProperty(wrappedFilter, 'name', { value: filterName });
+    Object.defineProperty(wrappedFilter, 'declaration', { value: filterParams === undefined ? filterName : { [filterName]: filterParams } }); // Keep the declared form, as the parameters are otherwise only reachable through the closure
 
     return wrappedFilter;
   }
@@ -145,6 +150,45 @@ export function getServiceFilters(serviceFilters, declaredFilters) {
 export async function getDeclaredServicesIds() {
   const fileNames = await fs.readdir(declarationsPath);
 
+  return declaredServicesIdsFromFileNames(fileNames);
+}
+
+export function getDeclarationsCommit() { // Resolves to null when the declarations are not versioned with Git
+  return Git.getHeadSha(declarationsPath);
+}
+
+// Returns the [{ serviceId, termsType }] declared at the given commit of the declarations repository.
+// Reads through git so the answer reflects the declarations exactly as they were at that commit, not as they are on disk; used by crash recovery to derive the coverage of a run that referenced this commit.
+export async function getDeclaredTermsAtCommit(commit) {
+  const fileNames = await Git.listFilesAtCommit(declarationsPath, commit);
+  const serviceIds = declaredServicesIdsFromFileNames(fileNames);
+
+  return declaredTermsOf(serviceIds, async serviceId => {
+    const rawDeclaration = await Git.readFileAtCommit(declarationsPath, commit, `${serviceId}${JSON_EXT}`);
+
+    try {
+      return JSON.parse(rawDeclaration);
+    } catch (error) {
+      throw new Error(`The "${serviceId}" service declaration at commit ${commit} is malformed and cannot be parsed`);
+    }
+  });
+}
+
+export async function getDeclaredTerms() { // Working-tree counterpart of getDeclaredTermsAtCommit; used as approximation when a declarations commit is not reachable anymore
+  return declaredTermsOf(await getDeclaredServicesIds(), loadServiceDeclaration);
+}
+
+async function declaredTermsOf(serviceIds, loadDeclaration) {
+  const declaredTermsPerService = await async.mapLimit(serviceIds, MAX_PARALLEL_DECLARATIONS_READS, async serviceId => {
+    const declaration = await loadDeclaration(serviceId);
+
+    return Object.keys(declaration.terms ?? {}).map(termsType => ({ serviceId, termsType }));
+  });
+
+  return declaredTermsPerService.flat(); // Flattened in serviceIds order to keep the result deterministic regardless of read completion order
+}
+
+function declaredServicesIdsFromFileNames(fileNames) {
   return fileNames
     .filter(fileName => fileName.endsWith(JSON_EXT) && !fileName.includes(`${HISTORY_SUFFIX}${JSON_EXT}`))
     .map(fileName => path.basename(fileName, JSON_EXT));

package/src/archivist/services/index.test.js

@@ -2,10 +2,12 @@ import fs from 'fs/promises';
 import path from 'path';
 
 import { expect, use } from 'chai';
+import config from 'config';
 import sinon from 'sinon';
 import sinonChai from 'sinon-chai';
 
 import expectedServices from '../../../test/fixtures/services.js';
+import Git, { GitObjectNotFoundError } from '../../git/index.js';
 import createWebPageDOM from '../extract/dom.js';
 import * as exposedFilters from '../extract/exposedFilters.js';
 
@@ -13,7 +15,7 @@ import Service from './service.js';
 import SourceDocument from './sourceDocument.js';
 import Terms from './terms.js';
 
-import { getDeclaredServicesIds, loadServiceDeclaration, loadServiceFilters, getServiceFilters, createSourceDocuments, createServiceFromDeclaration, load, loadWithHistory } from './index.js';
+import { getDeclaredServicesIds, getDeclaredTermsAtCommit, loadServiceDeclaration, loadServiceFilters, getServiceFilters, createSourceDocuments, createServiceFromDeclaration, load, loadWithHistory } from './index.js';
 
 use(sinonChai);
 
@@ -151,6 +153,36 @@ describe('Services', () => {
     });
   });
 
+  describe('#getDeclaredTermsAtCommit', () => {
+    const declarationsPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.collectionPath'), './declarations');
+
+    afterEach(() => sinon.restore());
+
+    it('returns the terms declared at the given commit, whatever the working tree contains', async function () {
+      this.timeout(10000);
+
+      const commit = await Git.getHeadSha(declarationsPath); // The test declarations are committed in the engine repository itself, so its HEAD reflects them exactly
+      const expected = Object.entries(expectedServices).flatMap(([ serviceId, service ]) => service.getTermsTypes().map(termsType => ({ serviceId, termsType })));
+
+      sinon.stub(fs, 'readdir').resolves(['uncommitted-service.json']); // Make the working tree differ from the commit: it must not be read
+      sinon.stub(fs, 'readFile').resolves(JSON.stringify({ name: 'Uncommitted service', terms: { Imprint: {} } }));
+
+      expect(await getDeclaredTermsAtCommit(commit)).to.have.deep.members(expected);
+    });
+
+    it('throws a GitObjectNotFoundError for an unknown commit', async () => {
+      try {
+        await getDeclaredTermsAtCommit('deadbeefdeadbeefdeadbeefdeadbeefdeadbeef');
+      } catch (error) {
+        expect(error).to.be.an.instanceOf(GitObjectNotFoundError);
+
+        return;
+      }
+
+      expect.fail('No error was thrown');
+    });
+  });
+
   describe('#loadServiceDeclaration', () => {
     let readFile;
 
@@ -310,6 +342,19 @@ describe('Services', () => {
       expect(result[0](null, 'context')).to.equal('foo');
     });
 
+    it('keeps the declared form of filters declared with parameters', () => {
+      const [filter] = getServiceFilters({ paramFilter: (dom, param) => param }, [{ paramFilter: [ 'foo', 'bar' ] }]);
+
+      expect(filter.declaration).to.deep.equal({ paramFilter: [ 'foo', 'bar' ] });
+    });
+
+    it('keeps the name as the declared form of exposed filters declared by string name', () => {
+      const [filterName] = Object.keys(exposedFilters);
+      const [filter] = getServiceFilters({}, [filterName]);
+
+      expect(filter.declaration).to.equal(filterName);
+    });
+
     describe('parameters passed to filters', () => {
       let serviceLoadedFilters;
       let passedDOM;
@@ -348,6 +393,12 @@ describe('Services', () => {
           testParameterPassing({ param1: 'param1', param2: 'param2' });
         });
       });
+
+      context('as a falsy value', () => {
+        it('passes parameters correctly', () => {
+          testParameterPassing(false);
+        });
+      });
     });
   });
 

package/src/archivist/services/sourceDocument.js

@@ -40,8 +40,14 @@ export default class SourceDocument {
   }
 
   clearContent() {
-    this.content = null;
+    this.content = null; // Only the potentially large content is cleared: the MIME type is read after the extraction, to record the tracking results
+  }
+
+  resetObservations() {
+    // mimeType and snapshotId are observations of a single tracking attempt, but they are stored on declaration objects that live for the whole process: without this reset, a failed fetch would expose the previous run's values as if they belonged to the failed attempt.
+    // The proper pattern would be for the fetch and extract pipeline to return its observations instead of mutating the declarations, letting consumers build their records from run-scoped data; this reset contains that debt rather than fixing it.
     this.mimeType = null;
+    this.snapshotId = null;
   }
 
   static extractCssSelectorsFromProperty(property) {
@@ -83,7 +89,7 @@ export default class SourceDocument {
       fetch: this.location,
       select: this.contentSelectors,
       remove: this.insignificantContentSelectors,
-      filter: this.filters ? this.filters.map(filter => filter.name) : undefined,
+      filter: this.filters ? this.filters.map(filter => filter.declaration ?? filter.name) : undefined, // Filters declared with parameters carry their declared form, so that a change of parameters is persisted
       executeClientScripts: this.executeClientScripts,
     };
   }

package/src/archivist/services/sourceDocument.test.js

@@ -214,6 +214,29 @@ describe('SourceDocument', () => {
     });
   });
 
+  describe('#clearContent', () => {
+    it('clears the content but keeps the MIME type', () => {
+      const sourceDocument = new SourceDocument({ location: URL, content: '<html></html>', mimeType: 'text/html' });
+
+      sourceDocument.clearContent();
+
+      expect(sourceDocument.content).to.be.null;
+      expect(sourceDocument.mimeType).to.equal('text/html');
+    });
+  });
+
+  describe('#resetObservations', () => {
+    it('clears the MIME type and the snapshot ID observed by a previous tracking', () => {
+      const sourceDocument = new SourceDocument({ location: URL, mimeType: 'text/html' });
+
+      sourceDocument.snapshotId = 'abc123';
+      sourceDocument.resetObservations();
+
+      expect(sourceDocument.mimeType).to.be.null;
+      expect(sourceDocument.snapshotId).to.be.null;
+    });
+  });
+
   describe('#toPersistence', () => {
     it('converts basic source document declarations into JSON representation', () => {
       const result = new SourceDocument({
@@ -275,5 +298,19 @@ describe('SourceDocument', () => {
 
       expect(result).to.deep.equal(expectedResult);
     });
+
+    it('converts filters declared with parameters to their declared form', () => {
+      const filterWithParameters = () => {};
+
+      Object.defineProperty(filterWithParameters, 'declaration', { value: { removeQueryParams: ['utm_source'] } });
+
+      const result = new SourceDocument({
+        location: URL,
+        contentSelectors: 'body',
+        filters: [ filterWithParameters, function filterSomething() {} ],
+      }).toPersistence();
+
+      expect(result.filter).to.deep.equal([{ removeQueryParams: ['utm_source'] }, 'filterSomething' ]);
+    });
   });
 });

package/src/archivist/tracking-results/errors.js

@@ -0,0 +1,5 @@
+/* eslint-disable max-classes-per-file */ // Grouping the module's error types here is the point of an errors module
+
+export class MissingCollectionIdError extends Error {} // The collection metadata does not provide the id that identifies the collection in every persisted run; tracking-results cannot record without it
+
+export class UnreadableRunError extends Error {} // The persisted run.json cannot be read as a valid Run (corrupted JSON, incompatible schema from another engine version): recovery is impossible by construction, unlike infrastructure failures for which a retry is meaningful

package/src/archivist/tracking-results/index.js

@@ -0,0 +1,233 @@
+import events from 'events';
+import { createRequire } from 'module';
+
+import config from 'config';
+import mime from 'mime';
+
+import { GitObjectNotFoundError } from '../../git/index.js';
+import { getCollection } from '../collection/index.js';
+import { ExtractDocumentError } from '../extract/index.js';
+import { FetchDocumentError } from '../fetcher/index.js';
+import * as declaredServices from '../services/index.js';
+import Service from '../services/service.js';
+
+import { MissingCollectionIdError, UnreadableRunError } from './errors.js';
+import TrackingResultsRecorder from './recorder.js';
+import TrackingResultsRepository from './repository.js';
+import { STATUSES } from './terms-result/index.js';
+
+export { MissingCollectionIdError } from './errors.js';
+export { RUN_ID_TRAILER_KEY } from './recorder.js';
+
+const require = createRequire(import.meta.url);
+const { version: PACKAGE_VERSION } = require('../../../package.json');
+
+export default class TrackingResults extends events.EventEmitter {
+  static async create(trackingResultsConfig) { // Resolves the engine-side wiring (collection identity, schedule, engine version) so the caller does not need to know it
+    if (trackingResultsConfig.storage.type !== 'git') { // Git is the only supported backend, as the audit trail relies on its tamper-evident properties
+      throw new Error(`Unsupported tracking-results storage type "${trackingResultsConfig.storage.type}"; only "git" is supported`);
+    }
+
+    const collection = await getCollection();
+    const collectionId = collection.metadata?.id; // getCollection always resolves to a Collection instance; metadata stays undefined when the metadata file is absent, which is exactly the missing-id case reported below
+
+    if (!collectionId) {
+      throw new MissingCollectionIdError('Collection metadata "id" is required to record tracking-results, as it identifies the collection in every persisted run. Add an "id" field to the collection metadata file.');
+    }
+
+    const repository = new TrackingResultsRepository(trackingResultsConfig.storage.git);
+    const recorder = new TrackingResultsRecorder({
+      repository,
+      collectionId,
+      schedule: config.get('@opentermsarchive/engine.trackingSchedule'),
+      engineVersion: PACKAGE_VERSION,
+    });
+
+    return new TrackingResults({ recorder });
+  }
+
+  constructor({ recorder }) {
+    super();
+    this.recorder = recorder;
+    this.ready = false;
+  }
+
+  async initialize() {
+    await this.ensureReady(); // Attempted as early as possible so readers of the repository see the previous run finalized without waiting for the next tracking run
+  }
+
+  async finalize() {
+    if (!this.ready) { // Nothing was recorded, and the repository may not even be initialized
+      return;
+    }
+
+    try {
+      await this.recorder.finalize();
+    } catch (error) {
+      this.emit('warn', { message: `Could not finalize the tracking-results repository: ${error.message}; recorded commits are kept locally and the finalization will be retried at the next run` }); // Everything is already committed locally, so a failed push or commit-graph update loses nothing: the next successful finalize pushes all accumulated commits
+    }
+  }
+
+  get hasRunInProgress() {
+    return Boolean(this.recorder.currentRun);
+  }
+
+  get currentRunId() {
+    return this.recorder.currentRun?.runId ?? null;
+  }
+
+  async getDeclarationsCommit() { // Meant to be called right before the declarations are loaded, so that the commit identifies the declarations applied by every run of this process, whatever happens to their repository afterwards
+    try {
+      const declarationsCommit = await declaredServices.getDeclarationsCommit();
+
+      if (!declarationsCommit) { // The audit trail premise (tamper-evident declarations commit) cannot be honoured
+        this.emit('warn', { message: 'The declarations directory is not a Git repository; tracking-results runs will not be recorded' });
+      }
+
+      return declarationsCommit;
+    } catch (error) { // An actual git failure, distinct from "declarations is not a Git repository" which is reported as null
+      this.emit('warn', { message: `Could not read the declarations commit: ${error.message}; tracking-results runs will not be recorded` });
+
+      return null;
+    }
+  }
+
+  async startRun({ services, declarationsCommit, selectedServicesIds, selectedTermsTypes }) {
+    if (!await this.ensureReady()) { // A new Start run commit would hide the crashed run's reference SHA and make its recovery impossible forever
+      return;
+    }
+
+    if (!declarationsCommit) { // Already reported by getDeclarationsCommit
+      return;
+    }
+
+    const servicesIds = Object.keys(services).sort((a, b) => a.localeCompare(b)); // Sorted so the persisted skipped list is deterministic
+
+    try {
+      await this.recorder.startRun({
+        declarationsCommit,
+        servicesCount: servicesIds.length, // Declared counts always cover the full declarations, whatever subset this run processes; coverage.skipped carries the difference so the coverage proof (declared = processed + skipped) stays derivable
+        termsCount: Service.getNumberOfTerms(services, servicesIds, []),
+        skippedTerms: unselectedTerms(services, servicesIds, { selectedServicesIds, selectedTermsTypes }),
+      });
+    } catch (error) {
+      this.emit('warn', { message: `Could not start the tracking-results run: ${error.message}; tracking-results is disabled for this run` }); // Like every other tracking-results failure, a failed run-start commit degrades the audit trail, never the tracking itself
+    }
+  }
+
+  recordSuccess(terms, { transientErrors } = {}) {
+    return this.record(terms, { status: STATUSES.ok, transientErrorReasons: transientErrors?.length ? categorizeReasons(transientErrors) : undefined });
+  }
+
+  recordFailure(terms, errors) {
+    return this.record(terms, { status: STATUSES.failed, reasons: categorizeReasons(errors) });
+  }
+
+  async record(terms, { status, reasons, transientErrorReasons }) {
+    if (!this.hasRunInProgress) { // No tracking-results run is active (technical upgrades, run start skipped or failed); nothing to record
+      return;
+    }
+
+    await this.recorder.recordTermsOutcome({
+      serviceId: terms.service.id,
+      termsType: terms.type,
+      serviceName: terms.service.name,
+      sourceDocuments: terms.sourceDocuments.map(sourceDocument => ({
+        id: sourceDocument.id,
+        ...sourceDocument.toPersistence(),
+        mimeType: normalizeMimeType(sourceDocument.mimeType),
+        snapshotId: sourceDocument.snapshotId ?? null,
+      })),
+      status,
+      reasons,
+      transientErrorReasons,
+    });
+  }
+
+  completeRun() {
+    return this.recorder.completeRun();
+  }
+
+  async ensureReady() { // Returns true when it is safe to write a new run-start commit: the repository is initialized and a pending in_progress run.json has been finalized. Success is memoised, and a failure is retried at the next call so a long-lived scheduled process self-heals without a restart
+    if (this.ready) {
+      return true;
+    }
+
+    try {
+      await this.recorder.initialize(); // Repeated at each attempt, as it also drops what a failed attempt may have left uncommitted
+
+      const recovered = await this.recorder.recoverCrashedRunIfAny({ getDeclaredTermsAtCommit: commit => this.declaredTermsAtCommit(commit) });
+
+      this.ready = true;
+
+      if (recovered) {
+        this.emit('warn', { message: `Recovered crashed run ${recovered.shortRunId}: persisted ${recovered.coverage.processed} processed terms and ${recovered.coverage.skipped.length} skipped terms before the new run starts` });
+      }
+
+      return true;
+    } catch (error) {
+      if (error instanceof UnreadableRunError) { // A stale or incompatible run.json cannot be recovered by construction: proceed, the next startRun will overwrite it with a valid one
+        this.ready = true;
+        this.emit('warn', { message: `Could not read previous tracking-results state: ${error.message}. The previous run.json may be from an incompatible engine version; it will be overwritten by the next run.` });
+
+        return true;
+      }
+
+      this.emit('warn', { message: `Could not prepare the tracking-results repository: ${error.message}; tracking-results is disabled for this run and the preparation will be retried at the next one. If this warning persists across runs, inspect or delete the repository at "${this.recorder.repository.path}" so it can be recreated` }); // An auxiliary audit trail must not prevent tracking itself, be it at initialization
+
+      return false;
+    }
+  }
+
+  async declaredTermsAtCommit(commit) {
+    try {
+      return await declaredServices.getDeclaredTermsAtCommit(commit);
+    } catch (error) {
+      if (error instanceof GitObjectNotFoundError) { // The commit is unreachable forever (shallow clone, rewritten declarations history): fall back to the currently declared terms so the crashed run can still be finalized, and surface the approximation
+        this.emit('warn', { message: `Declarations commit ${commit} is not reachable; crash recovery coverage falls back to the currently declared terms` });
+
+        return declaredServices.getDeclaredTerms();
+      }
+
+      throw error;
+    }
+  }
+}
+
+function normalizeMimeType(mimeType) { // Fetchers report the raw Content-Type header while Git snapshots report the type derived from their file extension; aligned on the latter, so that a parameter or an alias is not recorded as a MIME type change
+  if (!mimeType) {
+    return null;
+  }
+
+  return mime.getType(mime.getExtension(mimeType) || '') || mimeType.split(';')[0].trim().toLowerCase();
+}
+
+function categorizeReasons(errors) { // Tags each reason with [fetch], [extraction] or [internal] so that commit subjects and consumers of the persisted reasons can split by category without sniffing error types
+  return errors.map(error => {
+    if (error instanceof FetchDocumentError) {
+      return `[fetch] ${error.message}`;
+    }
+
+    if (error instanceof ExtractDocumentError) {
+      return `[extraction] ${error.message}`;
+    }
+
+    return '[internal] Unexpected engine error'; // The message of an unexpected error may expose server paths or git output, which must not enter a published history that cannot be rewritten; the details are in the logs
+  });
+}
+
+function unselectedTerms(services, servicesIds, { selectedServicesIds, selectedTermsTypes }) { // Enumerates the declared terms that this run will not process, so partial runs (CLI-filtered services or terms types) record them as explicitly skipped
+  const skipped = [];
+
+  for (const serviceId of servicesIds) {
+    const selectedTypes = new Set(selectedServicesIds.includes(serviceId) ? services[serviceId].getTermsTypes(selectedTermsTypes) : []);
+
+    for (const termsType of services[serviceId].getTermsTypes()) {
+      if (!selectedTypes.has(termsType)) {
+        skipped.push({ serviceId, termsType, reason: 'not selected for this run' });
+      }
+    }
+  }
+
+  return skipped;
+}