@opentermsarchive/engine 9.2.3 → 10.0.0

package/bin/ota-apply-technical-upgrades.js

@@ -0,0 +1,19 @@
+#! /usr/bin/env node
+import './env.js';
+
+import path from 'path';
+import { fileURLToPath, pathToFileURL } from 'url';
+
+import { program } from 'commander';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const { applyTechnicalUpgrades } = await import(pathToFileURL(path.resolve(__dirname, '../src/index.js'))); // load asynchronously to ensure env.js is loaded before
+
+program
+  .name('ota apply-technical-upgrades')
+  .description('Apply technical upgrades by generating new versions from the latest snapshots using updated declarations, engine logic, or dependencies, and by retrieving any missing snapshots for newly added source documents')
+  .option('-s, --services [serviceId...]', 'service IDs to apply technical upgrades to')
+  .option('-t, --types [termsType...]', 'terms types to apply technical upgrades to');
+
+applyTechnicalUpgrades(program.parse(process.argv).opts());

package/bin/ota-track.js

@@ -15,7 +15,6 @@ program
   .description('Retrieve declared documents, record snapshots, extract versions and publish the resulting records')
   .option('-s, --services [serviceId...]', 'service IDs of services to track')
   .option('-t, --types [termsType...]', 'terms types to track')
-  .option('-e, --extract-only', 'extract versions from existing snapshots with latest declarations and engine, without recording new snapshots')
   .option('--schedule', 'track automatically at a regular interval');
 
 track(program.parse(process.argv).opts());

package/bin/ota.js

@@ -11,6 +11,7 @@ program
   .description(description)
   .version(version)
   .command('track', 'Track the current terms of services according to provided declarations')
+  .command('apply-technical-upgrades', 'Apply technical upgrades by generating new versions from the latest snapshots using updated declarations, engine logic, or dependencies')
   .command('validate', 'Run a series of tests to check the validity of terms declarations')
   .command('lint', 'Check format and stylistic errors in declarations and auto fix them')
   .command('dataset', 'Export the versions dataset into a ZIP file and optionally publish it to GitHub releases')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "9.2.3",
+  "version": "10.0.0",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {

package/src/archivist/index.js

@@ -20,7 +20,7 @@ const { version: PACKAGE_VERSION } = require('../../package.json');
 // - too many requests on the same endpoint yield 403
 // - sometimes when creating a commit no SHA are returned for unknown reasons
 const MAX_PARALLEL_TRACKING = 1;
-const MAX_PARALLEL_EXTRACTING = 10;
+const MAX_PARALLEL_TECHNICAL_UPGRADES = 10;
 
 export const EVENTS = [
   'snapshotRecorded',
@@ -128,14 +128,32 @@ export default class Archivist extends events.EventEmitter {
     });
   }
 
-  async track({ services: servicesIds = this.servicesIds, types: termsTypes = [], extractOnly = false } = {}) {
+  async track({ services: servicesIds = this.servicesIds, types: termsTypes = [] } = {}) {
+    await this.processTerms({
+      servicesIds,
+      termsTypes,
+      technicalUpgradeOnly: false,
+      concurrency: MAX_PARALLEL_TRACKING,
+    });
+  }
+
+  async applyTechnicalUpgrades({ services: servicesIds = this.servicesIds, types: termsTypes = [] } = {}) {
+    await this.processTerms({
+      servicesIds,
+      termsTypes,
+      technicalUpgradeOnly: true,
+      concurrency: MAX_PARALLEL_TECHNICAL_UPGRADES,
+    });
+  }
+
+  async processTerms({ servicesIds, termsTypes, technicalUpgradeOnly, concurrency }) {
     const numberOfTerms = Service.getNumberOfTerms(this.services, servicesIds, termsTypes);
 
-    this.emit('trackingStarted', servicesIds.length, numberOfTerms, extractOnly);
+    this.emit('trackingStarted', servicesIds.length, numberOfTerms, technicalUpgradeOnly);
 
     await Promise.all([ launchHeadlessBrowser(), this.recorder.initialize() ]);
 
-    this.trackingQueue.concurrency = extractOnly ? MAX_PARALLEL_EXTRACTING : MAX_PARALLEL_TRACKING;
+    this.trackingQueue.concurrency = concurrency;
 
     servicesIds.forEach(serviceId => {
       this.services[serviceId].getTermsTypes().forEach(termsType => {
@@ -143,7 +161,7 @@ export default class Archivist extends events.EventEmitter {
           return;
         }
 
-        this.trackingQueue.push({ terms: this.services[serviceId].getTerms({ type: termsType }), extractOnly });
+        this.trackingQueue.push({ terms: this.services[serviceId].getTerms({ type: termsType }), technicalUpgradeOnly });
       });
     });
 
@@ -153,12 +171,14 @@ export default class Archivist extends events.EventEmitter {
 
     await Promise.all([ stopHeadlessBrowser(), this.recorder.finalize() ]);
 
-    this.emit('trackingCompleted', servicesIds.length, numberOfTerms, extractOnly);
+    this.emit('trackingCompleted', servicesIds.length, numberOfTerms, technicalUpgradeOnly);
   }
 
-  async trackTermsChanges({ terms, extractOnly = false }) {
-    if (!extractOnly) {
+  async trackTermsChanges({ terms, technicalUpgradeOnly = false }) {
+    if (!technicalUpgradeOnly) {
       await this.fetchAndRecordSnapshots(terms);
+    } else {
+      await this.fetchAndRecordNewSourceDocuments(terms); // In technical upgrade mode, fetch and record snapshots only for new source documents that don't have existing snapshots yet (e.g., when a declaration is updated to add a new source document)
     }
 
     const contents = await this.extractContentsFromSnapshots(terms);
@@ -167,7 +187,7 @@ export default class Archivist extends events.EventEmitter {
       return;
     }
 
-    await this.recordVersion(terms, contents.join(Version.SOURCE_DOCUMENTS_SEPARATOR), extractOnly);
+    await this.recordVersion(terms, contents.join(Version.SOURCE_DOCUMENTS_SEPARATOR), technicalUpgradeOnly);
   }
 
   async fetchAndRecordSnapshots(terms) {
@@ -190,6 +210,50 @@ export default class Archivist extends events.EventEmitter {
     }
   }
 
+  async fetchAndRecordNewSourceDocuments(terms) {
+    if (!terms.hasMultipleSourceDocuments) { // If the terms has only one source document, there is nothing to do
+      return;
+    }
+
+    const existingVersion = await this.recorder.versionsRepository.findLatest(terms.service.id, terms.type);
+
+    if (!existingVersion) { // If the terms does not have a version recorded, skip this step as the next version will be tagged as "First record…" anyway
+      return;
+    }
+
+    const missingSourceDocuments = [];
+
+    for (const sourceDocument of terms.sourceDocuments) {
+      const snapshot = await this.recorder.getLatestSnapshot(terms, sourceDocument.id);
+
+      if (!snapshot) {
+        missingSourceDocuments.push(sourceDocument);
+      }
+    }
+
+    if (!missingSourceDocuments.length) {
+      return;
+    }
+
+    terms.fetchDate = new Date();
+    const fetchDocumentErrors = [];
+
+    for (const sourceDocument of missingSourceDocuments) {
+      const error = await this.fetchSourceDocument(sourceDocument);
+
+      if (error) {
+        fetchDocumentErrors.push(error);
+      } else {
+        await this.recordSnapshot(terms, sourceDocument);
+        sourceDocument.clearContent(); // Reduce memory usage by clearing no longer needed large content strings
+      }
+    }
+
+    if (fetchDocumentErrors.length) {
+      throw new InaccessibleContentError(fetchDocumentErrors);
+    }
+  }
+
   async fetchSourceDocument(sourceDocument) {
     const { location: url, executeClientScripts, cssSelectors } = sourceDocument;
 
@@ -249,14 +313,14 @@ export default class Archivist extends events.EventEmitter {
     return contents;
   }
 
-  async recordVersion(terms, content, extractOnly) {
+  async recordVersion(terms, content, technicalUpgradeOnly) {
     const record = new Version({
       content,
       snapshotIds: terms.sourceDocuments.map(sourceDocuments => sourceDocuments.snapshotId),
       serviceId: terms.service.id,
       termsType: terms.type,
       fetchDate: terms.fetchDate,
-      isExtractOnly: extractOnly,
+      isTechnicalUpgrade: technicalUpgradeOnly,
       metadata: { 'x-engine-version': PACKAGE_VERSION },
     });
 

package/src/archivist/index.test.js

@@ -11,6 +11,7 @@ import sinonChai from 'sinon-chai';
 import { InaccessibleContentError } from './errors.js';
 import { FetchDocumentError } from './fetcher/index.js';
 import Git from './recorder/repositories/git/git.js';
+import SourceDocument from './services/sourceDocument.js';
 
 import Archivist, { EVENTS } from './index.js';
 
@@ -52,6 +53,31 @@ describe('Archivist', function () {
 
   const services = [ 'service·A', 'Service B!' ];
 
+  function setupNockForServices({ serviceA = true, serviceB = true } = {}) {
+    nock.cleanAll();
+    if (serviceA) {
+      nock('https://www.servicea.example')
+        .get('/tos')
+        .reply(200, serviceASnapshotExpectedContent, { 'Content-Type': 'text/html' });
+    }
+    if (serviceB) {
+      nock('https://www.serviceb.example')
+        .get('/privacy')
+        .reply(200, serviceBSnapshotExpectedContent, { 'Content-Type': 'application/pdf' });
+    }
+  }
+
+  async function createAndInitializeArchivist() {
+    const archivist = new Archivist({
+      recorderConfig: config.get('@opentermsarchive/engine.recorder'),
+      fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
+    });
+
+    await archivist.initialize();
+
+    return archivist;
+  }
+
   before(async () => {
     gitVersion = new Git({
       path: VERSIONS_PATH,
@@ -70,13 +96,8 @@ describe('Archivist', function () {
 
   describe('#track', () => {
     before(async () => {
-      nock('https://www.servicea.example').get('/tos').reply(200, serviceASnapshotExpectedContent, { 'Content-Type': 'text/html' });
-      nock('https://www.serviceb.example').get('/privacy').reply(200, serviceBSnapshotExpectedContent, { 'Content-Type': 'application/pdf' });
-      app = new Archivist({
-        recorderConfig: config.get('@opentermsarchive/engine.recorder'),
-        fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
-      });
-      await app.initialize();
+      setupNockForServices();
+      app = await createAndInitializeArchivist();
     });
 
     context('when everything works fine', () => {
@@ -112,8 +133,7 @@ describe('Archivist', function () {
     context('when there is an operational error with service A', () => {
       before(async () => {
         // as there is no more HTTP request mocks for service A, it should throw an `ENOTFOUND` error which is considered as an expected error in our workflow
-        nock.cleanAll();
-        nock('https://www.serviceb.example').get('/privacy').reply(200, serviceBSnapshotExpectedContent, { 'Content-Type': 'application/pdf' });
+        setupNockForServices({ serviceA: false, serviceB: true });
         await app.track({ services });
       });
 
@@ -139,107 +159,353 @@ describe('Archivist', function () {
         expect(resultingTerms).to.equal(serviceBVersionExpectedContent);
       });
     });
+  });
+
+  describe('#applyTechnicalUpgrades', () => {
+    context('when a service’s filter declaration changes', () => {
+      context('when everything works fine', () => {
+        let originalSnapshotId;
+        let firstVersionId;
+        let reExtractedVersionId;
+        let reExtractedVersionMessageBody;
+        let serviceBCommits;
+
+        before(async () => {
+          setupNockForServices();
+          app = await createAndInitializeArchivist();
+          await app.track({ services });
+
+          ({ id: originalSnapshotId } = await app.recorder.snapshotsRepository.findLatest(SERVICE_A_ID, SERVICE_A_TYPE));
+          ({ id: firstVersionId } = await app.recorder.versionsRepository.findLatest(SERVICE_A_ID, SERVICE_A_TYPE));
+
+          serviceBCommits = await gitVersion.log({ file: SERVICE_B_EXPECTED_VERSION_FILE_PATH });
+
+          app.services[SERVICE_A_ID].getTerms({ type: SERVICE_A_TYPE }).sourceDocuments[0].contentSelectors = 'h1';
+
+          await app.applyTechnicalUpgrades({ services: [ 'service·A', 'Service B!' ] });
+
+          const [reExtractedVersionCommit] = await gitVersion.log({ file: SERVICE_A_EXPECTED_VERSION_FILE_PATH });
+
+          reExtractedVersionId = reExtractedVersionCommit.hash;
+          reExtractedVersionMessageBody = reExtractedVersionCommit.body;
+        });
+
+        after(resetGitRepositories);
+
+        it('updates the version of the changed service', async () => {
+          const serviceAContent = await fs.readFile(path.resolve(__dirname, SERVICE_A_EXPECTED_VERSION_FILE_PATH), { encoding: 'utf8' });
+
+          expect(serviceAContent).to.equal('Terms of service with UTF-8 \'çhãràčtęrs"\n========================================');
+        });
+
+        it('generates a new version id', () => {
+          expect(reExtractedVersionId).to.not.equal(firstVersionId);
+        });
+
+        it('mentions the snapshot id in the changelog', () => {
+          expect(reExtractedVersionMessageBody).to.include(originalSnapshotId);
+        });
+
+        it('does not change other services', async () => {
+          const serviceBVersion = await fs.readFile(path.resolve(__dirname, SERVICE_B_EXPECTED_VERSION_FILE_PATH), { encoding: 'utf8' });
+
+          expect(serviceBVersion).to.equal(serviceBVersionExpectedContent);
+        });
+
+        it('does not generate a new id for other services', async () => {
+          const serviceBCommitsAfterExtraction = await gitVersion.log({ file: SERVICE_B_EXPECTED_VERSION_FILE_PATH });
+
+          expect(serviceBCommitsAfterExtraction.map(commit => commit.hash)).to.deep.equal(serviceBCommits.map(commit => commit.hash));
+        });
+      });
+
+      context('when there is an operational error with service A', () => {
+        let inaccessibleContentSpy;
+        let versionNotChangedSpy;
+        let versionB;
+
+        before(async () => {
+          setupNockForServices();
+          app = await createAndInitializeArchivist();
+          await app.track({ services });
+          app.services[SERVICE_A_ID].getTerms({ type: SERVICE_A_TYPE }).sourceDocuments[0].contentSelectors = 'inexistant-selector';
+          inaccessibleContentSpy = sinon.spy();
+          versionNotChangedSpy = sinon.spy();
+          app.on('inaccessibleContent', inaccessibleContentSpy);
+          app.on('versionNotChanged', record => {
+            if (record.serviceId == 'Service B!') {
+              versionB = record;
+            }
+            versionNotChangedSpy(record);
+          });
+          await app.applyTechnicalUpgrades({ services });
+        });
+
+        after(resetGitRepositories);
+
+        it('emits an inaccessibleContent event', () => {
+          expect(inaccessibleContentSpy).to.have.been.called;
+        });
+
+        it('still extracts the terms of other services', () => {
+          expect(versionNotChangedSpy).to.have.been.calledWith(versionB);
+        });
+      });
+
+      describe('with combined source documents', () => {
+        const MULTI_SOURCE_DOCS = {
+          SERVICE_ID: 'service_with_multiple_source_documents_terms',
+          TERMS_TYPE: 'Community Guidelines',
+          BASE_URL: 'https://www.service-with-multiple-source-documents-terms.example',
+          CONTENT: {
+            COMMUNITY_STANDARDS: '<html><body id="main"><h1>Community Standards</h1><p>Community Standards content</p></body></html>',
+            HATE_SPEECH: '<html><body><p>Hate speech content</p><div id="footer">Footer</div></body></html>',
+            VIOLENCE_INCITEMENT: '<html><body><p>Violence incitement content</p><button class="share">Share</button><button class="print">Print</button></body></html>',
+            NEW_POLICY: '<html><body><p>New additional policy</p></body></html>',
+          },
+          PATHS: {
+            COMMUNITY_STANDARDS: '/community-standards',
+            HATE_SPEECH: '/community-standards/hate-speech/',
+            VIOLENCE_INCITEMENT: '/community-standards/violence-incitement/',
+            NEW_POLICY: '/community-standards/new-policy/',
+          },
+          EXPECTED_TEXTS: {
+            COMMUNITY_STANDARDS: 'Community Standards',
+            HATE_SPEECH: 'Hate speech content',
+            VIOLENCE_INCITEMENT: 'Violence incitement content',
+            NEW_POLICY: 'New additional policy',
+          },
+        };
+
+        const { SERVICE_ID, TERMS_TYPE } = MULTI_SOURCE_DOCS;
+
+        function setupNockForMultiSourceDocs(pathKeys) {
+          pathKeys.forEach(pathKey => {
+            nock(MULTI_SOURCE_DOCS.BASE_URL)
+              .persist()
+              .get(MULTI_SOURCE_DOCS.PATHS[pathKey])
+              .reply(200, MULTI_SOURCE_DOCS.CONTENT[pathKey], { 'Content-Type': 'text/html' });
+          });
+        }
+
+        function disableClientScriptsForTerms(terms) {
+          terms.sourceDocuments.forEach(doc => {
+            doc.executeClientScripts = false;
+          });
+        }
 
-    context('extracting only', () => {
-      context('when a service’s filter declaration changes', () => {
-        context('when everything works fine', () => {
-          let originalSnapshotId;
-          let firstVersionId;
-          let reExtractedVersionId;
-          let reExtractedVersionMessageBody;
-          let serviceBCommits;
+        context('when a source document is added to existing combined terms', () => {
+          let initialVersion;
+          let upgradeVersion;
 
           before(async () => {
-            nock('https://www.servicea.example').get('/tos').reply(200, serviceASnapshotExpectedContent, { 'Content-Type': 'text/html' });
-            nock('https://www.serviceb.example').get('/privacy').reply(200, serviceBSnapshotExpectedContent, { 'Content-Type': 'application/pdf' });
-            app = new Archivist({
-              recorderConfig: config.get('@opentermsarchive/engine.recorder'),
-              fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
-            });
+            setupNockForMultiSourceDocs([ 'COMMUNITY_STANDARDS', 'HATE_SPEECH', 'VIOLENCE_INCITEMENT', 'NEW_POLICY' ]);
 
-            await app.initialize();
-            await app.track({ services });
+            app = await createAndInitializeArchivist();
 
-            ({ id: originalSnapshotId } = await app.recorder.snapshotsRepository.findLatest(SERVICE_A_ID, SERVICE_A_TYPE));
-            ({ id: firstVersionId } = await app.recorder.versionsRepository.findLatest(SERVICE_A_ID, SERVICE_A_TYPE));
+            let terms = app.services[SERVICE_ID].getTerms({ type: TERMS_TYPE });
 
-            serviceBCommits = await gitVersion.log({ file: SERVICE_B_EXPECTED_VERSION_FILE_PATH });
+            disableClientScriptsForTerms(terms);
 
-            app.services[SERVICE_A_ID].getTerms({ type: SERVICE_A_TYPE }).sourceDocuments[0].contentSelectors = 'h1';
+            // First, track the terms normally to create initial version
+            await app.track({ services: [SERVICE_ID], types: [TERMS_TYPE] });
+            initialVersion = await app.recorder.versionsRepository.findLatest(SERVICE_ID, TERMS_TYPE);
 
-            await app.track({ services: [ 'service·A', 'Service B!' ], extractOnly: true });
+            // Modify the declaration to add a new source document
+            terms = app.services[SERVICE_ID].getTerms({ type: TERMS_TYPE });
 
-            const [reExtractedVersionCommit] = await gitVersion.log({ file: SERVICE_A_EXPECTED_VERSION_FILE_PATH });
+            terms.sourceDocuments.push(new SourceDocument({
+              id: 'new-policy',
+              location: `${MULTI_SOURCE_DOCS.BASE_URL}${MULTI_SOURCE_DOCS.PATHS.NEW_POLICY}`,
+              contentSelectors: 'body',
+              executeClientScripts: false,
+              filters: [],
+            }));
 
-            reExtractedVersionId = reExtractedVersionCommit.hash;
-            reExtractedVersionMessageBody = reExtractedVersionCommit.body;
+            // Apply technical upgrades
+            await app.applyTechnicalUpgrades({ services: [SERVICE_ID], types: [TERMS_TYPE] });
+            upgradeVersion = await app.recorder.versionsRepository.findLatest(SERVICE_ID, TERMS_TYPE);
           });
 
-          after(resetGitRepositories);
+          after(async () => {
+            await resetGitRepositories();
+            nock.cleanAll();
+          });
 
-          it('updates the version of the changed service', async () => {
-            const serviceAContent = await fs.readFile(path.resolve(__dirname, SERVICE_A_EXPECTED_VERSION_FILE_PATH), { encoding: 'utf8' });
+          it('creates a new version', () => {
+            expect(upgradeVersion.id).to.not.equal(initialVersion.id);
+          });
 
-            expect(serviceAContent).to.equal('Terms of service with UTF-8 \'çhãràčtęrs"\n========================================');
+          it('marks the new version as technical upgrade', () => {
+            expect(upgradeVersion.isTechnicalUpgrade).to.be.true;
           });
 
-          it('generates a new version id', () => {
-            expect(reExtractedVersionId).to.not.equal(firstVersionId);
+          it('fetches and includes the new source document in the version', async () => {
+            const versionContent = await upgradeVersion.content;
+
+            expect(versionContent).to.include(MULTI_SOURCE_DOCS.EXPECTED_TEXTS.NEW_POLICY);
           });
 
-          it('mentions the snapshot id in the changelog', () => {
-            expect(reExtractedVersionMessageBody).to.include(originalSnapshotId);
+          it('includes all source documents in version', async () => {
+            const versionContent = await upgradeVersion.content;
+
+            expect(versionContent).to.include(MULTI_SOURCE_DOCS.EXPECTED_TEXTS.COMMUNITY_STANDARDS);
+            expect(versionContent).to.include(MULTI_SOURCE_DOCS.EXPECTED_TEXTS.HATE_SPEECH);
+            expect(versionContent).to.include(MULTI_SOURCE_DOCS.EXPECTED_TEXTS.VIOLENCE_INCITEMENT);
+            expect(versionContent).to.include(MULTI_SOURCE_DOCS.EXPECTED_TEXTS.NEW_POLICY);
           });
+        });
+
+        context('when a source document location is modified in combined terms', () => {
+          let initialVersion;
+          let latestVersion;
+          let newLocationScope;
+
+          before(async () => {
+            setupNockForMultiSourceDocs([ 'COMMUNITY_STANDARDS', 'HATE_SPEECH', 'VIOLENCE_INCITEMENT' ]);
+
+            app = await createAndInitializeArchivist();
+
+            let terms = app.services[SERVICE_ID].getTerms({ type: TERMS_TYPE });
+
+            disableClientScriptsForTerms(terms);
+
+            // First, track the terms normally
+            await app.track({ services: [SERVICE_ID], types: [TERMS_TYPE] });
+            initialVersion = await app.recorder.versionsRepository.findLatest(SERVICE_ID, TERMS_TYPE);
+
+            // Mock new location (but it won't be fetched during technical upgrade)
+            newLocationScope = nock(MULTI_SOURCE_DOCS.BASE_URL)
+              .persist()
+              .get('/community-standards/hate-speech-updated/')
+              .reply(200, '<html><body><p>Updated hate speech policy</p></body></html>', { 'Content-Type': 'text/html' });
 
-          it('does not change other services', async () => {
-            const serviceBVersion = await fs.readFile(path.resolve(__dirname, SERVICE_B_EXPECTED_VERSION_FILE_PATH), { encoding: 'utf8' });
+            // Modify the declaration to change location
+            terms = app.services[SERVICE_ID].getTerms({ type: TERMS_TYPE });
 
-            expect(serviceBVersion).to.equal(serviceBVersionExpectedContent);
+            terms.sourceDocuments[1].location = `${MULTI_SOURCE_DOCS.BASE_URL}/community-standards/hate-speech-updated/`;
+
+            // Apply technical upgrades
+            await app.applyTechnicalUpgrades({ services: [SERVICE_ID], types: [TERMS_TYPE] });
+            latestVersion = await app.recorder.versionsRepository.findLatest(SERVICE_ID, TERMS_TYPE);
+          });
+
+          after(async () => {
+            await resetGitRepositories();
+            nock.cleanAll();
+          });
+
+          it('does not create a new version', () => {
+            expect(latestVersion.id).to.equal(initialVersion.id);
+          });
+
+          it('does not fetch from new location', () => {
+            expect(newLocationScope.isDone()).to.be.false;
           });
 
-          it('does not generate a new id for other services', async () => {
-            const serviceBCommitsAfterExtraction = await gitVersion.log({ file: SERVICE_B_EXPECTED_VERSION_FILE_PATH });
+          it('does not include content from the new location', async () => {
+            const versionContent = await latestVersion.content;
 
-            expect(serviceBCommitsAfterExtraction.map(commit => commit.hash)).to.deep.equal(serviceBCommits.map(commit => commit.hash));
+            expect(versionContent).to.not.include('Updated hate speech policy');
           });
         });
 
-        context('when there is an operational error with service A', () => {
-          let inaccessibleContentSpy;
-          let versionNotChangedSpy;
-          let versionB;
+        context('when a source document selector is modified in combined terms', () => {
+          let initialVersion;
+          let latestVersion;
+          let initialVersionContent;
+          let upgradeVersionContent;
 
           before(async () => {
-            nock('https://www.servicea.example').get('/tos').reply(200, serviceASnapshotExpectedContent, { 'Content-Type': 'text/html' });
-            nock('https://www.serviceb.example').get('/privacy').reply(200, serviceBSnapshotExpectedContent, { 'Content-Type': 'application/pdf' });
-            app = new Archivist({
-              recorderConfig: config.get('@opentermsarchive/engine.recorder'),
-              fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
-            });
+            setupNockForMultiSourceDocs([ 'COMMUNITY_STANDARDS', 'HATE_SPEECH', 'VIOLENCE_INCITEMENT' ]);
+
+            app = await createAndInitializeArchivist();
+
+            let terms = app.services[SERVICE_ID].getTerms({ type: TERMS_TYPE });
+
+            disableClientScriptsForTerms(terms);
+
+            // First, track the terms normally
+            await app.track({ services: [SERVICE_ID], types: [TERMS_TYPE] });
+            initialVersion = await app.recorder.versionsRepository.findLatest(SERVICE_ID, TERMS_TYPE);
+            initialVersionContent = await initialVersion.content;
+
+            // Modify the declaration to change selector
+            terms = app.services[SERVICE_ID].getTerms({ type: TERMS_TYPE });
 
-            await app.initialize();
-            await app.track({ services });
-            app.services[SERVICE_A_ID].getTerms({ type: SERVICE_A_TYPE }).sourceDocuments[0].contentSelectors = 'inexistant-selector';
-            inaccessibleContentSpy = sinon.spy();
-            versionNotChangedSpy = sinon.spy();
-            app.on('inaccessibleContent', inaccessibleContentSpy);
-            app.on('versionNotChanged', record => {
-              if (record.serviceId == 'Service B!') {
-                versionB = record;
-              }
-              versionNotChangedSpy(record);
+            // Change from 'body' to 'h1' for the first source document
+            terms.sourceDocuments[0].contentSelectors = 'h1';
+
+            // Apply technical upgrades
+            await app.applyTechnicalUpgrades({ services: [SERVICE_ID], types: [TERMS_TYPE] });
+            latestVersion = await app.recorder.versionsRepository.findLatest(SERVICE_ID, TERMS_TYPE);
+            upgradeVersionContent = await latestVersion.content;
+          });
+
+          after(async () => {
+            await resetGitRepositories();
+            nock.cleanAll();
+          });
+
+          it('creates a new version', () => {
+            expect(latestVersion.id).to.not.equal(initialVersion.id);
+          });
+
+          it('marks the new version as technical upgrade', () => {
+            expect(latestVersion.isTechnicalUpgrade).to.be.true;
+          });
+
+          it('extracts content with the new selector from existing snapshot', () => {
+            // With new selector 'h1', should only extract the heading
+            expect(upgradeVersionContent).to.include(MULTI_SOURCE_DOCS.EXPECTED_TEXTS.COMMUNITY_STANDARDS);
+            // The rest should be from other source documents
+            expect(upgradeVersionContent).to.include(MULTI_SOURCE_DOCS.EXPECTED_TEXTS.HATE_SPEECH);
+            expect(upgradeVersionContent).to.include(MULTI_SOURCE_DOCS.EXPECTED_TEXTS.VIOLENCE_INCITEMENT);
+          });
+
+          it('regenerates version with updated extraction logic', () => {
+            expect(upgradeVersionContent).to.not.equal(initialVersionContent);
+          });
+        });
+
+        context('when adding source document but no version exists yet', () => {
+          let newSourceScope;
+
+          before(async () => {
+            newSourceScope = nock(MULTI_SOURCE_DOCS.BASE_URL)
+              .get(MULTI_SOURCE_DOCS.PATHS.NEW_POLICY)
+              .reply(200, MULTI_SOURCE_DOCS.CONTENT.NEW_POLICY, { 'Content-Type': 'text/html' });
+
+            app = await createAndInitializeArchivist();
+
+            // Modify declaration before any tracking
+            const terms = app.services[SERVICE_ID].getTerms({ type: TERMS_TYPE });
+
+            terms.sourceDocuments.push({
+              id: 'new-policy',
+              location: `${MULTI_SOURCE_DOCS.BASE_URL}${MULTI_SOURCE_DOCS.PATHS.NEW_POLICY}`,
+              contentSelectors: 'body',
+              executeClientScripts: false,
+              filters: [],
             });
-            await app.track({ services, extractOnly: true });
+
+            // Apply technical upgrades (should skip because no version exists)
+            await app.applyTechnicalUpgrades({ services: [SERVICE_ID], types: [TERMS_TYPE] });
           });
 
-          after(resetGitRepositories);
+          after(async () => {
+            await resetGitRepositories();
+            nock.cleanAll();
+          });
 
-          it('emits an inaccessibleContent event', () => {
-            expect(inaccessibleContentSpy).to.have.been.called;
+          it('does not create a version when none existed before', async () => {
+            const version = await app.recorder.versionsRepository.findLatest(SERVICE_ID, TERMS_TYPE);
+
+            expect(version).to.be.null;
           });
 
-          it('still extracts the terms of other services', () => {
-            expect(versionNotChangedSpy).to.have.been.calledWith(versionB);
+          it('does not fetch the new source document', () => {
+            expect(newSourceScope.isDone()).to.be.false;
           });
         });
       });
@@ -256,11 +522,7 @@ describe('Archivist', function () {
     const retryableError = new FetchDocumentError(FetchDocumentError.LIKELY_TRANSIENT_ERRORS[0]);
 
     before(async () => {
-      app = new Archivist({
-        recorderConfig: config.get('@opentermsarchive/engine.recorder'),
-        fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
-      });
-      await app.initialize();
+      app = await createAndInitializeArchivist();
     });
 
     beforeEach(() => {
@@ -345,11 +607,7 @@ describe('Archivist', function () {
 
     describe('#attach', () => {
       before(async () => {
-        app = new Archivist({
-          recorderConfig: config.get('@opentermsarchive/engine.recorder'),
-          fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
-        });
-        await app.initialize();
+        app = await createAndInitializeArchivist();
 
         EVENTS.forEach(event => {
           const handlerName = `on${event[0].toUpperCase()}${event.substring(1)}`;
@@ -378,14 +636,9 @@ describe('Archivist', function () {
       let plugin;
 
       before(async () => {
-        nock.cleanAll();
-        nock('https://www.servicea.example').get('/tos').reply(200, serviceASnapshotExpectedContent, { 'Content-Type': 'text/html' });
+        setupNockForServices({ serviceA: true, serviceB: false });
 
-        app = new Archivist({
-          recorderConfig: config.get('@opentermsarchive/engine.recorder'),
-          fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
-        });
-        await app.initialize();
+        app = await createAndInitializeArchivist();
 
         plugin = { onFirstVersionRecorded: () => { throw new Error('Plugin error'); } };
 
@@ -432,11 +685,7 @@ describe('Archivist', function () {
     }
 
     before(async () => {
-      app = new Archivist({
-        recorderConfig: config.get('@opentermsarchive/engine.recorder'),
-        fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
-      });
-      await app.initialize();
+      app = await createAndInitializeArchivist();
 
       EVENTS.forEach(event => {
         const handlerName = `on${event[0].toUpperCase()}${event.substr(1)}`;

package/src/archivist/recorder/index.js

@@ -12,11 +12,8 @@ export default class Recorder {
     return Promise.all([ this.versionsRepository.initialize(), this.snapshotsRepository.initialize() ]);
   }
 
-  async finalize() {
-    // Close repositories sequentially to avoid race conditions when both repositories  use the same MongoDB connection (same server/database).
-    // Parallel closing can cause "Operation interrupted because client was closed" errors, especially on Windows.
-    await this.versionsRepository.finalize();
-    await this.snapshotsRepository.finalize();
+  finalize() {
+    return Promise.all([ this.versionsRepository.finalize(), this.snapshotsRepository.finalize() ]);
   }
 
   getLatestSnapshot(terms, sourceDocumentId) {

package/src/archivist/recorder/index.test.js

@@ -6,6 +6,8 @@ import Version from './version.js';
 
 import Recorder from './index.js';
 
+const isWindows = process.platform === 'win32';
+
 const MIME_TYPE = 'text/html';
 const FETCH_DATE = new Date('2000-01-01T12:00:00.000Z');
 const FETCH_DATE_LATER = new Date('2000-01-02T12:00:00.000Z');
@@ -18,7 +20,14 @@ describe('Recorder', () => {
     describe(repositoryType, () => {
       let recorder;
 
-      before(async () => {
+      before(async function () {
+        if (repositoryType == 'mongo' && isWindows) {
+          console.log('MongoDB tests are unstable on Windows due to race condition in connection cleanup.');
+          console.log('Lacking a production use case for Mongo on Windows, we skip tests. Please reach out if you have a use case.');
+          // On Windows, when multiple repositories connect to the same MongoDB server and are closed in parallel or even sequentially, unhandled "Operation interrupted because client was closed" errors occur after all tests pass.
+          // The issue does not occur on Linux or macOS, so it appears to be a platform-specific difference in how the MongoDB driver handles connection pool cleanup during client.close().
+          this.skip();
+        }
         const options = config.util.cloneDeep(config.get('@opentermsarchive/engine.recorder'));
 
         options.versions.storage.type = repositoryType;
@@ -28,7 +37,7 @@ describe('Recorder', () => {
         await recorder.initialize();
       });
 
-      after(() => recorder.finalize());
+      after(() => recorder?.finalize());
 
       context('Snapshot', () => {
         describe('#record', () => {
@@ -258,8 +267,8 @@ describe('Recorder', () => {
               expect(await record.content).to.equal(UPDATED_CONTENT);
             });
 
-            it('records in the version that it is not an extracted only version', () => {
-              expect(record.isExtractOnly).to.equal(false);
+            it('records in the version that it is not a technical upgrade version', () => {
+              expect(record.isTechnicalUpgrade).to.equal(false);
             });
 
             it('returns the record id', () => {
@@ -315,7 +324,7 @@ describe('Recorder', () => {
                   content: CONTENT,
                   snapshotIds: [SNAPSHOT_ID],
                   fetchDate: FETCH_DATE,
-                  isExtractOnly: true,
+                  isTechnicalUpgrade: true,
                 })));
 
                 record = await recorder.versionsRepository.findLatest(SERVICE_ID, TYPE);
@@ -354,7 +363,7 @@ describe('Recorder', () => {
                   content: UPDATED_CONTENT,
                   snapshotIds: [SNAPSHOT_ID],
                   fetchDate: FETCH_DATE_LATER,
-                  isExtractOnly: true,
+                  isTechnicalUpgrade: true,
                 })));
 
                 record = await recorder.versionsRepository.findLatest(SERVICE_ID, TYPE);
@@ -366,8 +375,8 @@ describe('Recorder', () => {
                 expect(await record.content).to.equal(UPDATED_CONTENT);
               });
 
-              it('records in the version that it is an extracted only version', () => {
-                expect(record.isExtractOnly).to.equal(true);
+              it('records in the version that it is an technical upgrade version', () => {
+                expect(record.isTechnicalUpgrade).to.equal(true);
               });
 
               it('returns the record id', () => {
@@ -395,7 +404,7 @@ describe('Recorder', () => {
                   content: CONTENT,
                   snapshotIds: [SNAPSHOT_ID],
                   fetchDate: FETCH_DATE_LATER,
-                  isExtractOnly: true,
+                  isTechnicalUpgrade: true,
                 })));
 
                 record = await recorder.versionsRepository.findLatest(SERVICE_ID, TYPE);

package/src/archivist/recorder/repositories/git/dataMapper.js

@@ -7,7 +7,7 @@ import Version from '../../version.js';
 
 export const COMMIT_MESSAGE_PREFIXES = {
   startTracking: 'First record of',
-  extractOnly: 'Apply technical or declaration upgrade on',
+  technicalUpgrade: 'Apply technical or declaration upgrade on',
   update: 'Record new changes of',
   deprecated_startTracking: 'Start tracking',
   deprecated_refilter: 'Refilter',
@@ -22,9 +22,9 @@ const MULTIPLE_SOURCE_DOCUMENTS_PREFIX = 'This version was recorded after extrac
 export const COMMIT_MESSAGE_PREFIXES_REGEXP = new RegExp(`^(${Object.values(COMMIT_MESSAGE_PREFIXES).join('|')})`);
 
 export function toPersistence(record, snapshotIdentiferTemplate) {
-  const { serviceId, termsType, documentId, isExtractOnly, snapshotIds = [], mimeType, isFirstRecord, metadata } = record;
+  const { serviceId, termsType, documentId, isTechnicalUpgrade, snapshotIds = [], mimeType, isFirstRecord, metadata } = record;
 
-  let prefix = isExtractOnly ? COMMIT_MESSAGE_PREFIXES.extractOnly : COMMIT_MESSAGE_PREFIXES.update;
+  let prefix = isTechnicalUpgrade ? COMMIT_MESSAGE_PREFIXES.technicalUpgrade : COMMIT_MESSAGE_PREFIXES.update;
 
   prefix = isFirstRecord ? COMMIT_MESSAGE_PREFIXES.startTracking : prefix;
 
@@ -75,7 +75,7 @@ export function toDomain(commit) {
   const mimeTypeValue = mime.getType(relativeFilePath);
 
   if (mimeTypeValue == mime.getType('markdown')) {
-    attributes.isExtractOnly = message.startsWith(COMMIT_MESSAGE_PREFIXES.extractOnly) || message.startsWith(COMMIT_MESSAGE_PREFIXES.deprecated_refilter);
+    attributes.isTechnicalUpgrade = message.startsWith(COMMIT_MESSAGE_PREFIXES.technicalUpgrade) || message.startsWith(COMMIT_MESSAGE_PREFIXES.deprecated_refilter);
     attributes.snapshotIds = snapshotIdsMatch;
 
     return new Version(attributes);

package/src/archivist/recorder/repositories/git/index.test.js

@@ -208,7 +208,7 @@ describe('GitRepository', () => {
         });
       });
 
-      context('when it is an extracted only version', () => {
+      context('when it is an technical upgrade version', () => {
         const EXTRACTED_ONLY_CONTENT = `${CONTENT} extracted only`;
 
         before(async () => {
@@ -217,7 +217,7 @@ describe('GitRepository', () => {
             termsType: TERMS_TYPE,
             content: CONTENT,
             fetchDate: FETCH_DATE_EARLIER,
-          })); // An extracted only version cannot be the first record
+          })); // An technical upgrade version cannot be the first record
 
           numberOfRecordsBefore = (await git.log()).length;
 
@@ -226,7 +226,7 @@ describe('GitRepository', () => {
             termsType: TERMS_TYPE,
             content: EXTRACTED_ONLY_CONTENT,
             fetchDate: FETCH_DATE,
-            isExtractOnly: true,
+            isTechnicalUpgrade: true,
             snapshotIds: [SNAPSHOT_ID],
           })));
 
@@ -245,8 +245,8 @@ describe('GitRepository', () => {
           expect(commit.hash).to.include(id);
         });
 
-        it('stores information that it is an extracted only version', () => {
-          expect(commit.message).to.include(COMMIT_MESSAGE_PREFIXES.extractOnly);
+        it('stores information that it is an technical upgrade version', () => {
+          expect(commit.message).to.include(COMMIT_MESSAGE_PREFIXES.technicalUpgrade);
         });
       });
 
@@ -518,7 +518,7 @@ describe('GitRepository', () => {
           serviceId: SERVICE_PROVIDER_ID,
           termsType: TERMS_TYPE,
           content: `${CONTENT} - updated 2`,
-          isExtractOnly: true,
+          isTechnicalUpgrade: true,
           fetchDate: FETCH_DATE_EARLIER,
           snapshotIds: [SNAPSHOT_ID],
         }));
@@ -569,7 +569,7 @@ describe('GitRepository', () => {
           serviceId: SERVICE_PROVIDER_ID,
           termsType: TERMS_TYPE,
           content: `${CONTENT} - updated 2`,
-          isExtractOnly: true,
+          isTechnicalUpgrade: true,
           fetchDate: FETCH_DATE_EARLIER,
           snapshotIds: [SNAPSHOT_ID],
         }));
@@ -678,7 +678,7 @@ describe('GitRepository', () => {
           serviceId: SERVICE_PROVIDER_ID,
           termsType: TERMS_TYPE,
           content: `${CONTENT} - updated 2`,
-          isExtractOnly: true,
+          isTechnicalUpgrade: true,
           fetchDate: FETCH_DATE_EARLIER,
           snapshotIds: [SNAPSHOT_ID],
           mimeType: HTML_MIME_TYPE,
@@ -1079,7 +1079,7 @@ describe('GitRepository', () => {
           serviceId: SERVICE_PROVIDER_ID,
           termsType: TERMS_TYPE,
           content: `${CONTENT} - updated 2`,
-          isExtractOnly: true,
+          isTechnicalUpgrade: true,
           fetchDate: FETCH_DATE_EARLIER,
           mimeType: HTML_MIME_TYPE,
         }));
@@ -1130,7 +1130,7 @@ describe('GitRepository', () => {
           serviceId: SERVICE_PROVIDER_ID,
           termsType: TERMS_TYPE,
           content: `${CONTENT} - updated 2`,
-          isExtractOnly: true,
+          isTechnicalUpgrade: true,
           fetchDate: FETCH_DATE_EARLIER,
           mimeType: HTML_MIME_TYPE,
         }));
@@ -1269,7 +1269,7 @@ describe('GitRepository', () => {
           serviceId: SERVICE_PROVIDER_ID,
           termsType: TERMS_TYPE,
           content: `${CONTENT} - updated 2`,
-          isExtractOnly: true,
+          isTechnicalUpgrade: true,
           fetchDate: FETCH_DATE_EARLIER,
           mimeType: HTML_MIME_TYPE,
         }));
@@ -1398,24 +1398,24 @@ describe('GitRepository', () => {
     after(() => subject.removeAll());
 
     describe('Records attributes', () => {
-      describe('#isExtractOnly', () => {
+      describe('#isTechnicalUpgrade', () => {
         context('records with deprecated message', () => {
           it('returns the proper value', async () => {
-            expect((await subject.findById(commits.deprecatedRefilter.id)).isExtractOnly).to.be.true;
+            expect((await subject.findById(commits.deprecatedRefilter.id)).isTechnicalUpgrade).to.be.true;
           });
 
           it('returns the proper value', async () => {
-            expect((await subject.findById(commits.deprecatedFirstRecord.id)).isExtractOnly).to.be.false;
+            expect((await subject.findById(commits.deprecatedFirstRecord.id)).isTechnicalUpgrade).to.be.false;
           });
         });
 
         context('record with current message', () => {
           it('returns the proper value', async () => {
-            expect((await subject.findById(commits.currentExtractOnly.id)).isExtractOnly).to.be.true;
+            expect((await subject.findById(commits.currentExtractOnly.id)).isTechnicalUpgrade).to.be.true;
           });
 
           it('returns the proper value', async () => {
-            expect((await subject.findById(commits.currentFirstRecord.id)).isExtractOnly).to.be.false;
+            expect((await subject.findById(commits.currentFirstRecord.id)).isTechnicalUpgrade).to.be.false;
           });
         });
       });

package/src/archivist/recorder/repositories/mongo/dataMapper.js

@@ -17,7 +17,7 @@ export function toPersistence(record) {
 }
 
 export function toDomain(mongoDocument) {
-  const { _id, serviceId, termsType, documentId, fetchDate, mimeType, isExtractOnly, isRefilter, isFirstRecord, snapshotIds, metadata } = mongoDocument;
+  const { _id, serviceId, termsType, documentId, fetchDate, mimeType, isTechnicalUpgrade, isExtractOnly, isRefilter, isFirstRecord, snapshotIds, metadata } = mongoDocument;
 
   const attributes = {
     id: _id.toString(),
@@ -27,7 +27,7 @@ export function toDomain(mongoDocument) {
     mimeType,
     fetchDate: new Date(fetchDate),
     isFirstRecord: Boolean(isFirstRecord),
-    isExtractOnly: Boolean(isExtractOnly) || Boolean(isRefilter),
+    isTechnicalUpgrade: Boolean(isTechnicalUpgrade) || Boolean(isExtractOnly) || Boolean(isRefilter),
     snapshotIds: snapshotIds?.map(snapshotId => snapshotId.toString()) || [],
     metadata,
   };

package/src/archivist/recorder/repositories/mongo/index.test.js

@@ -16,6 +16,7 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 const { connectionURI } = config.get('@opentermsarchive/engine.recorder.snapshots.storage.mongo');
 const client = new MongoClient(connectionURI);
+const isWindows = process.platform === 'win32';
 
 const SERVICE_PROVIDER_ID = 'test_service';
 const TERMS_TYPE = 'Terms of Service';
@@ -41,6 +42,16 @@ const METADATA = {
 let collection;
 
 describe('MongoRepository', () => {
+  before(function () {
+    if (isWindows) {
+      console.log('MongoDB tests are unstable on Windows due to race condition in connection cleanup.');
+      console.log('Lacking a production use case for Mongo on Windows, we skip tests. Please reach out if you have a use case.');
+      // On Windows, when multiple repositories connect to the same MongoDB server and are closed in parallel or even sequentially, unhandled "Operation interrupted because client was closed" errors occur after all tests pass.
+      // The issue does not occur on Linux or macOS, so it appears to be a platform-specific difference in how the MongoDB driver handles connection pool cleanup during client.close().
+      this.skip();
+    }
+  });
+
   let subject;
 
   context('Version', () => {
@@ -220,7 +231,7 @@ describe('MongoRepository', () => {
         });
       });
 
-      context('when it is an extracted only version', () => {
+      context('when it is an technical upgrade version', () => {
         const EXTRACTED_ONLY_CONTENT = `${CONTENT} extracted only`;
 
         before(async () => {
@@ -230,7 +241,7 @@ describe('MongoRepository', () => {
             content: CONTENT,
             fetchDate: FETCH_DATE_EARLIER,
             snapshotIds: [SNAPSHOT_ID],
-          })); // An extracted only version cannot be the first record
+          })); // An technical upgrade version cannot be the first record
 
           numberOfRecordsBefore = await collection.countDocuments({
             serviceId: SERVICE_PROVIDER_ID,
@@ -243,7 +254,7 @@ describe('MongoRepository', () => {
             content: EXTRACTED_ONLY_CONTENT,
             fetchDate: FETCH_DATE,
             snapshotIds: [SNAPSHOT_ID],
-            isExtractOnly: true,
+            isTechnicalUpgrade: true,
           })));
 
           numberOfRecordsAfter = await collection.countDocuments({
@@ -267,8 +278,8 @@ describe('MongoRepository', () => {
           expect(mongoDocument._id.toString()).to.equal(record.id);
         });
 
-        it('stores information that it is an extracted only version', () => {
-          expect(mongoDocument.isExtractOnly).to.be.true;
+        it('stores information that it is an technical upgrade version', () => {
+          expect(mongoDocument.isTechnicalUpgrade).to.be.true;
         });
       });
 
@@ -596,7 +607,7 @@ describe('MongoRepository', () => {
           serviceId: SERVICE_PROVIDER_ID,
           termsType: TERMS_TYPE,
           content: `${CONTENT} - updated 2`,
-          isExtractOnly: true,
+          isTechnicalUpgrade: true,
           fetchDate: FETCH_DATE_EARLIER,
           snapshotIds: [SNAPSHOT_ID],
         }));
@@ -645,7 +656,7 @@ describe('MongoRepository', () => {
           serviceId: SERVICE_PROVIDER_ID,
           termsType: TERMS_TYPE,
           content: `${CONTENT} - updated 2`,
-          isExtractOnly: true,
+          isTechnicalUpgrade: true,
           fetchDate: FETCH_DATE_EARLIER,
           snapshotIds: [SNAPSHOT_ID],
         }));
@@ -810,7 +821,7 @@ describe('MongoRepository', () => {
           serviceId: SERVICE_PROVIDER_ID,
           termsType: TERMS_TYPE,
           content: `${CONTENT} - updated 2`,
-          isExtractOnly: true,
+          isTechnicalUpgrade: true,
           fetchDate: FETCH_DATE_EARLIER,
           snapshotIds: [SNAPSHOT_ID],
         }));
@@ -1164,7 +1175,7 @@ describe('MongoRepository', () => {
           serviceId: SERVICE_PROVIDER_ID,
           termsType: TERMS_TYPE,
           content: `${CONTENT} - updated 2`,
-          isExtractOnly: true,
+          isTechnicalUpgrade: true,
           fetchDate: FETCH_DATE_EARLIER,
           mimeType: HTML_MIME_TYPE,
         }));
@@ -1213,7 +1224,7 @@ describe('MongoRepository', () => {
           serviceId: SERVICE_PROVIDER_ID,
           termsType: TERMS_TYPE,
           content: `${CONTENT} - updated 2`,
-          isExtractOnly: true,
+          isTechnicalUpgrade: true,
           fetchDate: FETCH_DATE_EARLIER,
           mimeType: HTML_MIME_TYPE,
         }));
@@ -1421,7 +1432,7 @@ describe('MongoRepository', () => {
           serviceId: SERVICE_PROVIDER_ID,
           termsType: TERMS_TYPE,
           content: `${CONTENT} - updated 2`,
-          isExtractOnly: true,
+          isTechnicalUpgrade: true,
           fetchDate: FETCH_DATE_EARLIER,
           mimeType: HTML_MIME_TYPE,
         }));

package/src/index.js

@@ -13,7 +13,7 @@ import Reporter from './reporter/index.js';
 const require = createRequire(import.meta.url);
 const { version: PACKAGE_VERSION } = require('../package.json');
 
-export default async function track({ services, types, extractOnly, schedule }) {
+async function initialize(services) {
   const archivist = new Archivist({
     recorderConfig: config.get('@opentermsarchive/engine.recorder'),
     fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
@@ -40,13 +40,17 @@ export default async function track({ services, types, extractOnly, schedule })
     });
   }
 
-  // The result of the extraction step that generates the version from the snapshots may depend on changes to the engine or its dependencies.
-  // The process thus starts by only performing the extraction process so that any version following such changes can be labelled (to avoid sending notifications, for example)
-  await archivist.track({ services, types, extractOnly: true });
+  return { archivist, services };
+}
 
-  if (extractOnly) {
-    return;
-  }
+export default async function track({ services, types, schedule }) {
+  const { archivist, services: filteredServices } = await initialize(services);
+
+  // Technical upgrade pass: apply changes from engine, dependency, or declaration upgrades.
+  // This regenerates versions from existing snapshots with updated extraction logic.
+  // For terms with combined source documents, if a new document was added to the declaration, it will be fetched and combined with existing snapshots to regenerate the complete version.
+  // All versions from this pass are labeled as technical upgrades to avoid false notifications about content changes.
+  await archivist.applyTechnicalUpgrades({ services: filteredServices, types });
 
   if (process.env.OTA_ENGINE_SENDINBLUE_API_KEY) {
     try {
@@ -72,7 +76,7 @@ export default async function track({ services, types, extractOnly, schedule })
   }
 
   if (!schedule) {
-    await archivist.track({ services, types });
+    await archivist.track({ services: filteredServices, types });
 
     return;
   }
@@ -86,6 +90,12 @@ export default async function track({ services, types, extractOnly, schedule })
   new Cron( // eslint-disable-line no-new
     trackingSchedule,
     { protect: job => logger.warn(`Tracking scheduled at ${new Date().toISOString()} were blocked by an unfinished tracking started at ${job.currentRun().toISOString()}`) },
-    () => archivist.track({ services, types }),
+    () => archivist.track({ services: filteredServices, types }),
   );
 }
+
+export async function applyTechnicalUpgrades({ services, types }) {
+  const { archivist, services: filteredServices } = await initialize(services);
+
+  await archivist.applyTechnicalUpgrades({ services: filteredServices, types });
+}

package/src/logger/index.js

@@ -195,9 +195,9 @@ logger.onVersionNotChanged = ({ serviceId, termsType }) => {
   logger.info({ message: 'No changes after filtering, did not record version', serviceId, termsType });
 };
 
-logger.onTrackingStarted = (numberOfServices, numberOfTerms, extractOnly) => {
-  if (extractOnly) {
-    logger.info(`Examining ${numberOfTerms} terms from ${numberOfServices} services for extraction…`);
+logger.onTrackingStarted = (numberOfServices, numberOfTerms, technicalUpgradeOnly) => {
+  if (technicalUpgradeOnly) {
+    logger.info(`Applying technical upgrades to ${numberOfTerms} terms from ${numberOfServices} services…`);
   } else {
     logger.info(`Tracking changes of ${numberOfTerms} terms from ${numberOfServices} services…`);
   }
@@ -206,11 +206,11 @@ logger.onTrackingStarted = (numberOfServices, numberOfTerms, extractOnly) => {
   trackingStartTime = Date.now();
 };
 
-logger.onTrackingCompleted = (numberOfServices, numberOfTerms, extractOnly) => {
+logger.onTrackingCompleted = (numberOfServices, numberOfTerms, technicalUpgradeOnly) => {
   const duration = formatDuration(Date.now() - trackingStartTime);
 
-  if (extractOnly) {
-    logger.info(`Examined ${numberOfTerms} terms from ${numberOfServices} services for extraction in ${duration}`);
+  if (technicalUpgradeOnly) {
+    logger.info(`Applied technical upgrades to ${numberOfTerms} terms from ${numberOfServices} services in ${duration}`);
     logger.info(`Recorded ${recordedVersionsCount} new versions\n`);
   } else {
     logger.info(`Tracked changes of ${numberOfTerms} terms from ${numberOfServices} services in ${duration}`);