@opentermsarchive/engine 9.2.3 → 10.1.1

package/src/archivist/recorder/repositories/git/dataMapper.js

@@ -7,7 +7,7 @@ import Version from '../../version.js';
 
 export const COMMIT_MESSAGE_PREFIXES = {
   startTracking: 'First record of',
-  extractOnly: 'Apply technical or declaration upgrade on',
+  technicalUpgrade: 'Apply technical or declaration upgrade on',
   update: 'Record new changes of',
   deprecated_startTracking: 'Start tracking',
   deprecated_refilter: 'Refilter',
@@ -22,9 +22,9 @@ const MULTIPLE_SOURCE_DOCUMENTS_PREFIX = 'This version was recorded after extrac
 export const COMMIT_MESSAGE_PREFIXES_REGEXP = new RegExp(`^(${Object.values(COMMIT_MESSAGE_PREFIXES).join('|')})`);
 
 export function toPersistence(record, snapshotIdentiferTemplate) {
-  const { serviceId, termsType, documentId, isExtractOnly, snapshotIds = [], mimeType, isFirstRecord, metadata } = record;
+  const { serviceId, termsType, documentId, isTechnicalUpgrade, snapshotIds = [], mimeType, isFirstRecord, metadata } = record;
 
-  let prefix = isExtractOnly ? COMMIT_MESSAGE_PREFIXES.extractOnly : COMMIT_MESSAGE_PREFIXES.update;
+  let prefix = isTechnicalUpgrade ? COMMIT_MESSAGE_PREFIXES.technicalUpgrade : COMMIT_MESSAGE_PREFIXES.update;
 
   prefix = isFirstRecord ? COMMIT_MESSAGE_PREFIXES.startTracking : prefix;
 
@@ -75,7 +75,7 @@ export function toDomain(commit) {
   const mimeTypeValue = mime.getType(relativeFilePath);
 
   if (mimeTypeValue == mime.getType('markdown')) {
-    attributes.isExtractOnly = message.startsWith(COMMIT_MESSAGE_PREFIXES.extractOnly) || message.startsWith(COMMIT_MESSAGE_PREFIXES.deprecated_refilter);
+    attributes.isTechnicalUpgrade = message.startsWith(COMMIT_MESSAGE_PREFIXES.technicalUpgrade) || message.startsWith(COMMIT_MESSAGE_PREFIXES.deprecated_refilter);
     attributes.snapshotIds = snapshotIdsMatch;
 
     return new Version(attributes);

package/src/archivist/recorder/repositories/git/index.test.js

@@ -208,7 +208,7 @@ describe('GitRepository', () => {
         });
       });
 
-      context('when it is an extracted only version', () => {
+      context('when it is an technical upgrade version', () => {
         const EXTRACTED_ONLY_CONTENT = `${CONTENT} extracted only`;
 
         before(async () => {
@@ -217,7 +217,7 @@ describe('GitRepository', () => {
             termsType: TERMS_TYPE,
             content: CONTENT,
             fetchDate: FETCH_DATE_EARLIER,
-          })); // An extracted only version cannot be the first record
+          })); // An technical upgrade version cannot be the first record
 
           numberOfRecordsBefore = (await git.log()).length;
 
@@ -226,7 +226,7 @@ describe('GitRepository', () => {
             termsType: TERMS_TYPE,
             content: EXTRACTED_ONLY_CONTENT,
             fetchDate: FETCH_DATE,
-            isExtractOnly: true,
+            isTechnicalUpgrade: true,
             snapshotIds: [SNAPSHOT_ID],
           })));
 
@@ -245,8 +245,8 @@ describe('GitRepository', () => {
           expect(commit.hash).to.include(id);
         });
 
-        it('stores information that it is an extracted only version', () => {
-          expect(commit.message).to.include(COMMIT_MESSAGE_PREFIXES.extractOnly);
+        it('stores information that it is an technical upgrade version', () => {
+          expect(commit.message).to.include(COMMIT_MESSAGE_PREFIXES.technicalUpgrade);
         });
       });
 
@@ -518,7 +518,7 @@ describe('GitRepository', () => {
           serviceId: SERVICE_PROVIDER_ID,
           termsType: TERMS_TYPE,
           content: `${CONTENT} - updated 2`,
-          isExtractOnly: true,
+          isTechnicalUpgrade: true,
           fetchDate: FETCH_DATE_EARLIER,
           snapshotIds: [SNAPSHOT_ID],
         }));
@@ -569,7 +569,7 @@ describe('GitRepository', () => {
           serviceId: SERVICE_PROVIDER_ID,
           termsType: TERMS_TYPE,
           content: `${CONTENT} - updated 2`,
-          isExtractOnly: true,
+          isTechnicalUpgrade: true,
           fetchDate: FETCH_DATE_EARLIER,
           snapshotIds: [SNAPSHOT_ID],
         }));
@@ -678,7 +678,7 @@ describe('GitRepository', () => {
           serviceId: SERVICE_PROVIDER_ID,
           termsType: TERMS_TYPE,
           content: `${CONTENT} - updated 2`,
-          isExtractOnly: true,
+          isTechnicalUpgrade: true,
           fetchDate: FETCH_DATE_EARLIER,
           snapshotIds: [SNAPSHOT_ID],
           mimeType: HTML_MIME_TYPE,
@@ -1079,7 +1079,7 @@ describe('GitRepository', () => {
           serviceId: SERVICE_PROVIDER_ID,
           termsType: TERMS_TYPE,
           content: `${CONTENT} - updated 2`,
-          isExtractOnly: true,
+          isTechnicalUpgrade: true,
           fetchDate: FETCH_DATE_EARLIER,
           mimeType: HTML_MIME_TYPE,
         }));
@@ -1130,7 +1130,7 @@ describe('GitRepository', () => {
           serviceId: SERVICE_PROVIDER_ID,
           termsType: TERMS_TYPE,
           content: `${CONTENT} - updated 2`,
-          isExtractOnly: true,
+          isTechnicalUpgrade: true,
           fetchDate: FETCH_DATE_EARLIER,
           mimeType: HTML_MIME_TYPE,
         }));
@@ -1269,7 +1269,7 @@ describe('GitRepository', () => {
           serviceId: SERVICE_PROVIDER_ID,
           termsType: TERMS_TYPE,
           content: `${CONTENT} - updated 2`,
-          isExtractOnly: true,
+          isTechnicalUpgrade: true,
           fetchDate: FETCH_DATE_EARLIER,
           mimeType: HTML_MIME_TYPE,
         }));
@@ -1398,24 +1398,24 @@ describe('GitRepository', () => {
     after(() => subject.removeAll());
 
     describe('Records attributes', () => {
-      describe('#isExtractOnly', () => {
+      describe('#isTechnicalUpgrade', () => {
         context('records with deprecated message', () => {
           it('returns the proper value', async () => {
-            expect((await subject.findById(commits.deprecatedRefilter.id)).isExtractOnly).to.be.true;
+            expect((await subject.findById(commits.deprecatedRefilter.id)).isTechnicalUpgrade).to.be.true;
           });
 
           it('returns the proper value', async () => {
-            expect((await subject.findById(commits.deprecatedFirstRecord.id)).isExtractOnly).to.be.false;
+            expect((await subject.findById(commits.deprecatedFirstRecord.id)).isTechnicalUpgrade).to.be.false;
           });
         });
 
         context('record with current message', () => {
           it('returns the proper value', async () => {
-            expect((await subject.findById(commits.currentExtractOnly.id)).isExtractOnly).to.be.true;
+            expect((await subject.findById(commits.currentExtractOnly.id)).isTechnicalUpgrade).to.be.true;
           });
 
           it('returns the proper value', async () => {
-            expect((await subject.findById(commits.currentFirstRecord.id)).isExtractOnly).to.be.false;
+            expect((await subject.findById(commits.currentFirstRecord.id)).isTechnicalUpgrade).to.be.false;
           });
         });
       });

package/src/archivist/recorder/repositories/mongo/dataMapper.js

@@ -17,7 +17,7 @@ export function toPersistence(record) {
 }
 
 export function toDomain(mongoDocument) {
-  const { _id, serviceId, termsType, documentId, fetchDate, mimeType, isExtractOnly, isRefilter, isFirstRecord, snapshotIds, metadata } = mongoDocument;
+  const { _id, serviceId, termsType, documentId, fetchDate, mimeType, isTechnicalUpgrade, isExtractOnly, isRefilter, isFirstRecord, snapshotIds, metadata } = mongoDocument;
 
   const attributes = {
     id: _id.toString(),
@@ -27,7 +27,7 @@ export function toDomain(mongoDocument) {
     mimeType,
     fetchDate: new Date(fetchDate),
     isFirstRecord: Boolean(isFirstRecord),
-    isExtractOnly: Boolean(isExtractOnly) || Boolean(isRefilter),
+    isTechnicalUpgrade: Boolean(isTechnicalUpgrade) || Boolean(isExtractOnly) || Boolean(isRefilter),
     snapshotIds: snapshotIds?.map(snapshotId => snapshotId.toString()) || [],
     metadata,
   };

package/src/archivist/recorder/repositories/mongo/index.test.js

@@ -16,6 +16,7 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 const { connectionURI } = config.get('@opentermsarchive/engine.recorder.snapshots.storage.mongo');
 const client = new MongoClient(connectionURI);
+const isWindows = process.platform === 'win32';
 
 const SERVICE_PROVIDER_ID = 'test_service';
 const TERMS_TYPE = 'Terms of Service';
@@ -41,6 +42,16 @@ const METADATA = {
 let collection;
 
 describe('MongoRepository', () => {
+  before(function () {
+    if (isWindows) {
+      console.log('MongoDB tests are unstable on Windows due to race condition in connection cleanup.');
+      console.log('Lacking a production use case for Mongo on Windows, we skip tests. Please reach out if you have a use case.');
+      // On Windows, when multiple repositories connect to the same MongoDB server and are closed in parallel or even sequentially, unhandled "Operation interrupted because client was closed" errors occur after all tests pass.
+      // The issue does not occur on Linux or macOS, so it appears to be a platform-specific difference in how the MongoDB driver handles connection pool cleanup during client.close().
+      this.skip();
+    }
+  });
+
   let subject;
 
   context('Version', () => {
@@ -220,7 +231,7 @@ describe('MongoRepository', () => {
         });
       });
 
-      context('when it is an extracted only version', () => {
+      context('when it is an technical upgrade version', () => {
         const EXTRACTED_ONLY_CONTENT = `${CONTENT} extracted only`;
 
         before(async () => {
@@ -230,7 +241,7 @@ describe('MongoRepository', () => {
             content: CONTENT,
             fetchDate: FETCH_DATE_EARLIER,
             snapshotIds: [SNAPSHOT_ID],
-          })); // An extracted only version cannot be the first record
+          })); // An technical upgrade version cannot be the first record
 
           numberOfRecordsBefore = await collection.countDocuments({
             serviceId: SERVICE_PROVIDER_ID,
@@ -243,7 +254,7 @@ describe('MongoRepository', () => {
             content: EXTRACTED_ONLY_CONTENT,
             fetchDate: FETCH_DATE,
             snapshotIds: [SNAPSHOT_ID],
-            isExtractOnly: true,
+            isTechnicalUpgrade: true,
           })));
 
           numberOfRecordsAfter = await collection.countDocuments({
@@ -267,8 +278,8 @@ describe('MongoRepository', () => {
           expect(mongoDocument._id.toString()).to.equal(record.id);
         });
 
-        it('stores information that it is an extracted only version', () => {
-          expect(mongoDocument.isExtractOnly).to.be.true;
+        it('stores information that it is an technical upgrade version', () => {
+          expect(mongoDocument.isTechnicalUpgrade).to.be.true;
         });
       });
 
@@ -596,7 +607,7 @@ describe('MongoRepository', () => {
           serviceId: SERVICE_PROVIDER_ID,
           termsType: TERMS_TYPE,
           content: `${CONTENT} - updated 2`,
-          isExtractOnly: true,
+          isTechnicalUpgrade: true,
           fetchDate: FETCH_DATE_EARLIER,
           snapshotIds: [SNAPSHOT_ID],
         }));
@@ -645,7 +656,7 @@ describe('MongoRepository', () => {
           serviceId: SERVICE_PROVIDER_ID,
           termsType: TERMS_TYPE,
           content: `${CONTENT} - updated 2`,
-          isExtractOnly: true,
+          isTechnicalUpgrade: true,
           fetchDate: FETCH_DATE_EARLIER,
           snapshotIds: [SNAPSHOT_ID],
         }));
@@ -810,7 +821,7 @@ describe('MongoRepository', () => {
           serviceId: SERVICE_PROVIDER_ID,
           termsType: TERMS_TYPE,
           content: `${CONTENT} - updated 2`,
-          isExtractOnly: true,
+          isTechnicalUpgrade: true,
           fetchDate: FETCH_DATE_EARLIER,
           snapshotIds: [SNAPSHOT_ID],
         }));
@@ -1164,7 +1175,7 @@ describe('MongoRepository', () => {
           serviceId: SERVICE_PROVIDER_ID,
           termsType: TERMS_TYPE,
           content: `${CONTENT} - updated 2`,
-          isExtractOnly: true,
+          isTechnicalUpgrade: true,
           fetchDate: FETCH_DATE_EARLIER,
           mimeType: HTML_MIME_TYPE,
         }));
@@ -1213,7 +1224,7 @@ describe('MongoRepository', () => {
           serviceId: SERVICE_PROVIDER_ID,
           termsType: TERMS_TYPE,
           content: `${CONTENT} - updated 2`,
-          isExtractOnly: true,
+          isTechnicalUpgrade: true,
           fetchDate: FETCH_DATE_EARLIER,
           mimeType: HTML_MIME_TYPE,
         }));
@@ -1421,7 +1432,7 @@ describe('MongoRepository', () => {
           serviceId: SERVICE_PROVIDER_ID,
           termsType: TERMS_TYPE,
           content: `${CONTENT} - updated 2`,
-          isExtractOnly: true,
+          isTechnicalUpgrade: true,
           fetchDate: FETCH_DATE_EARLIER,
           mimeType: HTML_MIME_TYPE,
         }));

package/src/collection-api/server.js

@@ -19,7 +19,7 @@ app.use(errorsMiddleware);
 
 const port = config.get('@opentermsarchive/engine.collection-api.port');
 
-app.listen(port);
+app.listen(port, '127.0.0.1');
 
 if (process.env.NODE_ENV !== 'test') {
   logger.info(`Start Open Terms Archive API on http://localhost:${port}${BASE_PATH}`);

package/src/index.js

@@ -13,7 +13,7 @@ import Reporter from './reporter/index.js';
 const require = createRequire(import.meta.url);
 const { version: PACKAGE_VERSION } = require('../package.json');
 
-export default async function track({ services, types, extractOnly, schedule }) {
+async function initialize(services) {
   const archivist = new Archivist({
     recorderConfig: config.get('@opentermsarchive/engine.recorder'),
     fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
@@ -40,13 +40,17 @@ export default async function track({ services, types, extractOnly, schedule })
     });
   }
 
-  // The result of the extraction step that generates the version from the snapshots may depend on changes to the engine or its dependencies.
-  // The process thus starts by only performing the extraction process so that any version following such changes can be labelled (to avoid sending notifications, for example)
-  await archivist.track({ services, types, extractOnly: true });
+  return { archivist, services };
+}
 
-  if (extractOnly) {
-    return;
-  }
+export default async function track({ services, types, schedule }) {
+  const { archivist, services: filteredServices } = await initialize(services);
+
+  // Technical upgrade pass: apply changes from engine, dependency, or declaration upgrades.
+  // This regenerates versions from existing snapshots with updated extraction logic.
+  // For terms with combined source documents, if a new document was added to the declaration, it will be fetched and combined with existing snapshots to regenerate the complete version.
+  // All versions from this pass are labeled as technical upgrades to avoid false notifications about content changes.
+  await archivist.applyTechnicalUpgrades({ services: filteredServices, types });
 
   if (process.env.OTA_ENGINE_SENDINBLUE_API_KEY) {
     try {
@@ -72,7 +76,7 @@ export default async function track({ services, types, extractOnly, schedule })
   }
 
   if (!schedule) {
-    await archivist.track({ services, types });
+    await archivist.track({ services: filteredServices, types });
 
     return;
   }
@@ -86,6 +90,12 @@ export default async function track({ services, types, extractOnly, schedule })
   new Cron( // eslint-disable-line no-new
     trackingSchedule,
     { protect: job => logger.warn(`Tracking scheduled at ${new Date().toISOString()} were blocked by an unfinished tracking started at ${job.currentRun().toISOString()}`) },
-    () => archivist.track({ services, types }),
+    () => archivist.track({ services: filteredServices, types }),
   );
 }
+
+export async function applyTechnicalUpgrades({ services, types }) {
+  const { archivist, services: filteredServices } = await initialize(services);
+
+  await archivist.applyTechnicalUpgrades({ services: filteredServices, types });
+}

package/src/logger/index.js

@@ -195,9 +195,9 @@ logger.onVersionNotChanged = ({ serviceId, termsType }) => {
   logger.info({ message: 'No changes after filtering, did not record version', serviceId, termsType });
 };
 
-logger.onTrackingStarted = (numberOfServices, numberOfTerms, extractOnly) => {
-  if (extractOnly) {
-    logger.info(`Examining ${numberOfTerms} terms from ${numberOfServices} services for extraction…`);
+logger.onTrackingStarted = (numberOfServices, numberOfTerms, technicalUpgradeOnly) => {
+  if (technicalUpgradeOnly) {
+    logger.info(`Applying technical upgrades to ${numberOfTerms} terms from ${numberOfServices} services…`);
   } else {
     logger.info(`Tracking changes of ${numberOfTerms} terms from ${numberOfServices} services…`);
   }
@@ -206,11 +206,11 @@ logger.onTrackingStarted = (numberOfServices, numberOfTerms, extractOnly) => {
   trackingStartTime = Date.now();
 };
 
-logger.onTrackingCompleted = (numberOfServices, numberOfTerms, extractOnly) => {
+logger.onTrackingCompleted = (numberOfServices, numberOfTerms, technicalUpgradeOnly) => {
   const duration = formatDuration(Date.now() - trackingStartTime);
 
-  if (extractOnly) {
-    logger.info(`Examined ${numberOfTerms} terms from ${numberOfServices} services for extraction in ${duration}`);
+  if (technicalUpgradeOnly) {
+    logger.info(`Applied technical upgrades to ${numberOfTerms} terms from ${numberOfServices} services in ${duration}`);
     logger.info(`Recorded ${recordedVersionsCount} new versions\n`);
   } else {
     logger.info(`Tracked changes of ${numberOfTerms} terms from ${numberOfServices} services in ${duration}`);

package/src/reporter/gitlab/index.js

@@ -1,5 +1,5 @@
-import HttpProxyAgent from 'http-proxy-agent';
-import HttpsProxyAgent from 'https-proxy-agent';
+import { HttpProxyAgent } from 'http-proxy-agent';
+import { HttpsProxyAgent } from 'https-proxy-agent';
 import nodeFetch from 'node-fetch';
 
 import logger from '../../logger/index.js';