@opentermsarchive/engine 0.26.1 → 0.27.1

package/src/index.js

@@ -6,8 +6,11 @@ import logger from './logger/index.js';
 import Notifier from './notifier/index.js';
 import Tracker from './tracker/index.js';
 
-export default async function track({ services = [], termsTypes: documentTypes, refilterOnly, schedule }) {
-  const archivist = new Archivist({ recorderConfig: config.get('recorder') });
+export default async function track({ services, types, extractOnly, schedule }) {
+  const archivist = new Archivist({
+    recorderConfig: config.get('recorder'),
+    fetcherConfig: config.get('fetcher'),
+  });
 
   archivist.attach(logger);
 
@@ -15,11 +18,9 @@ export default async function track({ services = [], termsTypes: documentTypes,
 
   logger.info('Start Open Terms Archive\n');
 
-  let serviceIds;
-
-  if (services.length) {
-    serviceIds = services.filter(serviceId => {
-      const isServiceDeclared = archivist.serviceDeclarations[serviceId];
+  if (services?.length) {
+    services = services.filter(serviceId => {
+      const isServiceDeclared = archivist.services[serviceId];
 
       if (!isServiceDeclared) {
         logger.warn(`Parameter "${serviceId}" was interpreted as a service ID to update, but no matching declaration was found; it will be ignored`);
@@ -29,14 +30,16 @@ export default async function track({ services = [], termsTypes: documentTypes,
     });
   }
 
-  await archivist.refilterAndRecord(serviceIds, documentTypes);
+  // The result of the extraction step that generates the version from the snapshots may depend on changes to the engine or its dependencies.
+  // The process thus starts by only performing the extraction process so that any version following such changes can be labelled (to avoid sending notifications, for example)
+  await archivist.track({ services, types, extractOnly: true });
 
-  if (refilterOnly) {
+  if (extractOnly) {
     return;
   }
 
   if (process.env.NODE_ENV === 'production') {
-    archivist.attach(new Notifier(archivist.serviceDeclarations));
+    archivist.attach(new Notifier(archivist.services));
   }
 
   if (process.env.GITHUB_TOKEN) {
@@ -47,13 +50,13 @@ export default async function track({ services = [], termsTypes: documentTypes,
   }
 
   if (!schedule) {
-    await archivist.trackChanges(serviceIds, documentTypes);
+    await archivist.track({ services, types });
 
     return;
   }
 
   logger.info('The scheduler is running…');
-  logger.info('Documents will be tracked every six hours starting at half past midnight');
+  logger.info('Terms will be tracked every six hours starting at half past midnight');
 
-  cron('30 */6 * * *', () => archivist.trackChanges(serviceIds, documentTypes));
+  cron('30 */6 * * *', () => archivist.track({ services, types }));
 }

package/src/logger/index.js

@@ -11,15 +11,15 @@ const { combine, timestamp, printf, colorize } = winston.format;
 const alignedWithColorsAndTime = combine(
   colorize(),
   timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
-  printf(({ level, message, timestamp, serviceId, type, pageId }) => {
+  printf(({ level, message, timestamp, serviceId, termsType, documentId }) => {
     let prefix = '';
 
-    if (serviceId && type) {
-      prefix = `${serviceId} — ${type}`;
+    if (serviceId && termsType) {
+      prefix = `${serviceId} — ${termsType}`;
     }
 
-    if (pageId) {
-      prefix = `${prefix}:${pageId}`;
+    if (documentId) {
+      prefix = `${prefix}:${documentId}`;
     }
 
     if (prefix.length > 75) {
@@ -71,61 +71,60 @@ const logger = winston.createLogger({
   rejectionHandlers: transports,
 });
 
-logger.onFirstSnapshotRecorded = (serviceId, type, pageId, snapshotId) => {
-  logger.info({ message: `Recorded first snapshot with id ${snapshotId}`, serviceId, type, pageId });
+logger.onFirstSnapshotRecorded = ({ serviceId, termsType, documentId, id }) => {
+  logger.info({ message: `Recorded first snapshot with id ${id}`, serviceId, termsType, documentId });
   recordedSnapshotsCount++;
 };
 
-logger.onSnapshotRecorded = (serviceId, type, pageId, snapshotId) => {
-  logger.info({ message: `Recorded snapshot with id ${snapshotId}`, serviceId, type, pageId });
+logger.onSnapshotRecorded = ({ serviceId, termsType, documentId, id }) => {
+  logger.info({ message: `Recorded snapshot with id ${id}`, serviceId, termsType, documentId });
   recordedSnapshotsCount++;
 };
 
-logger.onSnapshotNotChanged = (serviceId, type, pageId) => {
-  logger.info({ message: 'No changes, did not record snapshot', serviceId, type, pageId });
+logger.onSnapshotNotChanged = ({ serviceId, termsType, documentId }) => {
+  logger.info({ message: 'No changes, did not record snapshot', serviceId, termsType, documentId });
 };
 
-logger.onFirstVersionRecorded = (serviceId, type, versionId) => {
-  logger.info({ message: `Recorded first version with id ${versionId}`, serviceId, type });
+logger.onFirstVersionRecorded = ({ serviceId, termsType, id }) => {
+  logger.info({ message: `Recorded first version with id ${id}`, serviceId, termsType });
   recordedVersionsCount++;
 };
 
-logger.onVersionRecorded = (serviceId, type, versionId) => {
-  logger.info({ message: `Recorded version with id ${versionId}`, serviceId, type });
+logger.onVersionRecorded = ({ serviceId, termsType, id }) => {
+  logger.info({ message: `Recorded version with id ${id}`, serviceId, termsType });
   recordedVersionsCount++;
 };
 
-logger.onVersionNotChanged = (serviceId, type) => {
-  logger.info({ message: 'No changes after filtering, did not record version', serviceId, type });
+logger.onVersionNotChanged = ({ serviceId, termsType }) => {
+  logger.info({ message: 'No changes after filtering, did not record version', serviceId, termsType });
 };
 
-logger.onRefilteringStarted = (numberOfServices, numberOfDocuments) => {
-  logger.info(`Examining ${numberOfDocuments} documents from ${numberOfServices} services for refiltering…`);
-  recordedVersionsCount = 0;
-};
-
-logger.onRefilteringCompleted = (numberOfServices, numberOfDocuments) => {
-  logger.info(`Examined ${numberOfDocuments} documents from ${numberOfServices} services for refiltering`);
-  logger.info(`Recorded ${recordedVersionsCount} new versions\n`);
-};
-
-logger.onTrackingStarted = (numberOfServices, numberOfDocuments) => {
-  logger.info(`Tracking changes of ${numberOfDocuments} documents from ${numberOfServices} services…`);
+logger.onTrackingStarted = (numberOfServices, numberOfTerms, extractOnly) => {
+  if (extractOnly) {
+    logger.info(`Examining ${numberOfTerms} terms from ${numberOfServices} services for extraction…`);
+  } else {
+    logger.info(`Tracking changes of ${numberOfTerms} terms from ${numberOfServices} services…`);
+  }
   recordedSnapshotsCount = 0;
   recordedVersionsCount = 0;
 };
 
-logger.onTrackingCompleted = (numberOfServices, numberOfDocuments) => {
-  logger.info(`Tracked changes of ${numberOfDocuments} documents from ${numberOfServices} services`);
-  logger.info(`Recorded ${recordedSnapshotsCount} new snapshots and ${recordedVersionsCount} new versions\n`);
+logger.onTrackingCompleted = (numberOfServices, numberOfTerms, extractOnly) => {
+  if (extractOnly) {
+    logger.info(`Examined ${numberOfTerms} terms from ${numberOfServices} services for extraction`);
+    logger.info(`Recorded ${recordedVersionsCount} new versions\n`);
+  } else {
+    logger.info(`Tracked changes of ${numberOfTerms} terms from ${numberOfServices} services`);
+    logger.info(`Recorded ${recordedSnapshotsCount} new snapshots and ${recordedVersionsCount} new versions\n`);
+  }
 };
 
-logger.onInaccessibleContent = ({ message }, serviceId, type) => {
-  logger.warn({ message, serviceId, type });
+logger.onInaccessibleContent = ({ message }, terms) => {
+  logger.warn({ message, serviceId: terms.serviceId, termsType: terms.type });
 };
 
-logger.onError = (error, serviceId, type, pageId) => {
-  logger.error({ message: error.stack, serviceId, type, pageId });
+logger.onError = (error, terms) => {
+  logger.error({ message: error.stack, serviceId: terms.serviceId, termsType: terms.type });
 };
 
 export default logger;

package/src/notifier/index.js

@@ -17,28 +17,28 @@ export default class Notifier {
     this.delayedVersionNotificationsParams = [];
   }
 
-  async onVersionRecorded(serviceId, type, versionId) {
-    this.delayedVersionNotificationsParams.push({ serviceId, type, versionId });
+  async onVersionRecorded(record) {
+    this.delayedVersionNotificationsParams.push(record);
   }
 
   async onTrackingCompleted() {
-    this.delayedVersionNotificationsParams.forEach(async ({ serviceId, type, versionId }) => {
+    this.delayedVersionNotificationsParams.forEach(async ({ serviceId, termsType, id }) => {
       try {
-        await this.notifyVersionRecorded(serviceId, type, versionId);
+        await this.notifyVersionRecorded(serviceId, termsType, id);
       } catch (error) {
-        console.error(`The notification for version ${versionId} of ${serviceId} — ${type} can not be sent:`, error);
+        console.error(`The notification for version ${id} of ${serviceId} — ${termsType} can not be sent:`, error);
       }
     });
 
     this.delayedVersionNotificationsParams = [];
   }
 
-  async notifyVersionRecorded(serviceProviderId, documentTypeId, versionId) {
+  async notifyVersionRecorded(serviceProviderId, termsType, versionId) {
     const sendParams = {
       templateId: config.get('notifier.sendInBlue.updateTemplateId'),
       params: {
         SERVICE_PROVIDER_NAME: this.serviceProviders[serviceProviderId].name,
-        DOCUMENT_TYPE: documentTypeId,
+        DOCUMENT_TYPE: termsType,
         SHA: versionId,
       },
     };
@@ -47,7 +47,7 @@ export default class Notifier {
       config.get('notifier.sendInBlue.updatesListId'),
     ];
 
-    const notificationListName = `${this.serviceProviders[serviceProviderId].name} - ${documentTypeId} - Update`;
+    const notificationListName = `${this.serviceProviders[serviceProviderId].name} - ${termsType} - Update`;
     const notificationList = await this.searchContactList(notificationListName);
 
     if (notificationList?.id) {

package/src/tracker/index.js

@@ -69,8 +69,8 @@ export default class Tracker {
     return this.onVersionRecorded(serviceId, type);
   }
 
-  async onInaccessibleContent(error, serviceId, type, documentDeclaration) {
-    const { title, body } = Tracker.formatIssueTitleAndBody({ message: error.toString(), repository: this.repository, documentDeclaration });
+  async onInaccessibleContent(error, serviceId, type, terms) {
+    const { title, body } = Tracker.formatIssueTitleAndBody({ message: error.toString(), repository: this.repository, terms });
 
     await this.createIssueIfNotExists({
       title,
@@ -179,9 +179,9 @@ export default class Tracker {
     }
   }
 
-  static formatIssueTitleAndBody({ message, repository, documentDeclaration }) {
-    const { service: { name }, type } = documentDeclaration;
-    const json = documentDeclaration.toPersistence();
+  static formatIssueTitleAndBody({ message, repository, terms }) {
+    const { service: { name }, type } = terms;
+    const json = terms.toPersistence();
     const title = `Fix ${name} - ${type}`;
 
     const encodedName = encodeURIComponent(name);
@@ -195,7 +195,7 @@ export default class Tracker {
     });
 
     const body = `
-This document is no longer properly tracked.
+These terms are no longer tracked.
 
 ${message}
 

package/src/archivist/filter/exports.js

@@ -1,3 +0,0 @@
-import filter from './index.js';
-
-export default filter;