@opentermsarchive/engine 0.26.1 → 0.27.1

package/README.md

@@ -1,8 +1,6 @@
-_The document you are reading now is targeted at developers wanting to use or contribute to the engine of [Open Terms Archive](https://opentermsarchive.org). For a high-level overview of Open Terms Archive’s wider goals and processes, please read its [public homepage](https://opentermsarchive.org)._
-
 # Open Terms Archive Engine
 
-This codebase is a Node.js module enabling downloading, archiving and publishing versions of documents obtained online. It can be used independently from the Open Terms Archive ecosystem.
+This codebase is a Node.js module enabling downloading, archiving and publishing versions of documents obtained online. It can be used independently from the Open Terms Archive ecosystem. For a high-level overview of Open Terms Archive’s wider goals and processes, please read its [public homepage](https://opentermsarchive.org).
 
 For documentation, visit [docs.opentermsarchive.org](https://docs.opentermsarchive.org/)
 

package/bin/ota-track.js

@@ -14,8 +14,8 @@ program
   .name('ota track')
   .description('Retrieve declared documents, record snapshots, extract versions and publish the resulting records')
   .option('-s, --services [serviceId...]', 'service IDs of services to track')
-  .option('-t, --terms-types [termsType...]', 'terms types to track')
-  .option('-r, --refilter-only', 'refilter existing snapshots with latest declarations and engine, without recording new snapshots')
-  .option('--schedule', 'schedule automatic document tracking');
+  .option('-t, --types [termsType...]', 'terms types to track')
+  .option('-e, --extract-only', 'extract versions from existing snapshots with latest declarations and engine, without recording new snapshots')
+  .option('--schedule', 'track automatically at a regular interval');
 
 track(program.parse(process.argv).opts());

package/bin/ota-validate.js

@@ -20,9 +20,9 @@ process.on('unhandledRejection', reason => {
 
 program
   .name('ota validate')
-  .description('Run a series of tests to check the validity of document declarations')
+  .description('Run a series of tests to check the validity of terms declarations')
   .option('-s, --services [serviceId...]', 'service IDs of services to validate')
-  .option('-t, --terms-types [termsType...]', 'terms types to validate')
+  .option('-t, --types [termsType...]', 'terms types to validate')
   .option('-m, --modified', 'target only services modified in the current git branch')
   .option('-o, --schema-only', 'much faster check of declarations, but does not check that the documents are actually accessible');
 

package/bin/ota.js

@@ -11,7 +11,7 @@ program
   .description(description)
   .version(version)
   .command('track', 'Track the current terms of services according to provided declarations')
-  .command('validate', 'Run a series of tests to check the validity of document declarations')
+  .command('validate', 'Run a series of tests to check the validity of terms declarations')
   .command('lint', 'Check format and stylistic errors in declarations and auto fix them')
   .command('dataset', 'Export the versions dataset into a ZIP file and optionally publish it to GitHub releases')
   .parse(process.argv);

package/config/default.json

@@ -60,7 +60,7 @@
       "label": {
         "name": "bot-report",
         "color": "FEF2C0",
-        "description": "Automatically created by Open Terms Archive when a document cannot be fetched"
+        "description": "Automatically created when terms cannot be tracked"
       }
     }
   },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "0.26.1",
+  "version": "0.27.1",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://github.com/OpenTermsArchive/engine#readme",
   "bugs": {
@@ -16,8 +16,8 @@
   "exports": {
     ".": "./src/exports.js",
     "./fetch": "./src/archivist/fetcher/exports.js",
-    "./filter": "./src/archivist/filter/exports.js",
-    "./page-declaration": "./src/archivist/services/pageDeclaration.js"
+    "./extract": "./src/archivist/extract/exports.js",
+    "./sourceDocument": "./src/archivist/services/sourceDocument.js"
   },
   "bin": {
     "ota": "./bin/ota.js"
@@ -38,7 +38,6 @@
     "declarations:validate:schema": "npm run declarations:validate -- --schema-only",
     "lint": "eslint src test scripts bin",
     "lint:fix": "npm run lint -- --fix",
-    "refilter": "npm start -- --refilter-only",
     "start": "node --max-http-header-size=32768 bin/ota.js track",
     "start:scheduler": "npm start -- --schedule",
     "test": "cross-env NODE_ENV=test mocha --recursive \"./src/**/*.test.js\" \"./scripts/**/*.test.js\" --exit",

package/scripts/dataset/export/index.js

@@ -31,7 +31,7 @@ export default async function generate({ archivePath, releaseDate }) {
 
   for await (const version of versionsRepository.iterate()) {
     const { content, fetchDate } = version;
-    const { serviceId, documentType } = renamer.applyRules(version.serviceId, version.documentType);
+    const { serviceId, termsType } = renamer.applyRules(version.serviceId, version.termsType);
 
     if (firstVersionDate > fetchDate) {
       firstVersionDate = fetchDate;
@@ -43,7 +43,7 @@ export default async function generate({ archivePath, releaseDate }) {
 
     services.add(serviceId);
 
-    const versionPath = generateVersionPath({ serviceId, documentType, fetchDate });
+    const versionPath = generateVersionPath({ serviceId, termsType, fetchDate });
 
     logger.info({ message: versionPath, counter: index, hash: version.id });
 
@@ -97,10 +97,10 @@ async function initializeArchive(targetPath) {
   return { basename, stream, done };
 }
 
-function generateVersionPath({ serviceId, documentType, fetchDate }) {
+function generateVersionPath({ serviceId, termsType, fetchDate }) {
   const fsCompliantDate = fetchDate.toISOString()
     .replace(/\.\d{3}/, '') // remove milliseconds
     .replace(/:|\./g, '-'); // replace `:` and `.` by `-` to be compliant with the file system
 
-  return `${serviceId}/${documentType}/${fsCompliantDate}.md`;
+  return `${serviceId}/${termsType}/${fsCompliantDate}.md`;
 }

package/scripts/dataset/export/index.test.js

@@ -8,8 +8,8 @@ import dircompare from 'dir-compare';
 import mime from 'mime';
 import StreamZip from 'node-stream-zip';
 
-import Record from '../../../src/archivist/recorder/record.js';
 import GitRepository from '../../../src/archivist/recorder/repositories/git/index.js';
+import Version from '../../../src/archivist/recorder/version.js';
 
 import generateArchive from './index.js';
 
@@ -20,8 +20,8 @@ const { expect } = chai;
 const FIRST_SERVICE_PROVIDER_ID = 'ServiceA';
 const SECOND_SERVICE_PROVIDER_ID = 'ServiceB';
 
-const FIRST_DOCUMENT_TYPE = 'Terms of Service';
-const SECOND_DOCUMENT_TYPE = 'Privacy Policy';
+const FIRST_TERMS_TYPE = 'Terms of Service';
+const SECOND_TERMS_TYPE = 'Privacy Policy';
 
 const FIRST_FETCH_DATE = '2021-01-01T11:27:00.000Z';
 const SECOND_FETCH_DATE = '2021-01-11T11:32:47.000Z';
@@ -31,8 +31,6 @@ const FOURTH_FETCH_DATE = '2022-01-01T12:12:24.000Z';
 const FIRST_CONTENT = 'First Content';
 const SECOND_CONTENT = 'Second Content';
 
-const MIME_TYPE = 'text/markdown';
-
 const SNAPSHOT_ID = '721ce4a63ad399ecbdb548a66d6d327e7bc97876';
 
 const RELEASE_DATE = '2022-01-01T18:21:00.000Z';
@@ -56,38 +54,34 @@ describe('Export', () => {
 
       await repository.initialize();
 
-      await repository.save(new Record({
+      await repository.save(new Version({
         serviceId: FIRST_SERVICE_PROVIDER_ID,
-        documentType: FIRST_DOCUMENT_TYPE,
+        termsType: FIRST_TERMS_TYPE,
         content: FIRST_CONTENT,
-        mimeType: MIME_TYPE,
         fetchDate: FIRST_FETCH_DATE,
         snapshotId: SNAPSHOT_ID,
       }));
 
-      await repository.save(new Record({
+      await repository.save(new Version({
         serviceId: FIRST_SERVICE_PROVIDER_ID,
-        documentType: FIRST_DOCUMENT_TYPE,
+        termsType: FIRST_TERMS_TYPE,
         content: SECOND_CONTENT,
-        mimeType: MIME_TYPE,
         fetchDate: SECOND_FETCH_DATE,
         snapshotId: SNAPSHOT_ID,
       }));
 
-      await repository.save(new Record({
+      await repository.save(new Version({
         serviceId: SECOND_SERVICE_PROVIDER_ID,
-        documentType: FIRST_DOCUMENT_TYPE,
+        termsType: FIRST_TERMS_TYPE,
         content: FIRST_CONTENT,
-        mimeType: MIME_TYPE,
         fetchDate: THIRD_FETCH_DATE,
         snapshotId: SNAPSHOT_ID,
       }));
 
-      await repository.save(new Record({
+      await repository.save(new Version({
         serviceId: SECOND_SERVICE_PROVIDER_ID,
-        documentType: SECOND_DOCUMENT_TYPE,
+        termsType: SECOND_TERMS_TYPE,
         content: FIRST_CONTENT,
-        mimeType: MIME_TYPE,
         fetchDate: FOURTH_FETCH_DATE,
         snapshotId: SNAPSHOT_ID,
       }));

package/scripts/declarations/lint/index.mocha.js

@@ -31,7 +31,7 @@ export default async options => {
   if (options.modified) {
     const declarationUtils = new DeclarationUtils(instancePath);
 
-    ({ services: servicesToValidate } = await declarationUtils.getModifiedServiceDocumentTypes());
+    ({ services: servicesToValidate } = await declarationUtils.getModifiedServiceTermsTypes());
   }
 
   const lintFile = lintAndFixFile(options.fix);

package/scripts/declarations/utils/index.js

@@ -24,27 +24,27 @@ export default class DeclarationUtils {
 
     const modifiedFilePaths = modifiedFilePathsAsString ? modifiedFilePathsAsString.split('\n') : [];
 
-    return { modifiedFilePaths, modifiedServiceIds: Array.from(new Set(modifiedFilePaths.map(DeclarationUtils.filePathToServiceId))) };
+    return { modifiedFilePaths, modifiedServicesIds: Array.from(new Set(modifiedFilePaths.map(DeclarationUtils.filePathToServiceId))) };
   }
 
   async getModifiedServices() {
-    const { modifiedServiceIds } = await this.getModifiedData();
+    const { modifiedServicesIds } = await this.getModifiedData();
 
-    return modifiedServiceIds;
+    return modifiedServicesIds;
   }
 
-  async getModifiedServiceDocumentTypes() {
-    const { modifiedFilePaths, modifiedServiceIds } = await this.getModifiedData();
-    const servicesDocumentTypes = {};
+  async getModifiedServiceTermsTypes() {
+    const { modifiedFilePaths, modifiedServicesIds } = await this.getModifiedData();
+    const servicesTermsTypes = {};
 
     await Promise.all(modifiedFilePaths.map(async modifiedFilePath => {
       const serviceId = DeclarationUtils.filePathToServiceId(modifiedFilePath);
 
       if (!modifiedFilePath.endsWith('.json')) {
         // Here we should compare AST of both files to detect on which function
-        // change has been made, and then find which document type depends on this
+        // change has been made, and then find which terms type depends on this
         // function.
-        // As this is a complicated process, we will just send back all document types
+        // As this is a complicated process, we will just send back all terms types
         const declaration = await this.getJSONFile(`declarations/${serviceId}.json`, this.defaultBranch);
 
         return Object.keys(declaration.documents);
@@ -60,7 +60,7 @@ export default class DeclarationUtils {
         return;
       }
 
-      const modifiedDocumentTypes = diff.reduce((acc, { path }) => {
+      const modifiedTermsTypes = diff.reduce((acc, { path }) => {
         if (modifiedFilePath.includes('.history')) {
           acc.add(path[0]);
         } else if (path[0] == 'documents') {
@@ -70,12 +70,12 @@ export default class DeclarationUtils {
         return acc;
       }, new Set());
 
-      servicesDocumentTypes[serviceId] = Array.from(new Set([ ...servicesDocumentTypes[serviceId] || [], ...modifiedDocumentTypes ]));
+      servicesTermsTypes[serviceId] = Array.from(new Set([ ...servicesTermsTypes[serviceId] || [], ...modifiedTermsTypes ]));
     }));
 
     return {
-      services: modifiedServiceIds,
-      servicesDocumentTypes,
+      services: modifiedServicesIds,
+      servicesTermsTypes,
     };
   }
 }

package/scripts/declarations/validate/definitions.js

@@ -25,7 +25,7 @@ const definitions = {
     ],
   },
   contentSelectors: { $ref: '#/definitions/selectors' },
-  noiseSelectors: { $ref: '#/definitions/selectors' },
+  insignificantContentSelectors: { $ref: '#/definitions/selectors' },
   filters: {
     type: 'array',
     items: {

package/scripts/declarations/validate/index.mocha.js

@@ -6,8 +6,8 @@ import { expect } from 'chai';
 import config from 'config';
 import jsonSourceMap from 'json-source-map';
 
+import extract from '../../../src/archivist/extract/index.js';
 import fetch, { launchHeadlessBrowser, stopHeadlessBrowser } from '../../../src/archivist/fetcher/index.js';
-import filter from '../../../src/archivist/filter/index.js';
 import * as services from '../../../src/archivist/services/index.js';
 import DeclarationUtils from '../utils/index.js';
 
@@ -25,8 +25,8 @@ const instancePath = path.resolve(declarationsPath, '../');
 export default async options => {
   const schemaOnly = options.schemaOnly || false;
   let servicesToValidate = options.services || [];
-  const documentTypes = options.termsTypes || [];
-  let servicesDocumentTypes = {};
+  const termsTypes = options.types || [];
+  let servicesTermsTypes = {};
 
   const serviceDeclarations = await services.loadWithHistory(servicesToValidate);
 
@@ -37,7 +37,7 @@ export default async options => {
   if (options.modified) {
     const declarationUtils = new DeclarationUtils(instancePath);
 
-    ({ services: servicesToValidate, servicesDocumentTypes } = await declarationUtils.getModifiedServiceDocumentTypes());
+    ({ services: servicesToValidate, servicesTermsTypes } = await declarationUtils.getModifiedServiceTermsTypes());
   }
 
   describe('Service declarations validation', async function () {
@@ -76,61 +76,57 @@ export default async options => {
         }
 
         if (!schemaOnly && service) {
-          service.getDocumentTypes()
-            .filter(documentType => {
-              if (servicesDocumentTypes[serviceId] && servicesDocumentTypes[serviceId].length > 0) {
-                return servicesDocumentTypes[serviceId].includes(documentType);
+          service.getTermsTypes()
+            .filter(termsType => {
+              if (servicesTermsTypes[serviceId] && servicesTermsTypes[serviceId].length > 0) {
+                return servicesTermsTypes[serviceId].includes(termsType);
               }
 
-              if (documentTypes.length > 0) {
-                return documentTypes.includes(documentType);
+              if (termsTypes.length > 0) {
+                return termsTypes.includes(termsType);
               }
 
               return true;
             })
             .forEach(type => {
               describe(type, () => {
-                const documentDeclaration = service.getDocumentDeclaration(type);
+                const terms = service.getTerms(type);
 
-                documentDeclaration.pages.forEach(page => {
-                  let content;
+                terms.sourceDocuments.forEach(sourceDocument => {
                   let filteredContent;
-                  let mimeType;
 
-                  context(page.location, () => {
+                  context(sourceDocument.location, () => {
                     before(async function () {
-                      if (!documentDeclaration) {
+                      if (!terms) {
                         console.log('      (Tests skipped as declaration has been archived)');
                         this.skip();
                       }
                     });
 
                     it('fetchable URL', async () => {
-                      const { location, executeClientScripts } = page;
-                      const document = await fetch({
+                      const { location, executeClientScripts } = sourceDocument;
+
+                      ({ content: sourceDocument.content, mimeType: sourceDocument.mimeType } = await fetch({
                         url: location,
                         executeClientScripts,
-                        cssSelectors: page.cssSelectors,
+                        cssSelectors: sourceDocument.cssSelectors,
                         config: config.get('fetcher'),
-                      });
-
-                      content = document.content;
-                      mimeType = document.mimeType;
+                      }));
                     });
 
-                    it('selector matches an element in the web page', async function checkSelector() {
-                      if (!content) {
+                    it('selector matches an element in the source document', async function checkSelector() {
+                      if (!sourceDocument.content) {
                         console.log('          [Tests skipped as URL is not fetchable]');
                         this.skip();
                       }
 
-                      filteredContent = await filter({ content, pageDeclaration: page, mimeType });
+                      filteredContent = await extract(sourceDocument);
 
                       expect(filteredContent).to.not.be.empty;
                     });
 
                     it(`filtered content has at least ${MIN_DOC_LENGTH} characters`, async function checkContentLength() {
-                      if (!content) {
+                      if (!sourceDocument.content) {
                         console.log('          [Tests skipped as URL is not fetchable]');
                         this.skip();
                       }
@@ -146,7 +142,7 @@ export default async options => {
                     it('content is consistent when fetched and filtered twice in a row', async function checkContentConsistency() {
                       this.slow(SLOW_DOCUMENT_THRESHOLD * 2);
 
-                      if (!content) {
+                      if (!sourceDocument.content) {
                         console.log('          [Tests skipped as URL is not fetchable]');
                         this.skip();
                       }
@@ -156,13 +152,13 @@ export default async options => {
                         this.skip();
                       }
 
-                      const document = await fetch({
-                        url: page.location,
-                        executeClientScripts: page.executeClientScripts,
-                        cssSelectors: page.cssSelectors,
+                      ({ content: sourceDocument.content, mimeType: sourceDocument.mimeType } = await fetch({
+                        url: sourceDocument.location,
+                        executeClientScripts: sourceDocument.executeClientScripts,
+                        cssSelectors: sourceDocument.cssSelectors,
                         config: config.get('fetcher'),
-                      });
-                      const secondFilteredContent = await filter({ content: document.content, pageDeclaration: page, mimeType: document.mimeType });
+                      }));
+                      const secondFilteredContent = await extract(sourceDocument);
 
                       expect(secondFilteredContent).to.equal(filteredContent);
                     });

package/scripts/declarations/validate/service.history.schema.js

@@ -1,10 +1,10 @@
-import { DOCUMENT_TYPES } from '../../../src/archivist/services/index.js';
+import TERMS_TYPES from '@opentermsarchive/terms-types';
 
 import definitions from './definitions.js';
 
-const AVAILABLE_TYPES_NAME = Object.keys(DOCUMENT_TYPES);
+const AVAILABLE_TYPES_NAME = Object.keys(TERMS_TYPES);
 
-const documentsProperties = () => {
+const termsProperties = () => {
   const result = {};
 
   AVAILABLE_TYPES_NAME.forEach(type => {
@@ -12,8 +12,8 @@ const documentsProperties = () => {
       type: 'array',
       items: {
         oneOf: [
-          { $ref: '#/definitions/singlePageDocumentHistory' },
-          { $ref: '#/definitions/multiPageDocumentHistory' },
+          { $ref: '#/definitions/singleSourceDocumentTermsHistory' },
+          { $ref: '#/definitions/multipleSourceDocumentsTermsHistory' },
           { $ref: '#/definitions/pdfDocumentHistory' },
         ],
       },
@@ -27,7 +27,7 @@ const schema = {
   type: 'object',
   additionalProperties: false,
   title: 'Service declaration history',
-  properties: documentsProperties(),
+  properties: termsProperties(),
   propertyNames: { enum: AVAILABLE_TYPES_NAME },
   definitions: {
     ...definitions,
@@ -40,7 +40,7 @@ const schema = {
         validUntil: { $ref: '#/definitions/validUntil' },
       },
     },
-    singlePageDocumentHistory: {
+    singleSourceDocumentTermsHistory: {
       type: 'object',
       additionalProperties: false,
       required: [ 'fetch', 'select', 'validUntil' ],
@@ -48,12 +48,12 @@ const schema = {
         fetch: { $ref: '#/definitions/location' },
         select: { $ref: '#/definitions/contentSelectors' },
         filter: { $ref: '#/definitions/filters' },
-        remove: { $ref: '#/definitions/noiseSelectors' },
+        remove: { $ref: '#/definitions/insignificantContentSelectors' },
         executeClientScripts: { $ref: '#/definitions/executeClientScripts' },
         validUntil: { $ref: '#/definitions/validUntil' },
       },
     },
-    multiPageDocumentHistory: {
+    multipleSourceDocumentsTermsHistory: {
       type: 'object',
       additionalProperties: false,
       required: ['combine'],
@@ -68,14 +68,14 @@ const schema = {
               fetch: { $ref: '#/definitions/location' },
               select: { $ref: '#/definitions/contentSelectors' },
               filter: { $ref: '#/definitions/filters' },
-              remove: { $ref: '#/definitions/noiseSelectors' },
+              remove: { $ref: '#/definitions/insignificantContentSelectors' },
               executeClientScripts: { $ref: '#/definitions/executeClientScripts' },
             },
           },
         },
         select: { $ref: '#/definitions/contentSelectors' },
         filter: { $ref: '#/definitions/filters' },
-        remove: { $ref: '#/definitions/noiseSelectors' },
+        remove: { $ref: '#/definitions/insignificantContentSelectors' },
         executeClientScripts: { $ref: '#/definitions/executeClientScripts' },
         validUntil: { $ref: '#/definitions/validUntil' },
       },

package/scripts/declarations/validate/service.schema.js

@@ -1,17 +1,17 @@
-import { DOCUMENT_TYPES } from '../../../src/archivist/services/index.js';
+import TERMS_TYPES from '@opentermsarchive/terms-types';
 
 import definitions from './definitions.js';
 
-const AVAILABLE_TYPES_NAME = Object.keys(DOCUMENT_TYPES);
+const AVAILABLE_TYPES_NAME = Object.keys(TERMS_TYPES);
 
-const documentsProperties = () => {
+const termsProperties = () => {
   const result = {};
 
   AVAILABLE_TYPES_NAME.forEach(type => {
     result[type] = {
       oneOf: [
-        { $ref: '#/definitions/singlePageDocument' },
-        { $ref: '#/definitions/multiPageDocument' },
+        { $ref: '#/definitions/singleSourceDocumentTerms' },
+        { $ref: '#/definitions/multipleSourceDocumentsTerms' },
         { $ref: '#/definitions/pdfDocument' },
       ],
     };
@@ -33,7 +33,7 @@ const schema = {
     },
     documents: {
       type: 'object',
-      properties: documentsProperties(),
+      properties: termsProperties(),
       propertyNames: { enum: AVAILABLE_TYPES_NAME },
     },
     importedFrom: {
@@ -52,7 +52,7 @@ const schema = {
       required: ['fetch'],
       properties: { fetch: { $ref: '#/definitions/pdfLocation' } },
     },
-    page: {
+    sourceDocument: {
       type: 'object',
       additionalProperties: false,
       required: ['fetch'],
@@ -60,28 +60,28 @@ const schema = {
         fetch: { $ref: '#/definitions/location' },
         select: { $ref: '#/definitions/contentSelectors' },
         filter: { $ref: '#/definitions/filters' },
-        remove: { $ref: '#/definitions/noiseSelectors' },
+        remove: { $ref: '#/definitions/insignificantContentSelectors' },
         executeClientScripts: { $ref: '#/definitions/executeClientScripts' },
       },
     },
-    singlePageDocument: {
+    singleSourceDocumentTerms: {
       allOf: [
-        { $ref: '#/definitions/page' },
+        { $ref: '#/definitions/sourceDocument' },
         { required: [ 'fetch', 'select' ] },
       ],
     },
-    multiPageDocument: {
+    multipleSourceDocumentsTerms: {
       type: 'object',
       additionalProperties: false,
       required: ['combine'],
       properties: {
         combine: {
           type: 'array',
-          items: { $ref: '#/definitions/page' },
+          items: { $ref: '#/definitions/sourceDocument' },
         },
         select: { $ref: '#/definitions/contentSelectors' },
         filter: { $ref: '#/definitions/filters' },
-        remove: { $ref: '#/definitions/noiseSelectors' },
+        remove: { $ref: '#/definitions/insignificantContentSelectors' },
         executeClientScripts: { $ref: '#/definitions/executeClientScripts' },
       },
     },

package/scripts/history/migrate-services.js

@@ -145,10 +145,10 @@ async function rewriteSnapshots(repository, records, idsMapping, logger) {
     idsMapping[record.id] = recordId; // Saves the mapping between the old ID and the new one.
 
     if (recordId) {
-      logger.info({ message: `Migrated snapshot with new ID: ${recordId}`, serviceId: record.serviceId, type: record.documentType, id: record.id, current: i++, total: records.length });
+      logger.info({ message: `Migrated snapshot with new ID: ${recordId}`, serviceId: record.serviceId, type: record.termsType, id: record.id, current: i++, total: records.length });
       counters.migrated++;
     } else {
-      logger.info({ message: 'Skipped snapshot', serviceId: record.serviceId, type: record.documentType, id: record.id, current: i++, total: records.length });
+      logger.info({ message: 'Skipped snapshot', serviceId: record.serviceId, type: record.termsType, id: record.id, current: i++, total: records.length });
       counters.skipped++;
     }
   }
@@ -169,10 +169,10 @@ async function rewriteVersions(repository, records, idsMapping, logger) {
     const { id: recordId } = await repository.save(record); // eslint-disable-line no-await-in-loop
 
     if (recordId) {
-      logger.info({ message: `Migrated version with new ID: ${recordId}`, serviceId: record.serviceId, type: record.documentType, id: record.id, current: i++, total: records.length });
+      logger.info({ message: `Migrated version with new ID: ${recordId}`, serviceId: record.serviceId, type: record.termsType, id: record.id, current: i++, total: records.length });
       counters.migrated++;
     } else {
-      logger.info({ message: 'Skipped version', serviceId: record.serviceId, type: record.documentType, id: record.id, current: i++, total: records.length });
+      logger.info({ message: 'Skipped version', serviceId: record.serviceId, type: record.termsType, id: record.id, current: i++, total: records.length });
       counters.skipped++;
     }
   }

package/scripts/history/update-to-full-hash.js

@@ -47,9 +47,9 @@ const ROOT_PATH = path.resolve(__dirname, '../../');
     const { id: recordId } = await versionsTargetRepository.save(record);
 
     if (!recordId) {
-      logger.warn({ message: 'Record skipped', serviceId: record.serviceId, type: record.documentType, id: record.id, current, total });
+      logger.warn({ message: 'Record skipped', serviceId: record.serviceId, type: record.termsType, id: record.id, current, total });
     } else {
-      logger.info({ message: `Update short sha ${record.snapshotId} to ${fullSnapshotId}`, serviceId: record.serviceId, type: record.documentType, id: record.id, current, total });
+      logger.info({ message: `Update short sha ${record.snapshotId} to ${fullSnapshotId}`, serviceId: record.serviceId, type: record.termsType, id: record.id, current, total });
     }
 
     current++;